1//===- ArithOps.cpp - MLIR Arith dialect ops implementation -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include <cassert>
10#include <cstdint>
11#include <functional>
12#include <type_traits>
13#include <utility>
14
15#include "mlir/Dialect/Arith/IR/Arith.h"
16#include "mlir/Dialect/CommonFolders.h"
17#include "mlir/Dialect/UB/IR/UBMatchers.h"
18#include "mlir/IR/Builders.h"
19#include "mlir/IR/BuiltinAttributeInterfaces.h"
20#include "mlir/IR/BuiltinAttributes.h"
21#include "mlir/IR/Matchers.h"
22#include "mlir/IR/OpImplementation.h"
23#include "mlir/IR/PatternMatch.h"
24#include "mlir/IR/TypeUtilities.h"
25#include "mlir/Interfaces/SideEffectInterfaces.h"
26#include "mlir/Support/LogicalResult.h"
27
28#include "llvm/ADT/APFloat.h"
29#include "llvm/ADT/APInt.h"
30#include "llvm/ADT/APSInt.h"
31#include "llvm/ADT/DenseMap.h"
32#include "llvm/ADT/FloatingPointMode.h"
33#include "llvm/ADT/STLExtras.h"
34#include "llvm/ADT/SmallVector.h"
35#include "llvm/ADT/TypeSwitch.h"
36
37using namespace mlir;
38using namespace mlir::arith;
39
40/// Default rounding mode according to default LLVM floating-point environment.
41static constexpr llvm::RoundingMode kDefaultRoundingMode =
42 llvm::RoundingMode::NearestTiesToEven;
43
44//===----------------------------------------------------------------------===//
45// Pattern helpers
46//===----------------------------------------------------------------------===//
47
48static IntegerAttr
49applyToIntegerAttrs(PatternRewriter &builder, Value res, Attribute lhs,
50 Attribute rhs,
51 function_ref<APInt(const APInt &, const APInt &)> binFn) {
52 const APInt &lhsVal = llvm::cast<IntegerAttr>(Val&: lhs).getValue();
53 const APInt &rhsVal = llvm::cast<IntegerAttr>(Val&: rhs).getValue();
54 APInt value = binFn(lhsVal, rhsVal);
55 return IntegerAttr::get(type: res.getType(), value);
56}
57
58static IntegerAttr addIntegerAttrs(PatternRewriter &builder, Value res,
59 Attribute lhs, Attribute rhs) {
60 return applyToIntegerAttrs(builder, res, lhs, rhs, binFn: std::plus<APInt>());
61}
62
63static IntegerAttr subIntegerAttrs(PatternRewriter &builder, Value res,
64 Attribute lhs, Attribute rhs) {
65 return applyToIntegerAttrs(builder, res, lhs, rhs, binFn: std::minus<APInt>());
66}
67
68static IntegerAttr mulIntegerAttrs(PatternRewriter &builder, Value res,
69 Attribute lhs, Attribute rhs) {
70 return applyToIntegerAttrs(builder, res, lhs, rhs, binFn: std::multiplies<APInt>());
71}
72
73static IntegerAttr andIntegerAttrs(PatternRewriter &builder, Value res,
74 Attribute lhs, Attribute rhs) {
75 return applyToIntegerAttrs(builder, res, lhs, rhs, binFn: std::bit_and<APInt>());
76}
77
78static IntegerAttr orIntegerAttrs(PatternRewriter &builder, Value res,
79 Attribute lhs, Attribute rhs) {
80 return applyToIntegerAttrs(builder, res, lhs, rhs, binFn: std::bit_or<APInt>());
81}
82
83static IntegerAttr xorIntegerAttrs(PatternRewriter &builder, Value res,
84 Attribute lhs, Attribute rhs) {
85 return applyToIntegerAttrs(builder, res, lhs, rhs, binFn: std::bit_xor<APInt>());
86}
87
88// Intersect overflow flags. Callers must ensure the common flags remain valid
89// after rewriting.
90static IntegerOverflowFlagsAttr
91mergeOverflowFlags(IntegerOverflowFlagsAttr val1,
92 IntegerOverflowFlagsAttr val2) {
93 return IntegerOverflowFlagsAttr::get(context: val1.getContext(),
94 value: val1.getValue() & val2.getValue());
95}
96
97using OverflowCheckedBinaryOp = APInt (APInt::*)(const APInt &, bool &) const;
98
99// For fixed-width integers, if combining the constants does not overflow, the
100// reassociated expression has the same mathematical result as the original
101// expression. Common source flags then rule out overflow in the new operation.
102static IntegerOverflowFlagsAttr mergeOverflowFlagsForReassociation(
103 IntegerOverflowFlagsAttr val1, IntegerOverflowFlagsAttr val2, Attribute lhs,
104 Attribute rhs, OverflowCheckedBinaryOp signedOp,
105 OverflowCheckedBinaryOp unsignedOp) {
106 IntegerOverflowFlags flags = val1.getValue() & val2.getValue();
107 if (flags == IntegerOverflowFlags::none)
108 return IntegerOverflowFlagsAttr::get(context: val1.getContext(), value: flags);
109
110 auto lhsAttr = llvm::cast<IntegerAttr>(Val&: lhs);
111 auto rhsAttr = llvm::cast<IntegerAttr>(Val&: rhs);
112 // TODO: Handle index separately; its storage width is not its target width.
113 if (isa<IndexType>(Val: lhsAttr.getType()))
114 return IntegerOverflowFlagsAttr::get(context: val1.getContext(), value: flags);
115
116 const APInt &lhsValue = lhsAttr.getValue();
117 const APInt &rhsValue = rhsAttr.getValue();
118 bool overflow;
119 if (bitEnumContainsAny(bits: flags, bit: IntegerOverflowFlags::nsw)) {
120 (void)(lhsValue.*signedOp)(rhsValue, overflow);
121 if (overflow)
122 flags = bitEnumClear(bits: flags, bit: IntegerOverflowFlags::nsw);
123 }
124 if (bitEnumContainsAny(bits: flags, bit: IntegerOverflowFlags::nuw)) {
125 (void)(lhsValue.*unsignedOp)(rhsValue, overflow);
126 if (overflow)
127 flags = bitEnumClear(bits: flags, bit: IntegerOverflowFlags::nuw);
128 }
129 return IntegerOverflowFlagsAttr::get(context: val1.getContext(), value: flags);
130}
131
132/// Invert an integer comparison predicate.
133arith::CmpIPredicate arith::invertPredicate(arith::CmpIPredicate pred) {
134 switch (pred) {
135 case arith::CmpIPredicate::eq:
136 return arith::CmpIPredicate::ne;
137 case arith::CmpIPredicate::ne:
138 return arith::CmpIPredicate::eq;
139 case arith::CmpIPredicate::slt:
140 return arith::CmpIPredicate::sge;
141 case arith::CmpIPredicate::sle:
142 return arith::CmpIPredicate::sgt;
143 case arith::CmpIPredicate::sgt:
144 return arith::CmpIPredicate::sle;
145 case arith::CmpIPredicate::sge:
146 return arith::CmpIPredicate::slt;
147 case arith::CmpIPredicate::ult:
148 return arith::CmpIPredicate::uge;
149 case arith::CmpIPredicate::ule:
150 return arith::CmpIPredicate::ugt;
151 case arith::CmpIPredicate::ugt:
152 return arith::CmpIPredicate::ule;
153 case arith::CmpIPredicate::uge:
154 return arith::CmpIPredicate::ult;
155 }
156 llvm_unreachable("unknown cmpi predicate kind");
157}
158
159/// Equivalent to
160/// convertRoundingModeToLLVM(convertArithRoundingModeToLLVM(roundingMode)).
161///
162/// Not possible to implement as chain of calls as this would introduce a
163/// circular dependency with MLIRArithAttrToLLVMConversion and make arith depend
164/// on the LLVM dialect and on translation to LLVM.
165static llvm::RoundingMode
166convertArithRoundingModeToLLVMIR(std::optional<RoundingMode> roundingMode) {
167 if (!roundingMode)
168 return kDefaultRoundingMode;
169 switch (*roundingMode) {
170 case RoundingMode::downward:
171 return llvm::RoundingMode::TowardNegative;
172 case RoundingMode::to_nearest_away:
173 return llvm::RoundingMode::NearestTiesToAway;
174 case RoundingMode::to_nearest_even:
175 return llvm::RoundingMode::NearestTiesToEven;
176 case RoundingMode::toward_zero:
177 return llvm::RoundingMode::TowardZero;
178 case RoundingMode::upward:
179 return llvm::RoundingMode::TowardPositive;
180 }
181 llvm_unreachable("Unhandled rounding mode");
182}
183
184static arith::CmpIPredicateAttr invertPredicate(arith::CmpIPredicateAttr pred) {
185 return arith::CmpIPredicateAttr::get(context: pred.getContext(),
186 val: invertPredicate(pred: pred.getValue()));
187}
188
189static int64_t getScalarOrElementWidth(Type type) {
190 Type elemTy = getElementTypeOrSelf(type);
191 if (elemTy.isIntOrFloat())
192 return elemTy.getIntOrFloatBitWidth();
193
194 return -1;
195}
196
197static int64_t getScalarOrElementWidth(Value value) {
198 return getScalarOrElementWidth(type: value.getType());
199}
200
201static FailureOr<APInt> getIntOrSplatIntValue(Attribute attr) {
202 APInt value;
203 if (matchPattern(attr, pattern: m_ConstantInt(bind_value: &value)))
204 return value;
205
206 return failure();
207}
208
209static Attribute getBoolAttribute(Type type, bool value) {
210 auto boolAttr = BoolAttr::get(context: type.getContext(), value);
211 ShapedType shapedType = dyn_cast_or_null<ShapedType>(Val&: type);
212 if (!shapedType)
213 return boolAttr;
214 // DenseElementsAttr requires a static shape.
215 if (!shapedType.hasStaticShape())
216 return {};
217 return DenseElementsAttr::get(type: shapedType, values: boolAttr);
218}
219
220/// Return a scalar or splat integer attribute of `type` (an integer/index type
221/// or a shaped type thereof) holding `value`. Returns a null attribute for
222/// shaped types with a dynamic shape, so callers can bail out of folding.
223static Attribute getIntegerAttrOfType(Type type, int64_t value) {
224 auto scalarAttr = IntegerAttr::get(type: getElementTypeOrSelf(type), value);
225 ShapedType shapedType = dyn_cast<ShapedType>(Val&: type);
226 if (!shapedType)
227 return scalarAttr;
228 if (!shapedType.hasStaticShape())
229 return {};
230 return DenseElementsAttr::get(type: shapedType, values: scalarAttr);
231}
232
233//===----------------------------------------------------------------------===//
234// TableGen'd canonicalization patterns
235//===----------------------------------------------------------------------===//
236
237namespace {
238#include "ArithCanonicalization.inc"
239} // namespace
240
241//===----------------------------------------------------------------------===//
242// Common helpers
243//===----------------------------------------------------------------------===//
244
245/// Return the type of the same shape (scalar, vector or tensor) containing i1.
246static Type getI1SameShape(Type type) {
247 auto i1Type = IntegerType::get(context: type.getContext(), width: 1);
248 if (auto shapedType = dyn_cast<ShapedType>(Val&: type))
249 return shapedType.cloneWith(shape: std::nullopt, elementType: i1Type);
250 if (llvm::isa<UnrankedTensorType>(Val: type))
251 return UnrankedTensorType::get(elementType: i1Type);
252 return i1Type;
253}
254
255//===----------------------------------------------------------------------===//
256// ConstantOp
257//===----------------------------------------------------------------------===//
258
259void arith::ConstantOp::getAsmResultNames(
260 function_ref<void(Value, StringRef)> setNameFn) {
261 auto type = getType();
262 if (auto intCst = dyn_cast<IntegerAttr>(Val: getValue())) {
263 auto intType = dyn_cast<IntegerType>(Val&: type);
264
265 // Sugar i1 constants with 'true' and 'false'.
266 if (intType && intType.getWidth() == 1)
267 return setNameFn(getResult(), (intCst.getInt() ? "true" : "false"));
268
269 // Otherwise, build a complex name with the value and type.
270 SmallString<32> specialNameBuffer;
271 llvm::raw_svector_ostream specialName(specialNameBuffer);
272 specialName << 'c' << intCst.getValue();
273 if (intType)
274 specialName << '_' << type;
275 setNameFn(getResult(), specialName.str());
276 } else {
277 setNameFn(getResult(), "cst");
278 }
279}
280
281/// TODO: disallow arith.constant to return anything other than signless integer
282/// or float like.
283LogicalResult arith::ConstantOp::verify() {
284 auto type = getType();
285 // Integer values must be signless.
286 if (auto intType = dyn_cast<IntegerType>(Val: getElementTypeOrSelf(type));
287 intType && !intType.isSignless())
288 return emitOpError(message: "integer return type must be signless");
289 // Any float or elements attribute are acceptable.
290 if (!llvm::isa<IntegerAttr, FloatAttr, ElementsAttr>(Val: getValue())) {
291 return emitOpError(
292 message: "value must be an integer, float, or elements attribute");
293 }
294
295 // Note, we could relax this for vectors with 1 scalable dim, e.g.:
296 // * arith.constant dense<[[3, 3], [1, 1]]> : vector<2 x [2] x i32>
297 // However, this would most likely require updating the lowerings to LLVM.
298 if (isa<ScalableVectorType>(Val: type) && !isa<SplatElementsAttr>(Val: getValue()))
299 return emitOpError(
300 message: "initializing scalable vectors with elements attribute is not supported"
301 " unless it's a vector splat");
302 return success();
303}
304
305bool arith::ConstantOp::isBuildableWith(Attribute value, Type type) {
306 // The value's type must be the same as the provided type.
307 auto typedAttr = dyn_cast<TypedAttr>(Val&: value);
308 if (!typedAttr || typedAttr.getType() != type)
309 return false;
310 // Integer values must be signless.
311 if (auto intType = dyn_cast<IntegerType>(Val: getElementTypeOrSelf(type))) {
312 if (!intType.isSignless())
313 return false;
314 }
315 // Integer, float, and element attributes are buildable.
316 return llvm::isa<IntegerAttr, FloatAttr, ElementsAttr>(Val: value);
317}
318
319ConstantOp arith::ConstantOp::materialize(OpBuilder &builder, Attribute value,
320 Type type, Location loc) {
321 if (isBuildableWith(value, type))
322 return arith::ConstantOp::create(builder, location: loc, value: cast<TypedAttr>(Val&: value));
323 return nullptr;
324}
325
326OpFoldResult arith::ConstantOp::fold(FoldAdaptor adaptor) { return getValue(); }
327
328void arith::ConstantIntOp::build(OpBuilder &builder, OperationState &result,
329 int64_t value, unsigned width) {
330 auto type = builder.getIntegerType(width);
331 arith::ConstantOp::build(odsBuilder&: builder, odsState&: result, result: type,
332 value: builder.getIntegerAttr(type, value));
333}
334
335arith::ConstantIntOp arith::ConstantIntOp::create(OpBuilder &builder,
336 Location location,
337 int64_t value,
338 unsigned width) {
339 mlir::OperationState state(location, getOperationName());
340 build(builder, result&: state, value, width);
341 auto result = dyn_cast<ConstantIntOp>(Val: builder.create(state));
342 assert(result && "builder didn't return the right type");
343 return result;
344}
345
346arith::ConstantIntOp arith::ConstantIntOp::create(ImplicitLocOpBuilder &builder,
347 int64_t value,
348 unsigned width) {
349 return create(builder, location: builder.getLoc(), value, width);
350}
351
352void arith::ConstantIntOp::build(OpBuilder &builder, OperationState &result,
353 Type type, int64_t value) {
354 arith::ConstantOp::build(odsBuilder&: builder, odsState&: result, result: type,
355 value: builder.getIntegerAttr(type, value));
356}
357
358arith::ConstantIntOp arith::ConstantIntOp::create(OpBuilder &builder,
359 Location location, Type type,
360 int64_t value) {
361 mlir::OperationState state(location, getOperationName());
362 build(builder, result&: state, type, value);
363 auto result = dyn_cast<ConstantIntOp>(Val: builder.create(state));
364 assert(result && "builder didn't return the right type");
365 return result;
366}
367
368arith::ConstantIntOp arith::ConstantIntOp::create(ImplicitLocOpBuilder &builder,
369 Type type, int64_t value) {
370 return create(builder, location: builder.getLoc(), type, value);
371}
372
373void arith::ConstantIntOp::build(OpBuilder &builder, OperationState &result,
374 Type type, const APInt &value) {
375 arith::ConstantOp::build(odsBuilder&: builder, odsState&: result, result: type,
376 value: builder.getIntegerAttr(type, value));
377}
378
379arith::ConstantIntOp arith::ConstantIntOp::create(OpBuilder &builder,
380 Location location, Type type,
381 const APInt &value) {
382 mlir::OperationState state(location, getOperationName());
383 build(builder, result&: state, type, value);
384 auto result = dyn_cast<ConstantIntOp>(Val: builder.create(state));
385 assert(result && "builder didn't return the right type");
386 return result;
387}
388
389arith::ConstantIntOp arith::ConstantIntOp::create(ImplicitLocOpBuilder &builder,
390 Type type,
391 const APInt &value) {
392 return create(builder, location: builder.getLoc(), type, value);
393}
394
395bool arith::ConstantIntOp::classof(Operation *op) {
396 if (auto constOp = dyn_cast_or_null<arith::ConstantOp>(Val: op))
397 return constOp.getType().isSignlessInteger();
398 return false;
399}
400
401void arith::ConstantFloatOp::build(OpBuilder &builder, OperationState &result,
402 FloatType type, const APFloat &value) {
403 arith::ConstantOp::build(odsBuilder&: builder, odsState&: result, result: type,
404 value: builder.getFloatAttr(type, value));
405}
406
407arith::ConstantFloatOp arith::ConstantFloatOp::create(OpBuilder &builder,
408 Location location,
409 FloatType type,
410 const APFloat &value) {
411 mlir::OperationState state(location, getOperationName());
412 build(builder, result&: state, type, value);
413 auto result = dyn_cast<ConstantFloatOp>(Val: builder.create(state));
414 assert(result && "builder didn't return the right type");
415 return result;
416}
417
418arith::ConstantFloatOp
419arith::ConstantFloatOp::create(ImplicitLocOpBuilder &builder, FloatType type,
420 const APFloat &value) {
421 return create(builder, location: builder.getLoc(), type, value);
422}
423
424bool arith::ConstantFloatOp::classof(Operation *op) {
425 if (auto constOp = dyn_cast_or_null<arith::ConstantOp>(Val: op))
426 return llvm::isa<FloatType>(Val: constOp.getType());
427 return false;
428}
429
430void arith::ConstantIndexOp::build(OpBuilder &builder, OperationState &result,
431 int64_t value) {
432 arith::ConstantOp::build(odsBuilder&: builder, odsState&: result, result: builder.getIndexType(),
433 value: builder.getIndexAttr(value));
434}
435
436arith::ConstantIndexOp arith::ConstantIndexOp::create(OpBuilder &builder,
437 Location location,
438 int64_t value) {
439 mlir::OperationState state(location, getOperationName());
440 build(builder, result&: state, value);
441 auto result = dyn_cast<ConstantIndexOp>(Val: builder.create(state));
442 assert(result && "builder didn't return the right type");
443 return result;
444}
445
446arith::ConstantIndexOp
447arith::ConstantIndexOp::create(ImplicitLocOpBuilder &builder, int64_t value) {
448 return create(builder, location: builder.getLoc(), value);
449}
450
451bool arith::ConstantIndexOp::classof(Operation *op) {
452 if (auto constOp = dyn_cast_or_null<arith::ConstantOp>(Val: op))
453 return constOp.getType().isIndex();
454 return false;
455}
456
457Value mlir::arith::getZeroConstant(OpBuilder &builder, Location loc,
458 Type type) {
459 // TODO: Incorporate this check to `FloatAttr::get*`.
460 assert(!isa<Float8E8M0FNUType>(getElementTypeOrSelf(type)) &&
461 "type doesn't have a zero representation");
462 TypedAttr zeroAttr = builder.getZeroAttr(type);
463 assert(zeroAttr && "unsupported type for zero attribute");
464 return arith::ConstantOp::create(builder, location: loc, value: zeroAttr);
465}
466
467//===----------------------------------------------------------------------===//
468// AddIOp
469//===----------------------------------------------------------------------===//
470
471OpFoldResult arith::AddIOp::fold(FoldAdaptor adaptor) {
472 // addi(x, 0) -> x
473 if (matchPattern(attr: adaptor.getRhs(), pattern: m_Zero()))
474 return getLhs();
475
476 // addi(subi(a, b), b) -> a
477 if (auto sub = getLhs().getDefiningOp<SubIOp>())
478 if (getRhs() == sub.getRhs())
479 return sub.getLhs();
480
481 // addi(b, subi(a, b)) -> a
482 if (auto sub = getRhs().getDefiningOp<SubIOp>())
483 if (getLhs() == sub.getRhs())
484 return sub.getLhs();
485
486 return constFoldBinaryOp<IntegerAttr>(
487 operands: adaptor.getOperands(),
488 calculate: [](APInt a, const APInt &b) { return std::move(a) + b; });
489}
490
491void arith::AddIOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
492 MLIRContext *context) {
493 patterns.add<AddIAddConstant, AddISubConstantRHS, AddISubConstantLHS,
494 AddIMulNegativeOneRhs, AddIMulNegativeOneLhs>(arg&: context);
495}
496
497//===----------------------------------------------------------------------===//
498// AddUIExtendedOp
499//===----------------------------------------------------------------------===//
500
501std::optional<SmallVector<int64_t, 4>>
502arith::AddUIExtendedOp::getIterationShapeToUnroll() {
503 if (auto vt = dyn_cast<VectorType>(Val: getType(i: 0)))
504 return llvm::to_vector<4>(Range: vt.getShape());
505 return std::nullopt;
506}
507
508// Returns the overflow bit, assuming that `sum` is the result of unsigned
509// addition of `operand` and another number.
510static APInt calculateUnsignedOverflow(const APInt &sum, const APInt &operand) {
511 return sum.ult(RHS: operand) ? APInt::getAllOnes(numBits: 1) : APInt::getZero(numBits: 1);
512}
513
514LogicalResult
515arith::AddUIExtendedOp::fold(FoldAdaptor adaptor,
516 SmallVectorImpl<OpFoldResult> &results) {
517 Type overflowTy = getOverflow().getType();
518 // addui_extended(x, 0) -> x, false
519 if (matchPattern(value: getRhs(), pattern: m_Zero())) {
520 Builder builder(getContext());
521 auto falseValue = builder.getZeroAttr(type: overflowTy);
522
523 results.push_back(Elt: getLhs());
524 results.push_back(Elt: falseValue);
525 return success();
526 }
527
528 // addui_extended(constant_a, constant_b) -> constant_sum, constant_carry
529 // Let the `constFoldBinaryOp` utility attempt to fold the sum of both
530 // operands. If that succeeds, calculate the overflow bit based on the sum
531 // and the first (constant) operand, `lhs`.
532 if (Attribute sumAttr = constFoldBinaryOp<IntegerAttr>(
533 operands: adaptor.getOperands(),
534 calculate: [](APInt a, const APInt &b) { return std::move(a) + b; })) {
535 // If any operand is poison, propagate poison to both results.
536 if (matchPattern(attr: sumAttr, pattern: ub::m_Poison())) {
537 results.push_back(Elt: sumAttr);
538 results.push_back(Elt: sumAttr);
539 return success();
540 }
541 Attribute overflowAttr = constFoldBinaryOp<IntegerAttr>(
542 operands: ArrayRef({sumAttr, adaptor.getLhs()}),
543 resultType: getI1SameShape(type: llvm::cast<TypedAttr>(Val&: sumAttr).getType()),
544 calculate&: calculateUnsignedOverflow);
545 if (!overflowAttr)
546 return failure();
547
548 results.push_back(Elt: sumAttr);
549 results.push_back(Elt: overflowAttr);
550 return success();
551 }
552
553 return failure();
554}
555
556void arith::AddUIExtendedOp::getCanonicalizationPatterns(
557 RewritePatternSet &patterns, MLIRContext *context) {
558 patterns.add<AddUIExtendedToAddI>(arg&: context);
559}
560
561//===----------------------------------------------------------------------===//
562// SubUIExtendedOp
563//===----------------------------------------------------------------------===//
564
565std::optional<SmallVector<int64_t, 4>>
566arith::SubUIExtendedOp::getIterationShapeToUnroll() {
567 if (auto vt = dyn_cast<VectorType>(Val: getType(i: 0)))
568 return llvm::to_vector<4>(Range: vt.getShape());
569 return std::nullopt;
570}
571
572// Returns the borrow bit, assuming `lhs` and `rhs` are operands of an unsigned
573// subtraction whose mathematical result underflows iff `lhs < rhs`.
574static APInt calculateUnsignedBorrow(const APInt &lhs, const APInt &rhs) {
575 return lhs.ult(RHS: rhs) ? APInt::getAllOnes(numBits: 1) : APInt::getZero(numBits: 1);
576}
577
578LogicalResult
579arith::SubUIExtendedOp::fold(FoldAdaptor adaptor,
580 SmallVectorImpl<OpFoldResult> &results) {
581 Type borrowTy = getBorrow().getType();
582 // subui_extended(x, 0) -> x, false
583 if (matchPattern(value: getRhs(), pattern: m_Zero())) {
584 Builder builder(getContext());
585 auto falseValue = builder.getZeroAttr(type: borrowTy);
586
587 results.push_back(Elt: getLhs());
588 results.push_back(Elt: falseValue);
589 return success();
590 }
591
592 // subui_extended(x, x) -> 0, false
593 if (getLhs() == getRhs()) {
594 // A dynamically-shaped result cannot be a constant; bail before
595 // getZeroAttr, which would assert on a non-static shape.
596 auto shapedType = dyn_cast<ShapedType>(Val: getDiff().getType());
597 if (shapedType && !shapedType.hasStaticShape())
598 return failure();
599 Builder builder(getContext());
600 auto zeroDiff = builder.getZeroAttr(type: getDiff().getType());
601 auto falseValue = builder.getZeroAttr(type: borrowTy);
602 if (!zeroDiff)
603 return failure();
604
605 results.push_back(Elt: zeroDiff);
606 results.push_back(Elt: falseValue);
607 return success();
608 }
609
610 // subui_extended(constant_a, constant_b) -> constant_diff, constant_borrow
611 if (Attribute diffAttr = constFoldBinaryOp<IntegerAttr>(
612 operands: adaptor.getOperands(),
613 calculate: [](APInt a, const APInt &b) { return std::move(a) - b; })) {
614 // If any operand is poison, propagate poison to both results.
615 if (matchPattern(attr: diffAttr, pattern: ub::m_Poison())) {
616 results.push_back(Elt: diffAttr);
617 results.push_back(Elt: diffAttr);
618 return success();
619 }
620 Attribute borrowAttr = constFoldBinaryOp<IntegerAttr>(
621 operands: adaptor.getOperands(),
622 resultType: getI1SameShape(type: llvm::cast<TypedAttr>(Val&: diffAttr).getType()),
623 calculate&: calculateUnsignedBorrow);
624 if (!borrowAttr)
625 return failure();
626
627 results.push_back(Elt: diffAttr);
628 results.push_back(Elt: borrowAttr);
629 return success();
630 }
631
632 return failure();
633}
634
635void arith::SubUIExtendedOp::getCanonicalizationPatterns(
636 RewritePatternSet &patterns, MLIRContext *context) {
637 patterns.add<SubUIExtendedToSubI>(arg&: context);
638}
639
640//===----------------------------------------------------------------------===//
641// SubIOp
642//===----------------------------------------------------------------------===//
643
644OpFoldResult arith::SubIOp::fold(FoldAdaptor adaptor) {
645 // subi(x,x) -> 0
646 if (getOperand(i: 0) == getOperand(i: 1)) {
647 auto shapedType = dyn_cast<ShapedType>(Val: getType());
648 // We can't generate a constant with a dynamic shaped tensor.
649 if (!shapedType || shapedType.hasStaticShape())
650 return Builder(getContext()).getZeroAttr(type: getType());
651 }
652 // subi(x,0) -> x
653 if (matchPattern(attr: adaptor.getRhs(), pattern: m_Zero()))
654 return getLhs();
655
656 if (auto add = getLhs().getDefiningOp<AddIOp>()) {
657 // subi(addi(a, b), b) -> a
658 if (getRhs() == add.getRhs())
659 return add.getLhs();
660 // subi(addi(a, b), a) -> b
661 if (getRhs() == add.getLhs())
662 return add.getRhs();
663 }
664
665 // subi(a, subi(a, b)) -> b
666 if (auto sub = getRhs().getDefiningOp<SubIOp>())
667 if (getLhs() == sub.getLhs())
668 return sub.getRhs();
669
670 return constFoldBinaryOp<IntegerAttr>(
671 operands: adaptor.getOperands(),
672 calculate: [](APInt a, const APInt &b) { return std::move(a) - b; });
673}
674
675void arith::SubIOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
676 MLIRContext *context) {
677 patterns.add<SubIRHSAddConstant, SubILHSAddConstant, SubIRHSSubConstantRHS,
678 SubIRHSSubConstantLHS, SubILHSSubConstantRHS,
679 SubILHSSubConstantLHS, SubISubILHSRHSLHS>(arg&: context);
680}
681
682//===----------------------------------------------------------------------===//
683// MulIOp
684//===----------------------------------------------------------------------===//
685
686OpFoldResult arith::MulIOp::fold(FoldAdaptor adaptor) {
687 // muli(x, 0) -> 0
688 if (matchPattern(attr: adaptor.getRhs(), pattern: m_Zero()))
689 return getRhs();
690 // muli(x, 1) -> x
691 if (matchPattern(attr: adaptor.getRhs(), pattern: m_One()))
692 return getLhs();
693 // TODO: Handle the overflow case.
694
695 // default folder
696 return constFoldBinaryOp<IntegerAttr>(
697 operands: adaptor.getOperands(),
698 calculate: [](const APInt &a, const APInt &b) { return a * b; });
699}
700
701void arith::MulIOp::getAsmResultNames(
702 function_ref<void(Value, StringRef)> setNameFn) {
703 if (!isa<IndexType>(Val: getType()))
704 return;
705
706 // Match vector.vscale by name to avoid depending on the vector dialect (which
707 // is a circular dependency).
708 auto isVscale = [](Operation *op) {
709 return op && op->getName().getStringRef() == "vector.vscale";
710 };
711
712 IntegerAttr baseValue;
713 auto isVscaleExpr = [&](Value a, Value b) {
714 return matchPattern(value: a, pattern: m_Constant(bind_value: &baseValue)) &&
715 isVscale(b.getDefiningOp());
716 };
717
718 if (!isVscaleExpr(getLhs(), getRhs()) && !isVscaleExpr(getRhs(), getLhs()))
719 return;
720
721 // Name `base * vscale` or `vscale * base` as `c<base_value>_vscale`.
722 SmallString<32> specialNameBuffer;
723 llvm::raw_svector_ostream specialName(specialNameBuffer);
724 specialName << 'c' << baseValue.getInt() << "_vscale";
725 setNameFn(getResult(), specialName.str());
726}
727
728void arith::MulIOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
729 MLIRContext *context) {
730 patterns.add<MulIMulIConstant>(arg&: context);
731}
732
733//===----------------------------------------------------------------------===//
734// MulSIExtendedOp
735//===----------------------------------------------------------------------===//
736
737std::optional<SmallVector<int64_t, 4>>
738arith::MulSIExtendedOp::getIterationShapeToUnroll() {
739 if (auto vt = dyn_cast<VectorType>(Val: getType(i: 0)))
740 return llvm::to_vector<4>(Range: vt.getShape());
741 return std::nullopt;
742}
743
744LogicalResult
745arith::MulSIExtendedOp::fold(FoldAdaptor adaptor,
746 SmallVectorImpl<OpFoldResult> &results) {
747 // mulsi_extended(x, 0) -> 0, 0
748 if (matchPattern(attr: adaptor.getRhs(), pattern: m_Zero())) {
749 Attribute zero = adaptor.getRhs();
750 results.push_back(Elt: zero);
751 results.push_back(Elt: zero);
752 return success();
753 }
754
755 // mulsi_extended(cst_a, cst_b) -> cst_low, cst_high
756 if (Attribute lowAttr = constFoldBinaryOp<IntegerAttr>(
757 operands: adaptor.getOperands(),
758 calculate: [](const APInt &a, const APInt &b) { return a * b; })) {
759 // Invoke the constant fold helper again to calculate the 'high' result.
760 Attribute highAttr = constFoldBinaryOp<IntegerAttr>(operands: adaptor.getOperands(),
761 calculate&: llvm::APIntOps::mulhs);
762 assert(highAttr && "Unexpected constant-folding failure");
763
764 results.push_back(Elt: lowAttr);
765 results.push_back(Elt: highAttr);
766 return success();
767 }
768
769 return failure();
770}
771
772void arith::MulSIExtendedOp::getCanonicalizationPatterns(
773 RewritePatternSet &patterns, MLIRContext *context) {
774 patterns.add<MulSIExtendedToMulI, MulSIExtendedRHSOne>(arg&: context);
775}
776
777//===----------------------------------------------------------------------===//
778// MulUIExtendedOp
779//===----------------------------------------------------------------------===//
780
781std::optional<SmallVector<int64_t, 4>>
782arith::MulUIExtendedOp::getIterationShapeToUnroll() {
783 if (auto vt = dyn_cast<VectorType>(Val: getType(i: 0)))
784 return llvm::to_vector<4>(Range: vt.getShape());
785 return std::nullopt;
786}
787
788LogicalResult
789arith::MulUIExtendedOp::fold(FoldAdaptor adaptor,
790 SmallVectorImpl<OpFoldResult> &results) {
791 // mului_extended(x, 0) -> 0, 0
792 if (matchPattern(attr: adaptor.getRhs(), pattern: m_Zero())) {
793 Attribute zero = adaptor.getRhs();
794 results.push_back(Elt: zero);
795 results.push_back(Elt: zero);
796 return success();
797 }
798
799 // mului_extended(x, 1) -> x, 0
800 if (matchPattern(attr: adaptor.getRhs(), pattern: m_One())) {
801 Builder builder(getContext());
802 Attribute zero = builder.getZeroAttr(type: getLhs().getType());
803 results.push_back(Elt: getLhs());
804 results.push_back(Elt: zero);
805 return success();
806 }
807
808 // mului_extended(cst_a, cst_b) -> cst_low, cst_high
809 if (Attribute lowAttr = constFoldBinaryOp<IntegerAttr>(
810 operands: adaptor.getOperands(),
811 calculate: [](const APInt &a, const APInt &b) { return a * b; })) {
812 // Invoke the constant fold helper again to calculate the 'high' result.
813 Attribute highAttr = constFoldBinaryOp<IntegerAttr>(operands: adaptor.getOperands(),
814 calculate&: llvm::APIntOps::mulhu);
815 assert(highAttr && "Unexpected constant-folding failure");
816
817 results.push_back(Elt: lowAttr);
818 results.push_back(Elt: highAttr);
819 return success();
820 }
821
822 return failure();
823}
824
825void arith::MulUIExtendedOp::getCanonicalizationPatterns(
826 RewritePatternSet &patterns, MLIRContext *context) {
827 patterns.add<MulUIExtendedToMulI>(arg&: context);
828}
829
830//===----------------------------------------------------------------------===//
831// DivUIOp
832//===----------------------------------------------------------------------===//
833
834/// Fold `(a * b) / b -> a`
835static Value foldDivMul(Value lhs, Value rhs,
836 arith::IntegerOverflowFlags ovfFlags) {
837 auto mul = lhs.getDefiningOp<mlir::arith::MulIOp>();
838 if (!mul || !bitEnumContainsAll(bits: mul.getOverflowFlags(), bit: ovfFlags))
839 return {};
840
841 if (mul.getLhs() == rhs)
842 return mul.getRhs();
843
844 if (mul.getRhs() == rhs)
845 return mul.getLhs();
846
847 return {};
848}
849
850OpFoldResult arith::DivUIOp::fold(FoldAdaptor adaptor) {
851 // TODO: divui (x, 0) -> poison. Division by zero is undefined behaviour and
852 // could fold to poison, but that would make the arith dialect depend on the
853 // ub dialect to materialize ub.poison; left out for now.
854
855 // divui (x, 1) -> x.
856 if (matchPattern(attr: adaptor.getRhs(), pattern: m_One()))
857 return getLhs();
858
859 // divui (0, x) -> 0. Division by zero is UB, so refining to 0 is valid.
860 if (matchPattern(attr: adaptor.getLhs(), pattern: m_Zero()))
861 return getLhs();
862
863 // divui (x, x) -> 1.
864 if (getLhs() == getRhs())
865 return getIntegerAttrOfType(type: getType(), value: 1);
866
867 // (a * b) / b -> a
868 if (Value val = foldDivMul(lhs: getLhs(), rhs: getRhs(), ovfFlags: IntegerOverflowFlags::nuw))
869 return val;
870
871 // Don't fold if it would require a division by zero.
872 bool div0 = false;
873 auto result = constFoldBinaryOp<IntegerAttr>(operands: adaptor.getOperands(),
874 calculate: [&](APInt a, const APInt &b) {
875 if (div0 || !b) {
876 div0 = true;
877 return a;
878 }
879 return a.udiv(RHS: b);
880 });
881
882 return div0 ? Attribute() : result;
883}
884
885/// Returns whether an unsigned division by `divisor` is speculatable.
886static Speculation::Speculatability getDivUISpeculatability(Value divisor) {
887 // X / 0 => UB
888 if (matchPattern(value: divisor, pattern: m_IntRangeWithoutZeroU()))
889 return Speculation::Speculatable;
890
891 return Speculation::NotSpeculatable;
892}
893
894Speculation::Speculatability arith::DivUIOp::getSpeculatability() {
895 return getDivUISpeculatability(divisor: getRhs());
896}
897
898//===----------------------------------------------------------------------===//
899// DivSIOp
900//===----------------------------------------------------------------------===//
901
902OpFoldResult arith::DivSIOp::fold(FoldAdaptor adaptor) {
903 // TODO: divsi (x, 0) -> poison. Division by zero is undefined behaviour and
904 // could fold to poison, but that would make the arith dialect depend on the
905 // ub dialect to materialize ub.poison; left out for now.
906
907 // divsi (x, 1) -> x.
908 if (matchPattern(attr: adaptor.getRhs(), pattern: m_One()))
909 return getLhs();
910
911 // divsi (0, x) -> 0. Division by zero is UB, so refining to 0 is valid.
912 if (matchPattern(attr: adaptor.getLhs(), pattern: m_Zero()))
913 return getLhs();
914
915 // divsi (x, x) -> 1.
916 if (getLhs() == getRhs())
917 return getIntegerAttrOfType(type: getType(), value: 1);
918
919 // (a * b) / b -> a
920 if (Value val = foldDivMul(lhs: getLhs(), rhs: getRhs(), ovfFlags: IntegerOverflowFlags::nsw))
921 return val;
922
923 // Don't fold if it would overflow or if it requires a division by zero.
924 bool overflowOrDiv0 = false;
925 auto result = constFoldBinaryOp<IntegerAttr>(
926 operands: adaptor.getOperands(), calculate: [&](APInt a, const APInt &b) {
927 if (overflowOrDiv0 || !b) {
928 overflowOrDiv0 = true;
929 return a;
930 }
931 return a.sdiv_ov(RHS: b, Overflow&: overflowOrDiv0);
932 });
933
934 return overflowOrDiv0 ? Attribute() : result;
935}
936
937/// Returns whether a signed division by `divisor` is speculatable. This
938/// function conservatively assumes that all signed division by -1 are not
939/// speculatable.
940static Speculation::Speculatability getDivSISpeculatability(Value divisor) {
941 // X / 0 => UB
942 // INT_MIN / -1 => UB
943 if (matchPattern(value: divisor, pattern: m_IntRangeWithoutZeroS()) &&
944 matchPattern(value: divisor, pattern: m_IntRangeWithoutNegOneS()))
945 return Speculation::Speculatable;
946
947 return Speculation::NotSpeculatable;
948}
949
950Speculation::Speculatability arith::DivSIOp::getSpeculatability() {
951 return getDivSISpeculatability(divisor: getRhs());
952}
953
954//===----------------------------------------------------------------------===//
955// CeilDivUIOp
956//===----------------------------------------------------------------------===//
957
958OpFoldResult arith::CeilDivUIOp::fold(FoldAdaptor adaptor) {
959 // TODO: ceildivui (x, 0) -> poison. Division by zero is undefined behaviour
960 // and could fold to poison, but that would make the arith dialect depend on
961 // the ub dialect to materialize ub.poison; left out for now.
962
963 // ceildivui (x, 1) -> x.
964 if (matchPattern(attr: adaptor.getRhs(), pattern: m_One()))
965 return getLhs();
966
967 // ceildivui (0, x) -> 0. Division by zero is UB, so refining to 0 is valid.
968 if (matchPattern(attr: adaptor.getLhs(), pattern: m_Zero()))
969 return getLhs();
970
971 // ceildivui (x, x) -> 1.
972 if (getLhs() == getRhs())
973 return getIntegerAttrOfType(type: getType(), value: 1);
974
975 bool overflowOrDiv0 = false;
976 auto result = constFoldBinaryOp<IntegerAttr>(
977 operands: adaptor.getOperands(), calculate: [&](APInt a, const APInt &b) {
978 if (overflowOrDiv0 || !b) {
979 overflowOrDiv0 = true;
980 return a;
981 }
982 APInt quotient = a.udiv(RHS: b);
983 if (!a.urem(RHS: b))
984 return quotient;
985 APInt one(a.getBitWidth(), 1, true);
986 return quotient.uadd_ov(RHS: one, Overflow&: overflowOrDiv0);
987 });
988
989 return overflowOrDiv0 ? Attribute() : result;
990}
991
992Speculation::Speculatability arith::CeilDivUIOp::getSpeculatability() {
993 return getDivUISpeculatability(divisor: getRhs());
994}
995
996//===----------------------------------------------------------------------===//
997// CeilDivSIOp
998//===----------------------------------------------------------------------===//
999
1000OpFoldResult arith::CeilDivSIOp::fold(FoldAdaptor adaptor) {
1001 // TODO: ceildivsi (x, 0) -> poison. Division by zero is undefined behaviour
1002 // and could fold to poison, but that would make the arith dialect depend on
1003 // the ub dialect to materialize ub.poison; left out for now.
1004
1005 // ceildivsi (x, 1) -> x.
1006 if (matchPattern(attr: adaptor.getRhs(), pattern: m_One()))
1007 return getLhs();
1008
1009 // ceildivsi (0, x) -> 0. Division by zero is UB, so refining to 0 is valid.
1010 if (matchPattern(attr: adaptor.getLhs(), pattern: m_Zero()))
1011 return getLhs();
1012
1013 // ceildivsi (x, x) -> 1.
1014 if (getLhs() == getRhs())
1015 return getIntegerAttrOfType(type: getType(), value: 1);
1016
1017 // Don't fold if it would overflow or if it requires a division by zero.
1018 bool overflowOrDiv0 = false;
1019 auto result = constFoldBinaryOp<IntegerAttr>(
1020 operands: adaptor.getOperands(), calculate: [&](const APInt &a, const APInt &b) {
1021 if (overflowOrDiv0 || !b) {
1022 overflowOrDiv0 = true;
1023 return a;
1024 }
1025 // Compute the ceiling without negating either operand, so that MININT
1026 // operands still fold whenever the result is representable.
1027 //
1028 // sdiv truncates towards zero, so it already rounds up whenever the
1029 // exact quotient is negative. When the exact quotient is positive, i.e.
1030 // when the operands have the same sign, an inexact division has to be
1031 // corrected by one. This mirrors the expansion in ExpandOps.cpp.
1032 bool overflowDiv = false;
1033 APInt quotient = a.sdiv_ov(RHS: b, Overflow&: overflowDiv);
1034 if (overflowDiv) {
1035 // MININT / -1. The exact result is -MININT, which is not
1036 // representable.
1037 overflowOrDiv0 = true;
1038 return a;
1039 }
1040 if (a.isNegative() != b.isNegative() || quotient * b == a)
1041 return quotient;
1042
1043 // The correction cannot overflow: it only applies when the exact
1044 // quotient is positive and the division is inexact, which bounds the
1045 // quotient well below the maximum. Check anyway, at no cost.
1046 APInt one(a.getBitWidth(), 1, /*isSigned=*/true);
1047 return quotient.sadd_ov(RHS: one, Overflow&: overflowOrDiv0);
1048 });
1049
1050 return overflowOrDiv0 ? Attribute() : result;
1051}
1052
1053Speculation::Speculatability arith::CeilDivSIOp::getSpeculatability() {
1054 return getDivSISpeculatability(divisor: getRhs());
1055}
1056
1057//===----------------------------------------------------------------------===//
1058// FloorDivSIOp
1059//===----------------------------------------------------------------------===//
1060
1061OpFoldResult arith::FloorDivSIOp::fold(FoldAdaptor adaptor) {
1062 // TODO: floordivsi (x, 0) -> poison. Division by zero is undefined behaviour
1063 // and could fold to poison, but that would make the arith dialect depend on
1064 // the ub dialect to materialize ub.poison; left out for now.
1065
1066 // floordivsi (x, 1) -> x.
1067 if (matchPattern(attr: adaptor.getRhs(), pattern: m_One()))
1068 return getLhs();
1069
1070 // floordivsi (0, x) -> 0. Division by zero is UB, so refining to 0 is valid.
1071 if (matchPattern(attr: adaptor.getLhs(), pattern: m_Zero()))
1072 return getLhs();
1073
1074 // floordivsi (x, x) -> 1.
1075 if (getLhs() == getRhs())
1076 return getIntegerAttrOfType(type: getType(), value: 1);
1077
1078 // Don't fold if it would overflow or if it requires a division by zero.
1079 bool overflowOrDiv = false;
1080 auto result = constFoldBinaryOp<IntegerAttr>(
1081 operands: adaptor.getOperands(), calculate: [&](APInt a, const APInt &b) {
1082 if (b.isZero()) {
1083 overflowOrDiv = true;
1084 return a;
1085 }
1086 return a.sfloordiv_ov(RHS: b, Overflow&: overflowOrDiv);
1087 });
1088
1089 return overflowOrDiv ? Attribute() : result;
1090}
1091
1092Speculation::Speculatability FloorDivSIOp::getSpeculatability() {
1093 // Division by zero is UB.
1094 // Signed division overflow (INT_MIN / -1) is also UB.
1095 Value divisor = getRhs();
1096 if (matchPattern(value: divisor, pattern: m_IntRangeWithoutZeroS()) &&
1097 matchPattern(value: divisor, pattern: m_IntRangeWithoutNegOneS()))
1098 return Speculation::Speculatable;
1099
1100 return Speculation::NotSpeculatable;
1101}
1102
1103//===----------------------------------------------------------------------===//
1104// RemUIOp
1105//===----------------------------------------------------------------------===//
1106
1107OpFoldResult arith::RemUIOp::fold(FoldAdaptor adaptor) {
1108 // TODO: remui (x, 0) -> poison. Remainder by zero is undefined behaviour and
1109 // could fold to poison, but that would make the arith dialect depend on the
1110 // ub dialect to materialize ub.poison; left out for now.
1111
1112 // remui (x, 1) -> 0.
1113 if (matchPattern(attr: adaptor.getRhs(), pattern: m_One()))
1114 return getIntegerAttrOfType(type: getType(), value: 0);
1115
1116 // remui (0, x) -> 0 and remui (x, x) -> 0. Division by zero is UB, so
1117 // refining to 0 is valid.
1118 if (matchPattern(attr: adaptor.getLhs(), pattern: m_Zero()) || getLhs() == getRhs())
1119 return getIntegerAttrOfType(type: getType(), value: 0);
1120
1121 // Don't fold if it would require a division by zero.
1122 bool div0 = false;
1123 auto result = constFoldBinaryOp<IntegerAttr>(operands: adaptor.getOperands(),
1124 calculate: [&](APInt a, const APInt &b) {
1125 if (div0 || b.isZero()) {
1126 div0 = true;
1127 return a;
1128 }
1129 return a.urem(RHS: b);
1130 });
1131
1132 return div0 ? Attribute() : result;
1133}
1134
1135Speculation::Speculatability arith::RemUIOp::getSpeculatability() {
1136 return getDivUISpeculatability(divisor: getRhs());
1137}
1138
1139//===----------------------------------------------------------------------===//
1140// RemSIOp
1141//===----------------------------------------------------------------------===//
1142
1143OpFoldResult arith::RemSIOp::fold(FoldAdaptor adaptor) {
1144 // TODO: remsi (x, 0) -> poison. Remainder by zero is undefined behaviour and
1145 // could fold to poison, but that would make the arith dialect depend on the
1146 // ub dialect to materialize ub.poison; left out for now.
1147
1148 // remsi (x, 1) -> 0.
1149 if (matchPattern(attr: adaptor.getRhs(), pattern: m_One()))
1150 return getIntegerAttrOfType(type: getType(), value: 0);
1151
1152 // remsi (0, x) -> 0 and remsi (x, x) -> 0. Division by zero is UB, so
1153 // refining to 0 is valid.
1154 if (matchPattern(attr: adaptor.getLhs(), pattern: m_Zero()) || getLhs() == getRhs())
1155 return getIntegerAttrOfType(type: getType(), value: 0);
1156
1157 // Don't fold if it would require a division by zero.
1158 bool div0 = false;
1159 auto result = constFoldBinaryOp<IntegerAttr>(operands: adaptor.getOperands(),
1160 calculate: [&](APInt a, const APInt &b) {
1161 if (div0 || b.isZero()) {
1162 div0 = true;
1163 return a;
1164 }
1165 return a.srem(RHS: b);
1166 });
1167
1168 return div0 ? Attribute() : result;
1169}
1170
1171Speculation::Speculatability arith::RemSIOp::getSpeculatability() {
1172 // X % 0 => UB
1173 // X % -1 is well-defined (always 0), unlike X / -1 which can overflow.
1174 if (matchPattern(value: getRhs(), pattern: m_IntRangeWithoutZeroS()))
1175 return Speculation::Speculatable;
1176
1177 return Speculation::NotSpeculatable;
1178}
1179
1180//===----------------------------------------------------------------------===//
1181// AndIOp
1182//===----------------------------------------------------------------------===//
1183
1184/// Fold `op(a, op(a, b))` to `op(a, b)` for an associative, commutative and
1185/// idempotent `op` (e.g. `and`, `or`).
1186template <typename OpTy>
1187static Value foldIdempotentOfSameOp(OpTy op) {
1188 for (bool reversePrev : {false, true}) {
1189 auto prev = (reversePrev ? op.getRhs() : op.getLhs())
1190 .template getDefiningOp<OpTy>();
1191 if (!prev)
1192 continue;
1193
1194 Value other = (reversePrev ? op.getLhs() : op.getRhs());
1195 if (other != prev.getLhs() && other != prev.getRhs())
1196 continue;
1197
1198 return prev.getResult();
1199 }
1200 return {};
1201}
1202
1203OpFoldResult arith::AndIOp::fold(FoldAdaptor adaptor) {
1204 /// and(x, 0) -> 0
1205 if (matchPattern(attr: adaptor.getRhs(), pattern: m_Zero()))
1206 return getRhs();
1207 /// and(x, allOnes) -> x
1208 APInt intValue;
1209 if (matchPattern(attr: adaptor.getRhs(), pattern: m_ConstantInt(bind_value: &intValue)) &&
1210 intValue.isAllOnes())
1211 return getLhs();
1212 /// and(x, not(x)) -> 0
1213 if (matchPattern(value: getRhs(), pattern: m_Op<XOrIOp>(matchers: matchers::m_Val(v: getLhs()),
1214 matchers: m_ConstantInt(bind_value: &intValue))) &&
1215 intValue.isAllOnes())
1216 return Builder(getContext()).getZeroAttr(type: getType());
1217 /// and(not(x), x) -> 0
1218 if (matchPattern(value: getLhs(), pattern: m_Op<XOrIOp>(matchers: matchers::m_Val(v: getRhs()),
1219 matchers: m_ConstantInt(bind_value: &intValue))) &&
1220 intValue.isAllOnes())
1221 return Builder(getContext()).getZeroAttr(type: getType());
1222
1223 /// and(a, and(a, b)) -> and(a, b)
1224 if (Value result = foldIdempotentOfSameOp(op: *this))
1225 return result;
1226
1227 return constFoldBinaryOp<IntegerAttr>(
1228 operands: adaptor.getOperands(),
1229 calculate: [](APInt a, const APInt &b) { return std::move(a) & b; });
1230}
1231
1232//===----------------------------------------------------------------------===//
1233// OrIOp
1234//===----------------------------------------------------------------------===//
1235
1236OpFoldResult arith::OrIOp::fold(FoldAdaptor adaptor) {
1237 if (APInt rhsVal; matchPattern(attr: adaptor.getRhs(), pattern: m_ConstantInt(bind_value: &rhsVal))) {
1238 /// or(x, 0) -> x
1239 if (rhsVal.isZero())
1240 return getLhs();
1241 /// or(x, <all ones>) -> <all ones>
1242 if (rhsVal.isAllOnes())
1243 return adaptor.getRhs();
1244 }
1245
1246 APInt intValue;
1247 /// or(x, xor(x, 1)) -> 1
1248 if (matchPattern(value: getRhs(), pattern: m_Op<XOrIOp>(matchers: matchers::m_Val(v: getLhs()),
1249 matchers: m_ConstantInt(bind_value: &intValue))) &&
1250 intValue.isAllOnes())
1251 return getRhs().getDefiningOp<XOrIOp>().getRhs();
1252 /// or(xor(x, 1), x) -> 1
1253 if (matchPattern(value: getLhs(), pattern: m_Op<XOrIOp>(matchers: matchers::m_Val(v: getRhs()),
1254 matchers: m_ConstantInt(bind_value: &intValue))) &&
1255 intValue.isAllOnes())
1256 return getLhs().getDefiningOp<XOrIOp>().getRhs();
1257
1258 /// or(a, or(a, b)) -> or(a, b)
1259 if (Value result = foldIdempotentOfSameOp(op: *this))
1260 return result;
1261
1262 return constFoldBinaryOp<IntegerAttr>(
1263 operands: adaptor.getOperands(),
1264 calculate: [](APInt a, const APInt &b) { return std::move(a) | b; });
1265}
1266
1267//===----------------------------------------------------------------------===//
1268// XOrIOp
1269//===----------------------------------------------------------------------===//
1270
1271OpFoldResult arith::XOrIOp::fold(FoldAdaptor adaptor) {
1272 /// xor(x, 0) -> x
1273 if (matchPattern(attr: adaptor.getRhs(), pattern: m_Zero()))
1274 return getLhs();
1275 /// xor(x, x) -> 0
1276 if (getLhs() == getRhs()) {
1277 // A dynamically-shaped result cannot be a constant; bail before
1278 // getZeroAttr, which would assert on a non-static shape.
1279 auto shapedType = dyn_cast<ShapedType>(Val: getType());
1280 if (!shapedType || shapedType.hasStaticShape())
1281 return Builder(getContext()).getZeroAttr(type: getType());
1282 }
1283 /// xor(xor(x, a), a) -> x
1284 /// xor(xor(a, x), a) -> x
1285 if (arith::XOrIOp prev = getLhs().getDefiningOp<arith::XOrIOp>()) {
1286 if (prev.getRhs() == getRhs())
1287 return prev.getLhs();
1288 if (prev.getLhs() == getRhs())
1289 return prev.getRhs();
1290 }
1291 /// xor(a, xor(x, a)) -> x
1292 /// xor(a, xor(a, x)) -> x
1293 if (arith::XOrIOp prev = getRhs().getDefiningOp<arith::XOrIOp>()) {
1294 if (prev.getRhs() == getLhs())
1295 return prev.getLhs();
1296 if (prev.getLhs() == getLhs())
1297 return prev.getRhs();
1298 }
1299
1300 return constFoldBinaryOp<IntegerAttr>(
1301 operands: adaptor.getOperands(),
1302 calculate: [](APInt a, const APInt &b) { return std::move(a) ^ b; });
1303}
1304
1305void arith::XOrIOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
1306 MLIRContext *context) {
1307 patterns.add<XOrIXOrIConstant, XOrINotCmpI, XOrIOfExtUI, XOrIOfExtSI>(
1308 arg&: context);
1309}
1310
1311//===----------------------------------------------------------------------===//
1312// NegFOp
1313//===----------------------------------------------------------------------===//
1314
1315OpFoldResult arith::NegFOp::fold(FoldAdaptor adaptor) {
1316 /// negf(negf(x)) -> x
1317 if (auto op = this->getOperand().getDefiningOp<arith::NegFOp>())
1318 return op.getOperand();
1319 return constFoldUnaryOpConditional<FloatAttr>(
1320 operands: adaptor.getOperands(), calculate: [](const APFloat &a) -> std::optional<APFloat> {
1321 if (!APFloat::semanticsHasSignedRepr(a.getSemantics()))
1322 return std::nullopt;
1323 return -a;
1324 });
1325}
1326
1327//===----------------------------------------------------------------------===//
1328// FlushDenormalsOp
1329//===----------------------------------------------------------------------===//
1330
1331OpFoldResult arith::FlushDenormalsOp::fold(FoldAdaptor adaptor) {
1332 // TODO: Fold flush_denormals if the floating-point type does not support
1333 // denormals. There is currently no API to query this information from
1334 // APFloat.
1335
1336 // flush_denormals(flush_denormals(x)) -> flush_denormals(x)
1337 if (auto op = this->getOperand().getDefiningOp<arith::FlushDenormalsOp>())
1338 return op.getResult();
1339
1340 // Constant-fold flush_denormals if the operand is a constant.
1341 return constFoldUnaryOp<FloatAttr>(
1342 operands: adaptor.getOperands(), calculate: [](const APFloat &a) {
1343 if (a.isDenormal())
1344 return APFloat::getZero(Sem: a.getSemantics(), Negative: a.isNegative());
1345 return a;
1346 });
1347}
1348
1349//===----------------------------------------------------------------------===//
1350// AddFOp
1351//===----------------------------------------------------------------------===//
1352
1353OpFoldResult arith::AddFOp::fold(FoldAdaptor adaptor) {
1354 // addf(x, -0) -> x
1355 if (matchPattern(attr: adaptor.getRhs(), pattern: m_NegZeroFloat()))
1356 return getLhs();
1357 if (matchPattern(attr: adaptor.getRhs(), pattern: m_PosZeroFloat()) &&
1358 bitEnumContainsAll(bits: adaptor.getFastmath(), bit: FastMathFlags::nsz))
1359 return getLhs();
1360
1361 auto rm = getRoundingmode();
1362 return constFoldBinaryOp<FloatAttr>(
1363 operands: adaptor.getOperands(), calculate: [rm](const APFloat &a, const APFloat &b) {
1364 APFloat result(a);
1365 result.add(RHS: b, RM: convertArithRoundingModeToLLVMIR(roundingMode: rm));
1366 return result;
1367 });
1368}
1369
1370void arith::AddFOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
1371 MLIRContext *context) {
1372 patterns.add<AddFOfNegFLhs, AddFOfNegFRhs>(arg&: context);
1373}
1374
1375//===----------------------------------------------------------------------===//
1376// SubFOp
1377//===----------------------------------------------------------------------===//
1378
1379OpFoldResult arith::SubFOp::fold(FoldAdaptor adaptor) {
1380 // subf(x, +0) -> x
1381 if (matchPattern(attr: adaptor.getRhs(), pattern: m_PosZeroFloat()))
1382 return getLhs();
1383 if (matchPattern(attr: adaptor.getRhs(), pattern: m_NegZeroFloat()) &&
1384 bitEnumContainsAll(bits: adaptor.getFastmath(), bit: FastMathFlags::nsz))
1385 return getLhs();
1386
1387 auto rm = getRoundingmode();
1388 return constFoldBinaryOpConditional<FloatAttr>(
1389 operands: adaptor.getOperands(),
1390 calculate: [rm](const APFloat &a, const APFloat &b) -> std::optional<APFloat> {
1391 // A negative difference is not representable in types such as
1392 // f8E8M0FNU, and APFloat::subtract does not support computing it.
1393 if (!APFloat::semanticsHasSignedRepr(a.getSemantics()) &&
1394 a.compare(RHS: b) == APFloat::cmpLessThan)
1395 return std::nullopt;
1396 APFloat result(a);
1397 result.subtract(RHS: b, RM: convertArithRoundingModeToLLVMIR(roundingMode: rm));
1398 return result;
1399 });
1400}
1401
1402void arith::SubFOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
1403 MLIRContext *context) {
1404 patterns.add<SubFOfNegZero>(arg&: context);
1405}
1406
1407namespace {
1408
1409/// Narrow an extremum whose operands were extended from the result type:
1410///
1411/// trunc(extremum(ext(lhs), ext(rhs))) -> extremum(lhs, rhs)
1412///
1413/// The concrete extension is part of the pattern so each extremum is only
1414/// registered with extensions that preserve its ordering.
1415/// For floating-point types, also require the extension to preserve every
1416/// source value relevant under the extremum's fast-math flags.
1417template <typename TruncOp, typename ExtOp, typename ExtremumOp>
1418struct NarrowExtremum final : OpRewritePattern<TruncOp> {
1419 using OpRewritePattern<TruncOp>::OpRewritePattern;
1420
1421 LogicalResult matchAndRewrite(TruncOp truncOp,
1422 PatternRewriter &rewriter) const override {
1423 auto extremumOp = truncOp.getIn().template getDefiningOp<ExtremumOp>();
1424 if (!extremumOp || !extremumOp->hasOneUse())
1425 return failure();
1426
1427 auto lhsExt = extremumOp.getLhs().template getDefiningOp<ExtOp>();
1428 auto rhsExt = extremumOp.getRhs().template getDefiningOp<ExtOp>();
1429 if (!lhsExt || !rhsExt)
1430 return failure();
1431
1432 Value lhs = lhsExt.getIn();
1433 Value rhs = rhsExt.getIn();
1434 Type narrowType = truncOp.getType();
1435 if (lhs.getType() != narrowType || rhs.getType() != narrowType)
1436 return failure();
1437
1438 // A floating-point extension is not necessarily lossless between arbitrary
1439 // floating-point semantics, even when the destination has a larger bit
1440 // width. In particular, it may lose the sign of zero or quiet a signaling
1441 // NaN, either of which can change an extremum's result. `nnan` lets us
1442 // disregard NaN representation differences, but all other relevant source
1443 // values must be preserved.
1444 if (auto narrowFloatType =
1445 dyn_cast<FloatType>(Val: getElementTypeOrSelf(type: narrowType))) {
1446 auto wideFloatType =
1447 dyn_cast<FloatType>(getElementTypeOrSelf(extremumOp.getType()));
1448 if (!wideFloatType)
1449 return failure();
1450
1451 const llvm::fltSemantics &narrowSemantics =
1452 narrowFloatType.getFloatSemantics();
1453 const llvm::fltSemantics &wideSemantics =
1454 wideFloatType.getFloatSemantics();
1455 bool ignoreNaNs = false;
1456 if constexpr (std::is_same_v<TruncOp, TruncFOp>)
1457 ignoreNaNs =
1458 bitEnumContainsAll(extremumOp.getFastmath(), FastMathFlags::nnan);
1459 if (!llvm::APFloatBase::isLosslesslyConvertibleTo(
1460 From: narrowSemantics, To: wideSemantics, IgnoreNaNs: ignoreNaNs))
1461 return failure();
1462 }
1463
1464 rewriter.replaceOpWithNewOp<ExtremumOp>(
1465 truncOp, TypeRange{narrowType}, ValueRange{lhs, rhs},
1466 extremumOp.getProperties(),
1467 extremumOp->getDiscardableAttrDictionary().getValue());
1468 return success();
1469 }
1470};
1471
1472} // namespace
1473
1474//===----------------------------------------------------------------------===//
1475// MaximumFOp
1476//===----------------------------------------------------------------------===//
1477
1478OpFoldResult arith::MaximumFOp::fold(FoldAdaptor adaptor) {
1479 // maximumf(x,x) -> x
1480 if (getLhs() == getRhs())
1481 return getRhs();
1482
1483 // maximumf(x, -inf) -> x
1484 if (matchPattern(attr: adaptor.getRhs(), pattern: m_NegInfFloat()))
1485 return getLhs();
1486
1487 return constFoldBinaryOp<FloatAttr>(operands: adaptor.getOperands(), calculate&: llvm::maximum);
1488}
1489
1490//===----------------------------------------------------------------------===//
1491// MaxNumFOp
1492//===----------------------------------------------------------------------===//
1493
1494OpFoldResult arith::MaxNumFOp::fold(FoldAdaptor adaptor) {
1495 // maxnumf(x,x) -> x
1496 if (getLhs() == getRhs())
1497 return getRhs();
1498
1499 // maxnumf(x, NaN) -> x
1500 if (matchPattern(attr: adaptor.getRhs(), pattern: m_NaNFloat()))
1501 return getLhs();
1502
1503 return constFoldBinaryOp<FloatAttr>(operands: adaptor.getOperands(), calculate&: llvm::maxnum);
1504}
1505
1506//===----------------------------------------------------------------------===//
1507// MaximumNumFOp
1508//===----------------------------------------------------------------------===//
1509
1510OpFoldResult arith::MaximumNumFOp::fold(FoldAdaptor adaptor) {
1511 // maximumnumf(x,x) -> x
1512 if (getLhs() == getRhs())
1513 return getRhs();
1514
1515 // maximumnumf(x, NaN) -> x
1516 if (matchPattern(attr: adaptor.getRhs(), pattern: m_NaNFloat()))
1517 return getLhs();
1518
1519 return constFoldBinaryOp<FloatAttr>(operands: adaptor.getOperands(), calculate&: llvm::maximumnum);
1520}
1521
1522//===----------------------------------------------------------------------===//
1523// MaxSIOp
1524//===----------------------------------------------------------------------===//
1525
1526OpFoldResult MaxSIOp::fold(FoldAdaptor adaptor) {
1527 // maxsi(x,x) -> x
1528 if (getLhs() == getRhs())
1529 return getRhs();
1530
1531 if (APInt intValue;
1532 matchPattern(attr: adaptor.getRhs(), pattern: m_ConstantInt(bind_value: &intValue))) {
1533 // maxsi(x,MAX_INT) -> MAX_INT
1534 if (intValue.isMaxSignedValue())
1535 return getRhs();
1536 // maxsi(x, MIN_INT) -> x
1537 if (intValue.isMinSignedValue())
1538 return getLhs();
1539 }
1540
1541 return constFoldBinaryOp<IntegerAttr>(operands: adaptor.getOperands(),
1542 calculate&: llvm::APIntOps::smax);
1543}
1544
1545//===----------------------------------------------------------------------===//
1546// MaxUIOp
1547//===----------------------------------------------------------------------===//
1548
1549OpFoldResult MaxUIOp::fold(FoldAdaptor adaptor) {
1550 // maxui(x,x) -> x
1551 if (getLhs() == getRhs())
1552 return getRhs();
1553
1554 if (APInt intValue;
1555 matchPattern(attr: adaptor.getRhs(), pattern: m_ConstantInt(bind_value: &intValue))) {
1556 // maxui(x,MAX_INT) -> MAX_INT
1557 if (intValue.isMaxValue())
1558 return getRhs();
1559 // maxui(x, MIN_INT) -> x
1560 if (intValue.isMinValue())
1561 return getLhs();
1562 }
1563
1564 return constFoldBinaryOp<IntegerAttr>(operands: adaptor.getOperands(),
1565 calculate&: llvm::APIntOps::umax);
1566}
1567
1568//===----------------------------------------------------------------------===//
1569// MinimumFOp
1570//===----------------------------------------------------------------------===//
1571
1572OpFoldResult arith::MinimumFOp::fold(FoldAdaptor adaptor) {
1573 // minimumf(x,x) -> x
1574 if (getLhs() == getRhs())
1575 return getRhs();
1576
1577 // minimumf(x, +inf) -> x
1578 if (matchPattern(attr: adaptor.getRhs(), pattern: m_PosInfFloat()))
1579 return getLhs();
1580
1581 return constFoldBinaryOp<FloatAttr>(operands: adaptor.getOperands(), calculate&: llvm::minimum);
1582}
1583
1584//===----------------------------------------------------------------------===//
1585// MinNumFOp
1586//===----------------------------------------------------------------------===//
1587
1588OpFoldResult arith::MinNumFOp::fold(FoldAdaptor adaptor) {
1589 // minnumf(x,x) -> x
1590 if (getLhs() == getRhs())
1591 return getRhs();
1592
1593 // minnumf(x, NaN) -> x
1594 if (matchPattern(attr: adaptor.getRhs(), pattern: m_NaNFloat()))
1595 return getLhs();
1596
1597 return constFoldBinaryOp<FloatAttr>(operands: adaptor.getOperands(), calculate&: llvm::minnum);
1598}
1599
1600//===----------------------------------------------------------------------===//
1601// MinimumNumFOp
1602//===----------------------------------------------------------------------===//
1603
1604OpFoldResult arith::MinimumNumFOp::fold(FoldAdaptor adaptor) {
1605 // minimumnumf(x,x) -> x
1606 if (getLhs() == getRhs())
1607 return getRhs();
1608
1609 // minimumnumf(x, NaN) -> x
1610 if (matchPattern(attr: adaptor.getRhs(), pattern: m_NaNFloat()))
1611 return getLhs();
1612
1613 return constFoldBinaryOp<FloatAttr>(operands: adaptor.getOperands(), calculate&: llvm::minimumnum);
1614}
1615
1616//===----------------------------------------------------------------------===//
1617// MinSIOp
1618//===----------------------------------------------------------------------===//
1619
1620OpFoldResult MinSIOp::fold(FoldAdaptor adaptor) {
1621 // minsi(x,x) -> x
1622 if (getLhs() == getRhs())
1623 return getRhs();
1624
1625 if (APInt intValue;
1626 matchPattern(attr: adaptor.getRhs(), pattern: m_ConstantInt(bind_value: &intValue))) {
1627 // minsi(x,MIN_INT) -> MIN_INT
1628 if (intValue.isMinSignedValue())
1629 return getRhs();
1630 // minsi(x, MAX_INT) -> x
1631 if (intValue.isMaxSignedValue())
1632 return getLhs();
1633 }
1634
1635 return constFoldBinaryOp<IntegerAttr>(operands: adaptor.getOperands(),
1636 calculate&: llvm::APIntOps::smin);
1637}
1638
1639//===----------------------------------------------------------------------===//
1640// MinUIOp
1641//===----------------------------------------------------------------------===//
1642
1643OpFoldResult MinUIOp::fold(FoldAdaptor adaptor) {
1644 // minui(x,x) -> x
1645 if (getLhs() == getRhs())
1646 return getRhs();
1647
1648 if (APInt intValue;
1649 matchPattern(attr: adaptor.getRhs(), pattern: m_ConstantInt(bind_value: &intValue))) {
1650 // minui(x,MIN_INT) -> MIN_INT
1651 if (intValue.isMinValue())
1652 return getRhs();
1653 // minui(x, MAX_INT) -> x
1654 if (intValue.isMaxValue())
1655 return getLhs();
1656 }
1657
1658 return constFoldBinaryOp<IntegerAttr>(operands: adaptor.getOperands(),
1659 calculate&: llvm::APIntOps::umin);
1660}
1661
1662//===----------------------------------------------------------------------===//
1663// MulFOp
1664//===----------------------------------------------------------------------===//
1665
1666OpFoldResult arith::MulFOp::fold(FoldAdaptor adaptor) {
1667 // mulf(x, 1) -> x
1668 if (matchPattern(attr: adaptor.getRhs(), pattern: m_OneFloat()))
1669 return getLhs();
1670
1671 // Match LLVM InstSimplify: with nnan+nsz, X * 0 -> 0 for a non-constant X.
1672 // When both operands are constants, fall through to APFloat so IEEE signed
1673 // zeros are preserved (e.g. (-c) * +0.0 == -0.0).
1674 if (arith::bitEnumContainsAll(bits: getFastmath(), bit: arith::FastMathFlags::nnan |
1675 arith::FastMathFlags::nsz) &&
1676 !(adaptor.getLhs() && adaptor.getRhs())) {
1677 // mulf(x, 0) -> 0
1678 if (matchPattern(attr: adaptor.getRhs(), pattern: m_AnyZeroFloat()))
1679 return getRhs();
1680 if (matchPattern(attr: adaptor.getLhs(), pattern: m_AnyZeroFloat()))
1681 return getLhs();
1682 }
1683
1684 auto rm = getRoundingmode();
1685 return constFoldBinaryOp<FloatAttr>(
1686 operands: adaptor.getOperands(), calculate: [rm](const APFloat &a, const APFloat &b) {
1687 APFloat result(a);
1688 result.multiply(RHS: b, RM: convertArithRoundingModeToLLVMIR(roundingMode: rm));
1689 return result;
1690 });
1691}
1692
1693void arith::MulFOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
1694 MLIRContext *context) {
1695 patterns.add<MulFOfNegF>(arg&: context);
1696}
1697
1698//===----------------------------------------------------------------------===//
1699// DivFOp
1700//===----------------------------------------------------------------------===//
1701
1702OpFoldResult arith::DivFOp::fold(FoldAdaptor adaptor) {
1703 // divf(x, 1) -> x
1704 if (matchPattern(attr: adaptor.getRhs(), pattern: m_OneFloat()))
1705 return getLhs();
1706
1707 auto rm = getRoundingmode();
1708 return constFoldBinaryOp<FloatAttr>(
1709 operands: adaptor.getOperands(), calculate: [rm](const APFloat &a, const APFloat &b) {
1710 APFloat result(a);
1711 result.divide(RHS: b, RM: convertArithRoundingModeToLLVMIR(roundingMode: rm));
1712 return result;
1713 });
1714}
1715
1716void arith::DivFOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
1717 MLIRContext *context) {
1718 patterns.add<DivFOfNegF>(arg&: context);
1719}
1720
1721//===----------------------------------------------------------------------===//
1722// RemFOp
1723//===----------------------------------------------------------------------===//
1724
1725OpFoldResult arith::RemFOp::fold(FoldAdaptor adaptor) {
1726 return constFoldBinaryOp<FloatAttr>(operands: adaptor.getOperands(),
1727 calculate: [](const APFloat &a, const APFloat &b) {
1728 APFloat result(a);
1729 // APFloat::mod() offers the remainder
1730 // behavior we want, i.e. the result has
1731 // the sign of LHS operand.
1732 (void)result.mod(RHS: b);
1733 return result;
1734 });
1735}
1736
1737//===----------------------------------------------------------------------===//
1738// Utility functions for verifying cast ops
1739//===----------------------------------------------------------------------===//
1740
1741template <typename... Types>
1742using type_list = std::tuple<Types...> *;
1743
1744/// Returns a non-null type only if the provided type is one of the allowed
1745/// types or one of the allowed shaped types of the allowed types. Returns the
1746/// element type if a valid shaped type is provided.
1747template <typename... ShapedTypes, typename... ElementTypes>
1748static Type getUnderlyingType(Type type, type_list<ShapedTypes...>,
1749 type_list<ElementTypes...>) {
1750 if (llvm::isa<ShapedType>(Val: type) && !llvm::isa<ShapedTypes...>(type))
1751 return {};
1752
1753 auto underlyingType = getElementTypeOrSelf(type);
1754 if (!llvm::isa<ElementTypes...>(underlyingType))
1755 return {};
1756
1757 return underlyingType;
1758}
1759
1760/// Get allowed underlying types for vectors and tensors.
1761template <typename... ElementTypes>
1762static Type getTypeIfLike(Type type) {
1763 return getUnderlyingType(type, type_list<VectorType, TensorType>(),
1764 type_list<ElementTypes...>());
1765}
1766
1767/// Get allowed underlying types for vectors, tensors, and memrefs.
1768template <typename... ElementTypes>
1769static Type getTypeIfLikeOrMemRef(Type type) {
1770 return getUnderlyingType(type,
1771 type_list<VectorType, TensorType, MemRefType>(),
1772 type_list<ElementTypes...>());
1773}
1774
1775/// Return false if both types are ranked tensor with mismatching encoding.
1776static bool hasSameEncoding(Type typeA, Type typeB) {
1777 auto rankedTensorA = dyn_cast<RankedTensorType>(Val&: typeA);
1778 auto rankedTensorB = dyn_cast<RankedTensorType>(Val&: typeB);
1779 if (!rankedTensorA || !rankedTensorB)
1780 return true;
1781 return rankedTensorA.getEncoding() == rankedTensorB.getEncoding();
1782}
1783
1784static bool areValidCastInputsAndOutputs(TypeRange inputs, TypeRange outputs) {
1785 if (inputs.size() != 1 || outputs.size() != 1)
1786 return false;
1787 if (!hasSameEncoding(typeA: inputs.front(), typeB: outputs.front()))
1788 return false;
1789 return succeeded(Result: verifyCompatibleShapes(types1: inputs.front(), types2: outputs.front()));
1790}
1791
1792//===----------------------------------------------------------------------===//
1793// Verifiers for integer and floating point extension/truncation ops
1794//===----------------------------------------------------------------------===//
1795
1796// Extend ops can only extend to a wider type.
1797template <typename ValType, typename Op>
1798static LogicalResult verifyExtOp(Op op) {
1799 Type srcType = getElementTypeOrSelf(op.getIn().getType());
1800 Type dstType = getElementTypeOrSelf(op.getType());
1801
1802 if (llvm::cast<ValType>(srcType).getWidth() >=
1803 llvm::cast<ValType>(dstType).getWidth())
1804 return op.emitError("result type ")
1805 << dstType << " must be wider than operand type " << srcType;
1806
1807 return success();
1808}
1809
1810// Truncate ops can only truncate to a shorter type.
1811template <typename ValType, typename Op>
1812static LogicalResult verifyTruncateOp(Op op) {
1813 Type srcType = getElementTypeOrSelf(op.getIn().getType());
1814 Type dstType = getElementTypeOrSelf(op.getType());
1815
1816 if (llvm::cast<ValType>(srcType).getWidth() <=
1817 llvm::cast<ValType>(dstType).getWidth())
1818 return op.emitError("result type ")
1819 << dstType << " must be shorter than operand type " << srcType;
1820
1821 return success();
1822}
1823
1824/// Validate a cast that changes the width of a type.
1825template <template <typename> class WidthComparator, typename... ElementTypes>
1826static bool checkWidthChangeCast(TypeRange inputs, TypeRange outputs) {
1827 if (!areValidCastInputsAndOutputs(inputs, outputs))
1828 return false;
1829
1830 auto srcType = getTypeIfLike<ElementTypes...>(inputs.front());
1831 auto dstType = getTypeIfLike<ElementTypes...>(outputs.front());
1832 if (!srcType || !dstType)
1833 return false;
1834
1835 return WidthComparator<unsigned>()(dstType.getIntOrFloatBitWidth(),
1836 srcType.getIntOrFloatBitWidth());
1837}
1838
1839/// Attempts to convert `sourceValue` to an APFloat value with
1840/// `targetSemantics` and `roundingMode`, without any information loss.
1841static FailureOr<APFloat>
1842convertFloatValue(APFloat sourceValue,
1843 const llvm::fltSemantics &targetSemantics,
1844 llvm::RoundingMode roundingMode = kDefaultRoundingMode) {
1845 // Reject special values that are not representable in the target type before
1846 // calling APFloat::convert, which would llvm_unreachable on them.
1847 using fltNonfiniteBehavior = llvm::fltNonfiniteBehavior;
1848 if (sourceValue.isInfinity() &&
1849 (targetSemantics.nonFiniteBehavior == fltNonfiniteBehavior::NanOnly ||
1850 targetSemantics.nonFiniteBehavior == fltNonfiniteBehavior::FiniteOnly))
1851 return failure();
1852 if (sourceValue.isNaN() &&
1853 targetSemantics.nonFiniteBehavior == fltNonfiniteBehavior::FiniteOnly)
1854 return failure();
1855 // A format without a sign (e.g. f8E8M0FNU) has no negative values, and
1856 // APFloat::convert reaches llvm_unreachable on a negative NaN or overflow.
1857 if (sourceValue.isNegative() &&
1858 !APFloat::semanticsHasSignedRepr(targetSemantics))
1859 return failure();
1860
1861 bool losesInfo = false;
1862 auto status = sourceValue.convert(ToSemantics: targetSemantics, RM: roundingMode, losesInfo: &losesInfo);
1863 if (losesInfo || status != APFloat::opOK)
1864 return failure();
1865
1866 return sourceValue;
1867}
1868
1869//===----------------------------------------------------------------------===//
1870// ExtUIOp
1871//===----------------------------------------------------------------------===//
1872
1873OpFoldResult arith::ExtUIOp::fold(FoldAdaptor adaptor) {
1874 if (auto lhs = getIn().getDefiningOp<ExtUIOp>()) {
1875 // Only the inner extension's nneg speaks about the surviving source; the
1876 // outer flag described the already-extended value.
1877 setNonNeg(lhs.getNonNeg());
1878 getInMutable().assign(value: lhs.getIn());
1879 return getResult();
1880 }
1881
1882 Type resType = getElementTypeOrSelf(type: getType());
1883 unsigned bitWidth = llvm::cast<IntegerType>(Val&: resType).getWidth();
1884 return constFoldCastOp<IntegerAttr, IntegerAttr>(
1885 operands: adaptor.getOperands(), resType: getType(),
1886 calculate: [bitWidth](const APInt &a, bool &castStatus) {
1887 return a.zext(width: bitWidth);
1888 });
1889}
1890
1891bool arith::ExtUIOp::areCastCompatible(TypeRange inputs, TypeRange outputs) {
1892 return checkWidthChangeCast<std::greater, IntegerType>(inputs, outputs);
1893}
1894
1895LogicalResult arith::ExtUIOp::verify() {
1896 return verifyExtOp<IntegerType>(op: *this);
1897}
1898
1899//===----------------------------------------------------------------------===//
1900// ExtSIOp
1901//===----------------------------------------------------------------------===//
1902
1903OpFoldResult arith::ExtSIOp::fold(FoldAdaptor adaptor) {
1904 if (auto lhs = getIn().getDefiningOp<ExtSIOp>()) {
1905 getInMutable().assign(value: lhs.getIn());
1906 return getResult();
1907 }
1908
1909 Type resType = getElementTypeOrSelf(type: getType());
1910 unsigned bitWidth = llvm::cast<IntegerType>(Val&: resType).getWidth();
1911 return constFoldCastOp<IntegerAttr, IntegerAttr>(
1912 operands: adaptor.getOperands(), resType: getType(),
1913 calculate: [bitWidth](const APInt &a, bool &castStatus) {
1914 return a.sext(width: bitWidth);
1915 });
1916}
1917
1918bool arith::ExtSIOp::areCastCompatible(TypeRange inputs, TypeRange outputs) {
1919 return checkWidthChangeCast<std::greater, IntegerType>(inputs, outputs);
1920}
1921
1922void arith::ExtSIOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
1923 MLIRContext *context) {
1924 patterns.add<ExtSIOfExtUI>(arg&: context);
1925}
1926
1927LogicalResult arith::ExtSIOp::verify() {
1928 return verifyExtOp<IntegerType>(op: *this);
1929}
1930
1931//===----------------------------------------------------------------------===//
1932// ExtFOp
1933//===----------------------------------------------------------------------===//
1934
1935/// Fold extension of float constants when there is no information loss due the
1936/// difference in fp semantics.
1937OpFoldResult arith::ExtFOp::fold(FoldAdaptor adaptor) {
1938 if (auto truncFOp = getOperand().getDefiningOp<TruncFOp>()) {
1939 if (truncFOp.getOperand().getType() == getType()) {
1940 arith::FastMathFlags truncFMF =
1941 truncFOp.getFastmath().value_or(u: arith::FastMathFlags::none);
1942 bool isTruncContract =
1943 bitEnumContainsAll(bits: truncFMF, bit: arith::FastMathFlags::contract);
1944 arith::FastMathFlags extFMF =
1945 getFastmath().value_or(u: arith::FastMathFlags::none);
1946 bool isExtContract =
1947 bitEnumContainsAll(bits: extFMF, bit: arith::FastMathFlags::contract);
1948 if (isTruncContract && isExtContract) {
1949 return truncFOp.getOperand();
1950 }
1951 }
1952 }
1953
1954 auto resElemType = cast<FloatType>(Val: getElementTypeOrSelf(type: getType()));
1955 const llvm::fltSemantics &targetSemantics = resElemType.getFloatSemantics();
1956 return constFoldCastOp<FloatAttr, FloatAttr>(
1957 operands: adaptor.getOperands(), resType: getType(),
1958 calculate: [&targetSemantics](const APFloat &a, bool &castStatus) {
1959 FailureOr<APFloat> result = convertFloatValue(sourceValue: a, targetSemantics);
1960 if (failed(Result: result)) {
1961 castStatus = false;
1962 return a;
1963 }
1964 return *result;
1965 });
1966}
1967
1968bool arith::ExtFOp::areCastCompatible(TypeRange inputs, TypeRange outputs) {
1969 return checkWidthChangeCast<std::greater, FloatType>(inputs, outputs);
1970}
1971
1972LogicalResult arith::ExtFOp::verify() { return verifyExtOp<FloatType>(op: *this); }
1973
1974//===----------------------------------------------------------------------===//
1975// ScalingExtFOp
1976//===----------------------------------------------------------------------===//
1977
1978/// Fold `calculate` element-wise over the operands of a scaling cast op. The
1979/// `constFoldBinaryOp` helpers cannot be used: they bail out unless both
1980/// operands have the same type, and `in` and `scale` never do.
1981static Attribute foldScalingCastOp(
1982 Attribute inAttr, Attribute scaleAttr, Type resultType,
1983 function_ref<std::optional<APFloat>(const APFloat &, const APFloat &)>
1984 calculate) {
1985 // Poison propagates, as it does in the generic constant folders.
1986 if (isa_and_nonnull<ub::PoisonAttr>(Val: inAttr))
1987 return inAttr;
1988 if (isa_and_nonnull<ub::PoisonAttr>(Val: scaleAttr))
1989 return scaleAttr;
1990
1991 if (!inAttr || !scaleAttr || !resultType)
1992 return {};
1993
1994 if (auto inFloat = dyn_cast<FloatAttr>(Val&: inAttr)) {
1995 auto scaleFloat = dyn_cast<FloatAttr>(Val&: scaleAttr);
1996 if (!scaleFloat)
1997 return {};
1998 std::optional<APFloat> result =
1999 calculate(inFloat.getValue(), scaleFloat.getValue());
2000 if (!result)
2001 return {};
2002 return FloatAttr::get(type: resultType, value: *result);
2003 }
2004
2005 auto inElements = dyn_cast<DenseFPElementsAttr>(Val&: inAttr);
2006 auto scaleElements = dyn_cast<DenseFPElementsAttr>(Val&: scaleAttr);
2007 auto shapedResultType = dyn_cast<ShapedType>(Val&: resultType);
2008 if (!inElements || !scaleElements || !shapedResultType ||
2009 !shapedResultType.hasStaticShape() ||
2010 inElements.getNumElements() != scaleElements.getNumElements())
2011 return {};
2012
2013 // Both operands are splats, so avoid expanding the elements out.
2014 if (inElements.isSplat() && scaleElements.isSplat()) {
2015 std::optional<APFloat> result =
2016 calculate(inElements.getSplatValue<APFloat>(),
2017 scaleElements.getSplatValue<APFloat>());
2018 if (!result)
2019 return {};
2020 return DenseElementsAttr::get(type: shapedResultType, values: *result);
2021 }
2022
2023 SmallVector<APFloat> results;
2024 results.reserve(N: inElements.getNumElements());
2025 for (const auto &[in, scale] : llvm::zip_equal(t&: inElements, u&: scaleElements)) {
2026 std::optional<APFloat> result = calculate(in, scale);
2027 if (!result)
2028 return {};
2029 results.push_back(Elt: *result);
2030 }
2031 return DenseElementsAttr::get(type: shapedResultType, values: results);
2032}
2033
2034/// Only scales that already are f8E8M0FNU fold. What a wider scale means is
2035/// unsettled -- the tree does not say whether truncating one to f8E8M0FNU
2036/// rounds or takes its exponent -- so a folder should not settle it, see
2037/// https://github.com/llvm/llvm-project/issues/215295.
2038static bool isFoldableScalingScale(Value scale) {
2039 return isa<Float8E8M0FNUType>(Val: getElementTypeOrSelf(type: scale.getType()));
2040}
2041
2042OpFoldResult arith::ScalingExtFOp::fold(FoldAdaptor adaptor) {
2043 // scaling_extf(in, scale) -> mulf(extf(in), extf(scale)), matching the
2044 // expansion in ExpandOps.cpp. As in arith.extf, the widening steps only fold
2045 // when they are lossless.
2046 if (!isFoldableScalingScale(scale: getScale()))
2047 return {};
2048
2049 auto resElemType = cast<FloatType>(Val: getElementTypeOrSelf(type: getType()));
2050 const llvm::fltSemantics &resSemantics = resElemType.getFloatSemantics();
2051 return foldScalingCastOp(
2052 inAttr: adaptor.getIn(), scaleAttr: adaptor.getScale(), resultType: getType(),
2053 calculate: [&resSemantics](const APFloat &in,
2054 const APFloat &scale) -> std::optional<APFloat> {
2055 FailureOr<APFloat> inExt = convertFloatValue(sourceValue: in, targetSemantics: resSemantics);
2056 FailureOr<APFloat> scaleExt = convertFloatValue(sourceValue: scale, targetSemantics: resSemantics);
2057 if (failed(Result: inExt) || failed(Result: scaleExt))
2058 return std::nullopt;
2059 APFloat result(*inExt);
2060 result.multiply(RHS: *scaleExt, RM: kDefaultRoundingMode);
2061 return result;
2062 });
2063}
2064
2065bool arith::ScalingExtFOp::areCastCompatible(TypeRange inputs,
2066 TypeRange outputs) {
2067 return checkWidthChangeCast<std::greater, FloatType>(inputs: inputs.front(), outputs);
2068}
2069
2070LogicalResult arith::ScalingExtFOp::verify() {
2071 return verifyExtOp<FloatType>(op: *this);
2072}
2073
2074//===----------------------------------------------------------------------===//
2075// TruncIOp
2076//===----------------------------------------------------------------------===//
2077
2078OpFoldResult arith::TruncIOp::fold(FoldAdaptor adaptor) {
2079 if (matchPattern(value: getOperand(), pattern: m_Op<arith::ExtUIOp>()) ||
2080 matchPattern(value: getOperand(), pattern: m_Op<arith::ExtSIOp>())) {
2081 Value src = getOperand().getDefiningOp()->getOperand(idx: 0);
2082 Type srcType = getElementTypeOrSelf(type: src.getType());
2083 Type dstType = getElementTypeOrSelf(type: getType());
2084 // trunci(zexti(a)) -> trunci(a)
2085 // trunci(sexti(a)) -> trunci(a)
2086 if (llvm::cast<IntegerType>(Val&: srcType).getWidth() >
2087 llvm::cast<IntegerType>(Val&: dstType).getWidth()) {
2088 setOperand(src);
2089 return getResult();
2090 }
2091
2092 // trunci(zexti(a)) -> a
2093 // trunci(sexti(a)) -> a
2094 if (srcType == dstType)
2095 return src;
2096 }
2097
2098 // trunci(trunci(a)) -> trunci(a)
2099 if (auto truncOp = getOperand().getDefiningOp<arith::TruncIOp>()) {
2100 // Preserve a flag only if both truncations are lossless in that signedness.
2101 setOverflowFlags(getOverflowFlags() & truncOp.getOverflowFlags());
2102 setOperand(truncOp.getIn());
2103 return getResult();
2104 }
2105
2106 Type resType = getElementTypeOrSelf(type: getType());
2107 unsigned bitWidth = llvm::cast<IntegerType>(Val&: resType).getWidth();
2108 return constFoldCastOp<IntegerAttr, IntegerAttr>(
2109 operands: adaptor.getOperands(), resType: getType(),
2110 calculate: [bitWidth](const APInt &a, bool &castStatus) {
2111 return a.trunc(width: bitWidth);
2112 });
2113}
2114
2115bool arith::TruncIOp::areCastCompatible(TypeRange inputs, TypeRange outputs) {
2116 return checkWidthChangeCast<std::less, IntegerType>(inputs, outputs);
2117}
2118
2119void arith::TruncIOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
2120 MLIRContext *context) {
2121 patterns.add<NarrowExtremum<TruncIOp, ExtSIOp, MaxSIOp>,
2122 NarrowExtremum<TruncIOp, ExtSIOp, MinSIOp>,
2123 NarrowExtremum<TruncIOp, ExtUIOp, MaxUIOp>,
2124 NarrowExtremum<TruncIOp, ExtUIOp, MinUIOp>, TruncIExtSIToExtSI,
2125 TruncIExtUIToExtUI, TruncIShrSIToTrunciShrUI>(arg&: context);
2126}
2127
2128LogicalResult arith::TruncIOp::verify() {
2129 return verifyTruncateOp<IntegerType>(op: *this);
2130}
2131
2132//===----------------------------------------------------------------------===//
2133// TruncFOp
2134//===----------------------------------------------------------------------===//
2135
2136/// Perform safe const propagation for truncf, i.e., only propagate if FP value
2137/// can be represented without precision loss.
2138OpFoldResult arith::TruncFOp::fold(FoldAdaptor adaptor) {
2139 auto resElemType = cast<FloatType>(Val: getElementTypeOrSelf(type: getType()));
2140 if (auto extOp = getOperand().getDefiningOp<arith::ExtFOp>()) {
2141 Value src = extOp.getIn();
2142 auto srcType = cast<FloatType>(Val: getElementTypeOrSelf(type: src.getType()));
2143 auto intermediateType =
2144 cast<FloatType>(Val: getElementTypeOrSelf(type: extOp.getType()));
2145 // Check whether every source value round-trips through the intermediate
2146 // type, including signaling NaNs and signed zero.
2147 if (llvm::APFloatBase::isLosslesslyConvertibleTo(
2148 From: srcType.getFloatSemantics(),
2149 To: intermediateType.getFloatSemantics())) {
2150 // truncf(extf(a)) -> truncf(a)
2151 if (srcType.getWidth() > resElemType.getWidth()) {
2152 setOperand(src);
2153 return getResult();
2154 }
2155
2156 // truncf(extf(a)) -> a
2157 if (srcType == resElemType)
2158 return src;
2159 }
2160 }
2161
2162 const llvm::fltSemantics &targetSemantics = resElemType.getFloatSemantics();
2163 return constFoldCastOp<FloatAttr, FloatAttr>(
2164 operands: adaptor.getOperands(), resType: getType(),
2165 calculate: [this, &targetSemantics](const APFloat &a, bool &castStatus) {
2166 llvm::RoundingMode llvmRoundingMode =
2167 convertArithRoundingModeToLLVMIR(roundingMode: getRoundingmode());
2168 FailureOr<APFloat> result =
2169 convertFloatValue(sourceValue: a, targetSemantics, roundingMode: llvmRoundingMode);
2170 if (failed(Result: result)) {
2171 castStatus = false;
2172 return a;
2173 }
2174 return *result;
2175 });
2176}
2177
2178void arith::TruncFOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
2179 MLIRContext *context) {
2180 patterns.add<NarrowExtremum<TruncFOp, ExtFOp, MaximumFOp>,
2181 NarrowExtremum<TruncFOp, ExtFOp, MaxNumFOp>,
2182 NarrowExtremum<TruncFOp, ExtFOp, MaximumNumFOp>,
2183 NarrowExtremum<TruncFOp, ExtFOp, MinimumFOp>,
2184 NarrowExtremum<TruncFOp, ExtFOp, MinNumFOp>,
2185 NarrowExtremum<TruncFOp, ExtFOp, MinimumNumFOp>,
2186 TruncFSIToFPToSIToFP, TruncFUIToFPToUIToFP>(arg&: context);
2187}
2188
2189bool arith::TruncFOp::areCastCompatible(TypeRange inputs, TypeRange outputs) {
2190 return checkWidthChangeCast<std::less, FloatType>(inputs, outputs);
2191}
2192
2193LogicalResult arith::TruncFOp::verify() {
2194 return verifyTruncateOp<FloatType>(op: *this);
2195}
2196
2197//===----------------------------------------------------------------------===//
2198// ConvertFOp
2199//===----------------------------------------------------------------------===//
2200
2201OpFoldResult arith::ConvertFOp::fold(FoldAdaptor adaptor) {
2202 auto resElemType = cast<FloatType>(Val: getElementTypeOrSelf(type: getType()));
2203 const llvm::fltSemantics &targetSemantics = resElemType.getFloatSemantics();
2204 return constFoldCastOp<FloatAttr, FloatAttr>(
2205 operands: adaptor.getOperands(), resType: getType(),
2206 calculate: [this, &targetSemantics](const APFloat &a, bool &castStatus) {
2207 llvm::RoundingMode llvmRoundingMode =
2208 convertArithRoundingModeToLLVMIR(roundingMode: getRoundingmode());
2209 FailureOr<APFloat> result =
2210 convertFloatValue(sourceValue: a, targetSemantics, roundingMode: llvmRoundingMode);
2211 if (failed(Result: result)) {
2212 castStatus = false;
2213 return a;
2214 }
2215 return *result;
2216 });
2217}
2218
2219bool arith::ConvertFOp::areCastCompatible(TypeRange inputs, TypeRange outputs) {
2220 if (!areValidCastInputsAndOutputs(inputs, outputs))
2221 return false;
2222 auto srcType = getTypeIfLike<FloatType>(type: inputs.front());
2223 auto dstType = getTypeIfLike<FloatType>(type: outputs.front());
2224 if (!srcType || !dstType)
2225 return false;
2226 return srcType != dstType &&
2227 srcType.getIntOrFloatBitWidth() == dstType.getIntOrFloatBitWidth();
2228}
2229
2230LogicalResult arith::ConvertFOp::verify() {
2231 auto srcType = cast<FloatType>(Val: getElementTypeOrSelf(type: getIn().getType()));
2232 auto dstType = cast<FloatType>(Val: getElementTypeOrSelf(type: getType()));
2233 if (srcType == dstType)
2234 return emitError(message: "result element type ")
2235 << dstType << " must be different from operand element type "
2236 << srcType;
2237 if (srcType.getWidth() != dstType.getWidth())
2238 return emitError(message: "result element type ")
2239 << dstType << " must have the same bitwidth as operand element type "
2240 << srcType;
2241 return success();
2242}
2243
2244//===----------------------------------------------------------------------===//
2245// ScalingTruncFOp
2246//===----------------------------------------------------------------------===//
2247
2248OpFoldResult arith::ScalingTruncFOp::fold(FoldAdaptor adaptor) {
2249 // scaling_truncf(in, scale) -> truncf(in / extf(scale)), matching the
2250 // expansion in ExpandOps.cpp. Unlike scaling_extf, the scale is widened to
2251 // the type of `in` rather than to the result type.
2252 if (!isFoldableScalingScale(scale: getScale()))
2253 return {};
2254
2255 auto inElemType = cast<FloatType>(Val: getElementTypeOrSelf(type: getIn().getType()));
2256 auto resElemType = cast<FloatType>(Val: getElementTypeOrSelf(type: getType()));
2257 const llvm::fltSemantics &inSemantics = inElemType.getFloatSemantics();
2258 const llvm::fltSemantics &resSemantics = resElemType.getFloatSemantics();
2259 llvm::RoundingMode roundingMode =
2260 convertArithRoundingModeToLLVMIR(roundingMode: getRoundingmode());
2261 return foldScalingCastOp(
2262 inAttr: adaptor.getIn(), scaleAttr: adaptor.getScale(), resultType: getType(),
2263 calculate: [&](const APFloat &in, const APFloat &scale) -> std::optional<APFloat> {
2264 FailureOr<APFloat> scaleExt = convertFloatValue(sourceValue: scale, targetSemantics: inSemantics);
2265 if (failed(Result: scaleExt))
2266 return std::nullopt;
2267 APFloat quotient(in);
2268 quotient.divide(RHS: *scaleExt, RM: kDefaultRoundingMode);
2269 FailureOr<APFloat> result =
2270 convertFloatValue(sourceValue: quotient, targetSemantics: resSemantics, roundingMode);
2271 if (failed(Result: result))
2272 return std::nullopt;
2273 return *result;
2274 });
2275}
2276
2277bool arith::ScalingTruncFOp::areCastCompatible(TypeRange inputs,
2278 TypeRange outputs) {
2279 return checkWidthChangeCast<std::less, FloatType>(inputs: inputs.front(), outputs);
2280}
2281
2282LogicalResult arith::ScalingTruncFOp::verify() {
2283 return verifyTruncateOp<FloatType>(op: *this);
2284}
2285
2286//===----------------------------------------------------------------------===//
2287// AndIOp
2288//===----------------------------------------------------------------------===//
2289
2290void arith::AndIOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
2291 MLIRContext *context) {
2292 patterns.add<AndIAndIConstant, AndOfExtUI, AndOfExtSI>(arg&: context);
2293}
2294
2295//===----------------------------------------------------------------------===//
2296// OrIOp
2297//===----------------------------------------------------------------------===//
2298
2299void arith::OrIOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
2300 MLIRContext *context) {
2301 patterns.add<OrIOrIConstant, OrOfExtUI, OrOfExtSI>(arg&: context);
2302}
2303
2304//===----------------------------------------------------------------------===//
2305// Verifiers for casts between integers and floats.
2306//===----------------------------------------------------------------------===//
2307
2308template <typename From, typename To>
2309static bool checkIntFloatCast(TypeRange inputs, TypeRange outputs) {
2310 if (!areValidCastInputsAndOutputs(inputs, outputs))
2311 return false;
2312
2313 auto srcType = getTypeIfLike<From>(inputs.front());
2314 auto dstType = getTypeIfLike<To>(outputs.back());
2315
2316 return srcType && dstType;
2317}
2318
2319//===----------------------------------------------------------------------===//
2320// UIToFPOp
2321//===----------------------------------------------------------------------===//
2322
2323bool arith::UIToFPOp::areCastCompatible(TypeRange inputs, TypeRange outputs) {
2324 return checkIntFloatCast<IntegerType, FloatType>(inputs, outputs);
2325}
2326
2327OpFoldResult arith::UIToFPOp::fold(FoldAdaptor adaptor) {
2328 Type resEleType = getElementTypeOrSelf(type: getType());
2329 return constFoldCastOp<IntegerAttr, FloatAttr>(
2330 operands: adaptor.getOperands(), resType: getType(),
2331 calculate: [&resEleType](const APInt &a, bool &castStatus) {
2332 FloatType floatTy = llvm::cast<FloatType>(Val&: resEleType);
2333 APFloat apf(floatTy.getFloatSemantics(),
2334 APInt::getZero(numBits: floatTy.getWidth()));
2335 apf.convertFromAPInt(Input: a, /*IsSigned=*/false,
2336 RM: APFloat::rmNearestTiesToEven);
2337 return apf;
2338 });
2339}
2340
2341void arith::UIToFPOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
2342 MLIRContext *context) {
2343 patterns.add<UIToFPOfExtUI>(arg&: context);
2344}
2345
2346//===----------------------------------------------------------------------===//
2347// SIToFPOp
2348//===----------------------------------------------------------------------===//
2349
2350bool arith::SIToFPOp::areCastCompatible(TypeRange inputs, TypeRange outputs) {
2351 return checkIntFloatCast<IntegerType, FloatType>(inputs, outputs);
2352}
2353
2354OpFoldResult arith::SIToFPOp::fold(FoldAdaptor adaptor) {
2355 Type resEleType = getElementTypeOrSelf(type: getType());
2356 return constFoldCastOp<IntegerAttr, FloatAttr>(
2357 operands: adaptor.getOperands(), resType: getType(),
2358 calculate: [&resEleType](const APInt &a, bool &castStatus) {
2359 FloatType floatTy = llvm::cast<FloatType>(Val&: resEleType);
2360 APFloat apf(floatTy.getFloatSemantics(),
2361 APInt::getZero(numBits: floatTy.getWidth()));
2362 if (a.isNegative() &&
2363 !APFloat::semanticsHasSignedRepr(floatTy.getFloatSemantics())) {
2364 castStatus = false;
2365 return apf;
2366 }
2367 apf.convertFromAPInt(Input: a, /*IsSigned=*/true,
2368 RM: APFloat::rmNearestTiesToEven);
2369 return apf;
2370 });
2371}
2372
2373void arith::SIToFPOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
2374 MLIRContext *context) {
2375 patterns.add<SIToFPOfExtSI, SIToFPOfExtUI>(arg&: context);
2376}
2377
2378//===----------------------------------------------------------------------===//
2379// FPToUIOp
2380//===----------------------------------------------------------------------===//
2381
2382bool arith::FPToUIOp::areCastCompatible(TypeRange inputs, TypeRange outputs) {
2383 return checkIntFloatCast<FloatType, IntegerType>(inputs, outputs);
2384}
2385
2386OpFoldResult arith::FPToUIOp::fold(FoldAdaptor adaptor) {
2387 Type resType = getElementTypeOrSelf(type: getType());
2388 unsigned bitWidth = llvm::cast<IntegerType>(Val&: resType).getWidth();
2389 return constFoldCastOp<FloatAttr, IntegerAttr>(
2390 operands: adaptor.getOperands(), resType: getType(),
2391 calculate: [&bitWidth](const APFloat &a, bool &castStatus) {
2392 bool ignored;
2393 APSInt api(bitWidth, /*isUnsigned=*/true);
2394 castStatus = APFloat::opInvalidOp !=
2395 a.convertToInteger(Result&: api, RM: APFloat::rmTowardZero, IsExact: &ignored);
2396 return api;
2397 });
2398}
2399
2400//===----------------------------------------------------------------------===//
2401// FPToSIOp
2402//===----------------------------------------------------------------------===//
2403
2404bool arith::FPToSIOp::areCastCompatible(TypeRange inputs, TypeRange outputs) {
2405 return checkIntFloatCast<FloatType, IntegerType>(inputs, outputs);
2406}
2407
2408OpFoldResult arith::FPToSIOp::fold(FoldAdaptor adaptor) {
2409 Type resType = getElementTypeOrSelf(type: getType());
2410 unsigned bitWidth = llvm::cast<IntegerType>(Val&: resType).getWidth();
2411 return constFoldCastOp<FloatAttr, IntegerAttr>(
2412 operands: adaptor.getOperands(), resType: getType(),
2413 calculate: [&bitWidth](const APFloat &a, bool &castStatus) {
2414 bool ignored;
2415 APSInt api(bitWidth, /*isUnsigned=*/false);
2416 castStatus = APFloat::opInvalidOp !=
2417 a.convertToInteger(Result&: api, RM: APFloat::rmTowardZero, IsExact: &ignored);
2418 return api;
2419 });
2420}
2421
2422//===----------------------------------------------------------------------===//
2423// IndexCastOp
2424//===----------------------------------------------------------------------===//
2425
2426/// Return the bit-width of \p t for the purpose of index_cast width checks.
2427/// For vector types use the element type; index maps to its internal storage
2428/// width (64 on all current targets).
2429static unsigned getIndexCastWidth(Type t) {
2430 if (auto intTy = dyn_cast<IntegerType>(Val: getElementTypeOrSelf(type: t)))
2431 return intTy.getWidth();
2432 return IndexType::kInternalStorageBitWidth;
2433}
2434
2435static bool areIndexCastCompatible(TypeRange inputs, TypeRange outputs) {
2436 if (!areValidCastInputsAndOutputs(inputs, outputs))
2437 return false;
2438
2439 auto srcType = getTypeIfLikeOrMemRef<IntegerType, IndexType>(type: inputs.front());
2440 auto dstType = getTypeIfLikeOrMemRef<IntegerType, IndexType>(type: outputs.front());
2441 if (!srcType || !dstType)
2442 return false;
2443
2444 return (srcType.isIndex() && dstType.isSignlessInteger()) ||
2445 (srcType.isSignlessInteger() && dstType.isIndex());
2446}
2447
2448bool arith::IndexCastOp::areCastCompatible(TypeRange inputs,
2449 TypeRange outputs) {
2450 return areIndexCastCompatible(inputs, outputs);
2451}
2452
2453OpFoldResult arith::IndexCastOp::fold(FoldAdaptor adaptor) {
2454 // index_cast(constant) -> constant
2455 unsigned resultBitwidth = 64; // Default for index integer attributes.
2456 if (auto intTy = dyn_cast<IntegerType>(Val: getElementTypeOrSelf(type: getType())))
2457 resultBitwidth = intTy.getWidth();
2458
2459 if (auto foldResult = constFoldCastOp<IntegerAttr, IntegerAttr>(
2460 operands: adaptor.getOperands(), resType: getType(),
2461 calculate: [resultBitwidth](const APInt &a, bool & /*castStatus*/) {
2462 return a.sextOrTrunc(width: resultBitwidth);
2463 }))
2464 return foldResult;
2465
2466 // index_cast(index_cast(x : A) : B) : A -> x, but only when B is at least
2467 // as wide as A. If B is narrower, the inner cast truncates and the outer
2468 // cast sign-extends, so the round-trip is lossy.
2469 if (auto inner = getOperand().getDefiningOp<arith::IndexCastOp>()) {
2470 Value x = inner.getOperand();
2471 if (x.getType() == getType()) {
2472 if (getIndexCastWidth(t: inner.getType()) >= getIndexCastWidth(t: x.getType()))
2473 return x;
2474 }
2475 }
2476 return {};
2477}
2478
2479void arith::IndexCastOp::getCanonicalizationPatterns(
2480 RewritePatternSet &patterns, MLIRContext *context) {
2481 patterns.add<IndexCastOfExtSI>(arg&: context);
2482}
2483
2484//===----------------------------------------------------------------------===//
2485// IndexCastUIOp
2486//===----------------------------------------------------------------------===//
2487
2488bool arith::IndexCastUIOp::areCastCompatible(TypeRange inputs,
2489 TypeRange outputs) {
2490 return areIndexCastCompatible(inputs, outputs);
2491}
2492
2493OpFoldResult arith::IndexCastUIOp::fold(FoldAdaptor adaptor) {
2494 // index_castui(constant) -> constant
2495 unsigned resultBitwidth = 64; // Default for index integer attributes.
2496 if (auto intTy = dyn_cast<IntegerType>(Val: getElementTypeOrSelf(type: getType())))
2497 resultBitwidth = intTy.getWidth();
2498
2499 if (auto foldResult = constFoldCastOp<IntegerAttr, IntegerAttr>(
2500 operands: adaptor.getOperands(), resType: getType(),
2501 calculate: [resultBitwidth](const APInt &a, bool & /*castStatus*/) {
2502 return a.zextOrTrunc(width: resultBitwidth);
2503 }))
2504 return foldResult;
2505
2506 // index_castui(index_castui(x : A) : B) : A -> x, but only when B is at
2507 // least as wide as A. If B is narrower, the inner cast truncates and the
2508 // outer cast zero-extends, so the round-trip is lossy.
2509 if (auto inner = getOperand().getDefiningOp<arith::IndexCastUIOp>()) {
2510 Value x = inner.getOperand();
2511 if (x.getType() == getType()) {
2512 if (getIndexCastWidth(t: inner.getType()) >= getIndexCastWidth(t: x.getType()))
2513 return x;
2514 }
2515 }
2516 return {};
2517}
2518
2519void arith::IndexCastUIOp::getCanonicalizationPatterns(
2520 RewritePatternSet &patterns, MLIRContext *context) {
2521 patterns.add<IndexCastUIOfExtUI>(arg&: context);
2522}
2523
2524//===----------------------------------------------------------------------===//
2525// BitcastOp
2526//===----------------------------------------------------------------------===//
2527
2528bool arith::BitcastOp::areCastCompatible(TypeRange inputs, TypeRange outputs) {
2529 if (!areValidCastInputsAndOutputs(inputs, outputs))
2530 return false;
2531
2532 auto srcType = getTypeIfLikeOrMemRef<IntegerType, FloatType>(type: inputs.front());
2533 auto dstType = getTypeIfLikeOrMemRef<IntegerType, FloatType>(type: outputs.front());
2534 if (!srcType || !dstType)
2535 return false;
2536
2537 return srcType.getIntOrFloatBitWidth() == dstType.getIntOrFloatBitWidth();
2538}
2539
2540OpFoldResult arith::BitcastOp::fold(FoldAdaptor adaptor) {
2541 auto resType = getType();
2542 auto operand = adaptor.getIn();
2543 if (!operand)
2544 return {};
2545
2546 /// Bitcast dense elements.
2547 if (auto denseAttr = dyn_cast_or_null<DenseElementsAttr>(Val&: operand))
2548 return denseAttr.bitcast(newElType: llvm::cast<ShapedType>(Val&: resType).getElementType());
2549 /// Other shaped types unhandled.
2550 if (llvm::isa<ShapedType>(Val: resType))
2551 return {};
2552
2553 /// Bitcast poison.
2554 if (matchPattern(attr: operand, pattern: ub::m_Poison()))
2555 return ub::PoisonAttr::get(ctx: getContext());
2556
2557 /// Bitcast integer or float to integer or float.
2558 if (!llvm::isa<FloatAttr, IntegerAttr>(Val: operand))
2559 return {};
2560
2561 APInt bits = llvm::isa<FloatAttr>(Val: operand)
2562 ? llvm::cast<FloatAttr>(Val&: operand).getValue().bitcastToAPInt()
2563 : llvm::cast<IntegerAttr>(Val&: operand).getValue();
2564 assert(resType.getIntOrFloatBitWidth() == bits.getBitWidth() &&
2565 "trying to fold on broken IR: operands have incompatible types");
2566
2567 if (auto resFloatType = dyn_cast<FloatType>(Val&: resType))
2568 return FloatAttr::get(type: resType,
2569 value: APFloat(resFloatType.getFloatSemantics(), bits));
2570 return IntegerAttr::get(type: resType, value: bits);
2571}
2572
2573void arith::BitcastOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
2574 MLIRContext *context) {
2575 patterns.add<BitcastOfBitcast>(arg&: context);
2576}
2577
2578//===----------------------------------------------------------------------===//
2579// CmpIOp
2580//===----------------------------------------------------------------------===//
2581
2582/// Compute `lhs` `pred` `rhs`, where `pred` is one of the known integer
2583/// comparison predicates.
2584bool mlir::arith::applyCmpPredicate(arith::CmpIPredicate predicate,
2585 const APInt &lhs, const APInt &rhs) {
2586 switch (predicate) {
2587 case arith::CmpIPredicate::eq:
2588 return lhs.eq(RHS: rhs);
2589 case arith::CmpIPredicate::ne:
2590 return lhs.ne(RHS: rhs);
2591 case arith::CmpIPredicate::slt:
2592 return lhs.slt(RHS: rhs);
2593 case arith::CmpIPredicate::sle:
2594 return lhs.sle(RHS: rhs);
2595 case arith::CmpIPredicate::sgt:
2596 return lhs.sgt(RHS: rhs);
2597 case arith::CmpIPredicate::sge:
2598 return lhs.sge(RHS: rhs);
2599 case arith::CmpIPredicate::ult:
2600 return lhs.ult(RHS: rhs);
2601 case arith::CmpIPredicate::ule:
2602 return lhs.ule(RHS: rhs);
2603 case arith::CmpIPredicate::ugt:
2604 return lhs.ugt(RHS: rhs);
2605 case arith::CmpIPredicate::uge:
2606 return lhs.uge(RHS: rhs);
2607 }
2608 llvm_unreachable("unknown cmpi predicate kind");
2609}
2610
2611/// Returns true if the predicate is true for two equal operands.
2612static bool applyCmpPredicateToEqualOperands(arith::CmpIPredicate predicate) {
2613 switch (predicate) {
2614 case arith::CmpIPredicate::eq:
2615 case arith::CmpIPredicate::sle:
2616 case arith::CmpIPredicate::sge:
2617 case arith::CmpIPredicate::ule:
2618 case arith::CmpIPredicate::uge:
2619 return true;
2620 case arith::CmpIPredicate::ne:
2621 case arith::CmpIPredicate::slt:
2622 case arith::CmpIPredicate::sgt:
2623 case arith::CmpIPredicate::ult:
2624 case arith::CmpIPredicate::ugt:
2625 return false;
2626 }
2627 llvm_unreachable("unknown cmpi predicate kind");
2628}
2629
2630static std::optional<int64_t> getIntegerWidth(Type t) {
2631 if (auto intType = dyn_cast<IntegerType>(Val&: t)) {
2632 return intType.getWidth();
2633 }
2634 if (auto vectorIntType = dyn_cast<VectorType>(Val&: t)) {
2635 return llvm::cast<IntegerType>(Val: vectorIntType.getElementType()).getWidth();
2636 }
2637 return std::nullopt;
2638}
2639
2640OpFoldResult arith::CmpIOp::fold(FoldAdaptor adaptor) {
2641 // cmpi(pred, x, x)
2642 if (getLhs() == getRhs()) {
2643 auto val = applyCmpPredicateToEqualOperands(predicate: getPredicate());
2644 return getBoolAttribute(type: getType(), value: val);
2645 }
2646
2647 if (matchPattern(attr: adaptor.getRhs(), pattern: m_Zero())) {
2648 if (auto extOp = getLhs().getDefiningOp<ExtSIOp>()) {
2649 // extsi(%x : i1 -> iN) != 0 -> %x
2650 std::optional<int64_t> integerWidth =
2651 getIntegerWidth(t: extOp.getOperand().getType());
2652 if (integerWidth && integerWidth.value() == 1 &&
2653 getPredicate() == arith::CmpIPredicate::ne)
2654 return extOp.getOperand();
2655 }
2656 if (auto extOp = getLhs().getDefiningOp<ExtUIOp>()) {
2657 // extui(%x : i1 -> iN) != 0 -> %x
2658 std::optional<int64_t> integerWidth =
2659 getIntegerWidth(t: extOp.getOperand().getType());
2660 if (integerWidth && integerWidth.value() == 1 &&
2661 getPredicate() == arith::CmpIPredicate::ne)
2662 return extOp.getOperand();
2663 }
2664
2665 // arith.cmpi ne, %val, %zero : i1 -> %val
2666 if (getElementTypeOrSelf(type: getLhs().getType()).isInteger(width: 1) &&
2667 getPredicate() == arith::CmpIPredicate::ne)
2668 return getLhs();
2669 }
2670
2671 if (matchPattern(attr: adaptor.getRhs(), pattern: m_One())) {
2672 // arith.cmpi eq, %val, %one : i1 -> %val
2673 if (getElementTypeOrSelf(type: getLhs().getType()).isInteger(width: 1) &&
2674 getPredicate() == arith::CmpIPredicate::eq)
2675 return getLhs();
2676 }
2677
2678 // Move constant to the right side.
2679 if (adaptor.getLhs() && !adaptor.getRhs()) {
2680 // Do not use invertPredicate, as it will change eq to ne and vice versa.
2681 using Pred = CmpIPredicate;
2682 const std::pair<Pred, Pred> invPreds[] = {
2683 {Pred::slt, Pred::sgt}, {Pred::sgt, Pred::slt}, {Pred::sle, Pred::sge},
2684 {Pred::sge, Pred::sle}, {Pred::ult, Pred::ugt}, {Pred::ugt, Pred::ult},
2685 {Pred::ule, Pred::uge}, {Pred::uge, Pred::ule}, {Pred::eq, Pred::eq},
2686 {Pred::ne, Pred::ne},
2687 };
2688 Pred origPred = getPredicate();
2689 for (auto pred : invPreds) {
2690 if (origPred == pred.first) {
2691 setPredicate(pred.second);
2692 Value lhs = getLhs();
2693 Value rhs = getRhs();
2694 getLhsMutable().assign(value: rhs);
2695 getRhsMutable().assign(value: lhs);
2696 return getResult();
2697 }
2698 }
2699 llvm_unreachable("unknown cmpi predicate kind");
2700 }
2701
2702 // We are moving constants to the right side; So if lhs is constant rhs is
2703 // guaranteed to be a constant.
2704 if (auto lhs = dyn_cast_if_present<TypedAttr>(Val: adaptor.getLhs())) {
2705 return constFoldBinaryOp<IntegerAttr>(
2706 operands: adaptor.getOperands(), resultType: getI1SameShape(type: lhs.getType()),
2707 calculate: [pred = getPredicate()](const APInt &lhs, const APInt &rhs) {
2708 return APInt(1,
2709 static_cast<int64_t>(applyCmpPredicate(predicate: pred, lhs, rhs)));
2710 });
2711 }
2712
2713 return {};
2714}
2715
2716void arith::CmpIOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
2717 MLIRContext *context) {
2718 patterns.insert<CmpIExtSI, CmpIExtUI>(arg&: context);
2719}
2720
2721//===----------------------------------------------------------------------===//
2722// CmpFOp
2723//===----------------------------------------------------------------------===//
2724
2725/// Compute `lhs` `pred` `rhs`, where `pred` is one of the known floating point
2726/// comparison predicates.
2727bool mlir::arith::applyCmpPredicate(arith::CmpFPredicate predicate,
2728 const APFloat &lhs, const APFloat &rhs) {
2729 auto cmpResult = lhs.compare(RHS: rhs);
2730 switch (predicate) {
2731 case arith::CmpFPredicate::AlwaysFalse:
2732 return false;
2733 case arith::CmpFPredicate::OEQ:
2734 return cmpResult == APFloat::cmpEqual;
2735 case arith::CmpFPredicate::OGT:
2736 return cmpResult == APFloat::cmpGreaterThan;
2737 case arith::CmpFPredicate::OGE:
2738 return cmpResult == APFloat::cmpGreaterThan ||
2739 cmpResult == APFloat::cmpEqual;
2740 case arith::CmpFPredicate::OLT:
2741 return cmpResult == APFloat::cmpLessThan;
2742 case arith::CmpFPredicate::OLE:
2743 return cmpResult == APFloat::cmpLessThan || cmpResult == APFloat::cmpEqual;
2744 case arith::CmpFPredicate::ONE:
2745 return cmpResult != APFloat::cmpUnordered && cmpResult != APFloat::cmpEqual;
2746 case arith::CmpFPredicate::ORD:
2747 return cmpResult != APFloat::cmpUnordered;
2748 case arith::CmpFPredicate::UEQ:
2749 return cmpResult == APFloat::cmpUnordered || cmpResult == APFloat::cmpEqual;
2750 case arith::CmpFPredicate::UGT:
2751 return cmpResult == APFloat::cmpUnordered ||
2752 cmpResult == APFloat::cmpGreaterThan;
2753 case arith::CmpFPredicate::UGE:
2754 return cmpResult == APFloat::cmpUnordered ||
2755 cmpResult == APFloat::cmpGreaterThan ||
2756 cmpResult == APFloat::cmpEqual;
2757 case arith::CmpFPredicate::ULT:
2758 return cmpResult == APFloat::cmpUnordered ||
2759 cmpResult == APFloat::cmpLessThan;
2760 case arith::CmpFPredicate::ULE:
2761 return cmpResult == APFloat::cmpUnordered ||
2762 cmpResult == APFloat::cmpLessThan || cmpResult == APFloat::cmpEqual;
2763 case arith::CmpFPredicate::UNE:
2764 return cmpResult != APFloat::cmpEqual;
2765 case arith::CmpFPredicate::UNO:
2766 return cmpResult == APFloat::cmpUnordered;
2767 case arith::CmpFPredicate::AlwaysTrue:
2768 return true;
2769 }
2770 llvm_unreachable("unknown cmpf predicate kind");
2771}
2772
2773OpFoldResult arith::CmpFOp::fold(FoldAdaptor adaptor) {
2774 auto lhs = dyn_cast_if_present<FloatAttr>(Val: adaptor.getLhs());
2775 auto rhs = dyn_cast_if_present<FloatAttr>(Val: adaptor.getRhs());
2776
2777 // If one operand is NaN, making them both NaN does not change the result.
2778 if (lhs && lhs.getValue().isNaN())
2779 rhs = lhs;
2780 if (rhs && rhs.getValue().isNaN())
2781 lhs = rhs;
2782
2783 if (!lhs || !rhs)
2784 return {};
2785
2786 auto val = applyCmpPredicate(predicate: getPredicate(), lhs: lhs.getValue(), rhs: rhs.getValue());
2787 return BoolAttr::get(context: getContext(), value: val);
2788}
2789
2790namespace {
2791class CmpFIntToFPConst final : public OpRewritePattern<CmpFOp> {
2792public:
2793 using Base::Base;
2794
2795 static CmpIPredicate convertToIntegerPredicate(CmpFPredicate pred,
2796 bool isUnsigned) {
2797 using namespace arith;
2798 switch (pred) {
2799 case CmpFPredicate::UEQ:
2800 case CmpFPredicate::OEQ:
2801 return CmpIPredicate::eq;
2802 case CmpFPredicate::UGT:
2803 case CmpFPredicate::OGT:
2804 return isUnsigned ? CmpIPredicate::ugt : CmpIPredicate::sgt;
2805 case CmpFPredicate::UGE:
2806 case CmpFPredicate::OGE:
2807 return isUnsigned ? CmpIPredicate::uge : CmpIPredicate::sge;
2808 case CmpFPredicate::ULT:
2809 case CmpFPredicate::OLT:
2810 return isUnsigned ? CmpIPredicate::ult : CmpIPredicate::slt;
2811 case CmpFPredicate::ULE:
2812 case CmpFPredicate::OLE:
2813 return isUnsigned ? CmpIPredicate::ule : CmpIPredicate::sle;
2814 case CmpFPredicate::UNE:
2815 case CmpFPredicate::ONE:
2816 return CmpIPredicate::ne;
2817 default:
2818 llvm_unreachable("Unexpected predicate!");
2819 }
2820 }
2821
2822 LogicalResult matchAndRewrite(CmpFOp op,
2823 PatternRewriter &rewriter) const override {
2824 FloatAttr flt;
2825 if (!matchPattern(value: op.getRhs(), pattern: m_Constant(bind_value: &flt)))
2826 return failure();
2827
2828 const APFloat &rhs = flt.getValue();
2829
2830 // Don't attempt to fold a nan.
2831 if (rhs.isNaN())
2832 return failure();
2833
2834 // Get the width of the mantissa. We don't want to hack on conversions that
2835 // might lose information from the integer, e.g. "i64 -> float"
2836 FloatType floatTy = llvm::cast<FloatType>(Val: op.getRhs().getType());
2837 int mantissaWidth = floatTy.getFPMantissaWidth();
2838 if (mantissaWidth <= 0)
2839 return failure();
2840
2841 bool isUnsigned;
2842 Value intVal;
2843
2844 if (auto si = op.getLhs().getDefiningOp<SIToFPOp>()) {
2845 isUnsigned = false;
2846 intVal = si.getIn();
2847 } else if (auto ui = op.getLhs().getDefiningOp<UIToFPOp>()) {
2848 isUnsigned = true;
2849 intVal = ui.getIn();
2850 } else {
2851 return failure();
2852 }
2853
2854 // Check to see that the input is converted from an integer type that is
2855 // small enough that preserves all bits.
2856 auto intTy = llvm::cast<IntegerType>(Val: intVal.getType());
2857 auto intWidth = intTy.getWidth();
2858
2859 // Number of bits representing values, as opposed to the sign
2860 auto valueBits = isUnsigned ? intWidth : (intWidth - 1);
2861
2862 const llvm::fltSemantics &semantics = floatTy.getFloatSemantics();
2863
2864 // Conservatively require an exact representation of integer zero. For
2865 // example, f8E8M0FNU converts zero to its smallest positive value.
2866 if (!APFloat::semanticsHasZero(semantics))
2867 return failure();
2868
2869 // Signed inputs may be negative, which e.g. f8E5M3FNU cannot represent.
2870 // Do not try to convert the signed minimum below for such formats.
2871 if (!isUnsigned && !APFloat::semanticsHasSignedRepr(semantics))
2872 return failure();
2873
2874 if (!APFloat::semanticsHasInf(semantics)) {
2875 // Overflow can produce NaN or saturate to a finite value:
2876 // - sitofp(1000 : i32) to f8E4M3FN is NaN, so cmpf ogt against 0.0
2877 // is false, whereas cmpi sgt against integer zero would be true.
2878 // - uitofp(8 : i4) to f6E2M3FN saturates to 7.5. Equality against 7.5
2879 // is true, but the fractional-constant logic below would fold it to
2880 // false. Its four bits of precision do not prevent overflow.
2881 //
2882 // Conservatively require the enclosing 2^valueBits (and its negative
2883 // for signed inputs) to be representable, so rounding stays within
2884 // finite bounds. Equality at the exponent bound is safe.
2885 if (APFloat::semanticsMaxExponent(semantics) <
2886 static_cast<int>(valueBits))
2887 return failure();
2888 }
2889
2890 // Following test does NOT adjust intWidth downwards for signed inputs,
2891 // because the most negative value still requires all the mantissa bits
2892 // to distinguish it from one less than that value.
2893 if ((int)intWidth > mantissaWidth) {
2894 // Conversion would lose accuracy. Check if loss can impact comparison.
2895 int exponent = ilogb(Arg: rhs);
2896 if (exponent == APFloat::IEK_Inf) {
2897 int maxExponent = ilogb(Arg: APFloat::getLargest(Sem: rhs.getSemantics()));
2898 if (maxExponent < static_cast<int>(valueBits)) {
2899 // Conversion could create infinity. For example, sitofp(65536 : i32)
2900 // to f16 is +inf, so equality against +inf can be true.
2901 return failure();
2902 }
2903 } else {
2904 // Note that if rhs is zero or NaN, then Exp is negative
2905 // and first condition is trivially false.
2906 if (mantissaWidth <= exponent && exponent <= (int)valueBits) {
2907 // Conversion could affect comparison.
2908 return failure();
2909 }
2910 }
2911 }
2912
2913 // Convert to equivalent cmpi predicate
2914 CmpIPredicate pred;
2915 switch (op.getPredicate()) {
2916 case CmpFPredicate::AlwaysTrue:
2917 case CmpFPredicate::ORD:
2918 // Int to fp conversion doesn't create a nan (ord checks neither is a nan)
2919 rewriter.replaceOpWithNewOp<ConstantIntOp>(op, /*value=*/args: true,
2920 /*width=*/args: 1);
2921 return success();
2922 case CmpFPredicate::AlwaysFalse:
2923 case CmpFPredicate::UNO:
2924 // Int to fp conversion doesn't create a nan (uno checks either is a nan)
2925 rewriter.replaceOpWithNewOp<ConstantIntOp>(op, /*value=*/args: false,
2926 /*width=*/args: 1);
2927 return success();
2928 default:
2929 pred = convertToIntegerPredicate(pred: op.getPredicate(), isUnsigned);
2930 break;
2931 }
2932
2933 if (!isUnsigned) {
2934 // If the rhs value is > SignedMax, fold the comparison. This handles
2935 // +INF and large values.
2936 APFloat signedMax(rhs.getSemantics());
2937 signedMax.convertFromAPInt(Input: APInt::getSignedMaxValue(numBits: intWidth), IsSigned: true,
2938 RM: APFloat::rmNearestTiesToEven);
2939 if (signedMax < rhs) { // smax < 13123.0
2940 if (pred == CmpIPredicate::ne || pred == CmpIPredicate::slt ||
2941 pred == CmpIPredicate::sle)
2942 rewriter.replaceOpWithNewOp<ConstantIntOp>(op, /*value=*/args: true,
2943 /*width=*/args: 1);
2944 else
2945 rewriter.replaceOpWithNewOp<ConstantIntOp>(op, /*value=*/args: false,
2946 /*width=*/args: 1);
2947 return success();
2948 }
2949 } else {
2950 // If the rhs value is > UnsignedMax, fold the comparison. This handles
2951 // +INF and large values.
2952 APFloat unsignedMax(rhs.getSemantics());
2953 unsignedMax.convertFromAPInt(Input: APInt::getMaxValue(numBits: intWidth), IsSigned: false,
2954 RM: APFloat::rmNearestTiesToEven);
2955 if (unsignedMax < rhs) { // umax < 13123.0
2956 if (pred == CmpIPredicate::ne || pred == CmpIPredicate::ult ||
2957 pred == CmpIPredicate::ule)
2958 rewriter.replaceOpWithNewOp<ConstantIntOp>(op, /*value=*/args: true,
2959 /*width=*/args: 1);
2960 else
2961 rewriter.replaceOpWithNewOp<ConstantIntOp>(op, /*value=*/args: false,
2962 /*width=*/args: 1);
2963 return success();
2964 }
2965 }
2966
2967 if (!isUnsigned) {
2968 // See if the rhs value is < SignedMin.
2969 APFloat signedMin(rhs.getSemantics());
2970 signedMin.convertFromAPInt(Input: APInt::getSignedMinValue(numBits: intWidth), IsSigned: true,
2971 RM: APFloat::rmNearestTiesToEven);
2972 if (signedMin > rhs) { // smin > 12312.0
2973 if (pred == CmpIPredicate::ne || pred == CmpIPredicate::sgt ||
2974 pred == CmpIPredicate::sge)
2975 rewriter.replaceOpWithNewOp<ConstantIntOp>(op, /*value=*/args: true,
2976 /*width=*/args: 1);
2977 else
2978 rewriter.replaceOpWithNewOp<ConstantIntOp>(op, /*value=*/args: false,
2979 /*width=*/args: 1);
2980 return success();
2981 }
2982 } else {
2983 // See if the rhs value is < UnsignedMin.
2984 APFloat unsignedMin(rhs.getSemantics());
2985 unsignedMin.convertFromAPInt(Input: APInt::getMinValue(numBits: intWidth), IsSigned: false,
2986 RM: APFloat::rmNearestTiesToEven);
2987 if (unsignedMin > rhs) { // umin > 12312.0
2988 if (pred == CmpIPredicate::ne || pred == CmpIPredicate::ugt ||
2989 pred == CmpIPredicate::uge)
2990 rewriter.replaceOpWithNewOp<ConstantIntOp>(op, /*value=*/args: true,
2991 /*width=*/args: 1);
2992 else
2993 rewriter.replaceOpWithNewOp<ConstantIntOp>(op, /*value=*/args: false,
2994 /*width=*/args: 1);
2995 return success();
2996 }
2997 }
2998
2999 // Okay, now we know that the FP constant fits in the range [SMIN, SMAX] or
3000 // [0, UMAX], but it may still be fractional. See if it is fractional by
3001 // casting the FP value to the integer value and back, checking for
3002 // equality. Don't do this for zero, because -0.0 is not fractional.
3003 bool ignored;
3004 APSInt rhsInt(intWidth, isUnsigned);
3005 if (APFloat::opInvalidOp ==
3006 rhs.convertToInteger(Result&: rhsInt, RM: APFloat::rmTowardZero, IsExact: &ignored)) {
3007 // Undefined behavior invoked - the destination type can't represent
3008 // the input constant.
3009 return failure();
3010 }
3011
3012 if (!rhs.isZero()) {
3013 APFloat apf(floatTy.getFloatSemantics(),
3014 APInt::getZero(numBits: floatTy.getWidth()));
3015 apf.convertFromAPInt(Input: rhsInt, IsSigned: !isUnsigned, RM: APFloat::rmNearestTiesToEven);
3016
3017 bool equal = apf == rhs;
3018 if (!equal) {
3019 // If we had a comparison against a fractional value, we have to adjust
3020 // the compare predicate and sometimes the value. rhsInt is rounded
3021 // towards zero at this point.
3022 switch (pred) {
3023 case CmpIPredicate::ne: // (float)int != 4.4 --> true
3024 rewriter.replaceOpWithNewOp<ConstantIntOp>(op, /*value=*/args: true,
3025 /*width=*/args: 1);
3026 return success();
3027 case CmpIPredicate::eq: // (float)int == 4.4 --> false
3028 rewriter.replaceOpWithNewOp<ConstantIntOp>(op, /*value=*/args: false,
3029 /*width=*/args: 1);
3030 return success();
3031 case CmpIPredicate::ule:
3032 // (float)int <= 4.4 --> int <= 4
3033 // (float)int <= -4.4 --> false
3034 if (rhs.isNegative()) {
3035 rewriter.replaceOpWithNewOp<ConstantIntOp>(op, /*value=*/args: false,
3036 /*width=*/args: 1);
3037 return success();
3038 }
3039 break;
3040 case CmpIPredicate::sle:
3041 // (float)int <= 4.4 --> int <= 4
3042 // (float)int <= -4.4 --> int < -4
3043 if (rhs.isNegative())
3044 pred = CmpIPredicate::slt;
3045 break;
3046 case CmpIPredicate::ult:
3047 // (float)int < -4.4 --> false
3048 // (float)int < 4.4 --> int <= 4
3049 if (rhs.isNegative()) {
3050 rewriter.replaceOpWithNewOp<ConstantIntOp>(op, /*value=*/args: false,
3051 /*width=*/args: 1);
3052 return success();
3053 }
3054 pred = CmpIPredicate::ule;
3055 break;
3056 case CmpIPredicate::slt:
3057 // (float)int < -4.4 --> int < -4
3058 // (float)int < 4.4 --> int <= 4
3059 if (!rhs.isNegative())
3060 pred = CmpIPredicate::sle;
3061 break;
3062 case CmpIPredicate::ugt:
3063 // (float)int > 4.4 --> int > 4
3064 // (float)int > -4.4 --> true
3065 if (rhs.isNegative()) {
3066 rewriter.replaceOpWithNewOp<ConstantIntOp>(op, /*value=*/args: true,
3067 /*width=*/args: 1);
3068 return success();
3069 }
3070 break;
3071 case CmpIPredicate::sgt:
3072 // (float)int > 4.4 --> int > 4
3073 // (float)int > -4.4 --> int >= -4
3074 if (rhs.isNegative())
3075 pred = CmpIPredicate::sge;
3076 break;
3077 case CmpIPredicate::uge:
3078 // (float)int >= -4.4 --> true
3079 // (float)int >= 4.4 --> int > 4
3080 if (rhs.isNegative()) {
3081 rewriter.replaceOpWithNewOp<ConstantIntOp>(op, /*value=*/args: true,
3082 /*width=*/args: 1);
3083 return success();
3084 }
3085 pred = CmpIPredicate::ugt;
3086 break;
3087 case CmpIPredicate::sge:
3088 // (float)int >= -4.4 --> int >= -4
3089 // (float)int >= 4.4 --> int > 4
3090 if (!rhs.isNegative())
3091 pred = CmpIPredicate::sgt;
3092 break;
3093 }
3094 }
3095 }
3096
3097 // Lower this FP comparison into an appropriate integer version of the
3098 // comparison.
3099 rewriter.replaceOpWithNewOp<CmpIOp>(
3100 op, args&: pred, args&: intVal,
3101 args: ConstantOp::create(builder&: rewriter, location: op.getLoc(), result: intVal.getType(),
3102 value: rewriter.getIntegerAttr(type: intVal.getType(), value: rhsInt)));
3103 return success();
3104 }
3105};
3106} // namespace
3107
3108void arith::CmpFOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
3109 MLIRContext *context) {
3110 patterns.insert<CmpFIntToFPConst>(arg&: context);
3111}
3112
3113//===----------------------------------------------------------------------===//
3114// SelectOp
3115//===----------------------------------------------------------------------===//
3116
3117namespace {
3118// select %arg, %c1, %c0 => extui %arg
3119struct SelectToExtUI : public OpRewritePattern<arith::SelectOp> {
3120 using Base::Base;
3121
3122 LogicalResult matchAndRewrite(arith::SelectOp op,
3123 PatternRewriter &rewriter) const override {
3124 // Cannot extui i1 to i1, or i1 to f32
3125 if (!llvm::isa<IntegerType>(Val: op.getType()) || op.getType().isInteger(width: 1))
3126 return failure();
3127
3128 // select %x, c1, %c0 => extui %arg
3129 if (matchPattern(value: op.getTrueValue(), pattern: m_One()) &&
3130 matchPattern(value: op.getFalseValue(), pattern: m_Zero())) {
3131 rewriter.replaceOpWithNewOp<arith::ExtUIOp>(op, args: op.getType(),
3132 args: op.getCondition());
3133 return success();
3134 }
3135
3136 // select %x, c0, %c1 => extui (xor %arg, true)
3137 if (matchPattern(value: op.getTrueValue(), pattern: m_Zero()) &&
3138 matchPattern(value: op.getFalseValue(), pattern: m_One())) {
3139 rewriter.replaceOpWithNewOp<arith::ExtUIOp>(
3140 op, args: op.getType(),
3141 args: arith::XOrIOp::create(
3142 builder&: rewriter, location: op.getLoc(), lhs: op.getCondition(),
3143 rhs: arith::ConstantIntOp::create(builder&: rewriter, location: op.getLoc(),
3144 type: op.getCondition().getType(), value: 1)));
3145 return success();
3146 }
3147
3148 return failure();
3149 }
3150};
3151} // namespace
3152
3153/// select(M, F(a0, a1, ..., a_n), F(b0, b1, ..., b_n))
3154/// -> F(select(M, a0, b0), select(M, a1, b1), ...) for pure and
3155/// elementwise F.
3156///
3157/// Correctness provement:
3158///
3159/// Let's discuss the cases ignoring poison first.
3160///
3161/// As F is elementwise, we have
3162///
3163/// F(a1, a2, ..., a_n)[p] = F(a1[p], a2[p], ..., a_n[p])
3164///
3165/// then for
3166///
3167/// L = select(M, F(a0, a1, ..., a_n), F(b0, b1, ..., b_n))
3168/// R = F(select(M, a0, b0), select(M, a1, b1), ....)
3169///
3170/// for any position p, we have:
3171///
3172/// if M[p] == true, then L[p] == F(a)[p] == F(a[p]) == F(select(true, a[p], b[p])) == R[p]
3173/// if M[p] == false, then L[p] == F(b)[p] == F(b[p]) == F(select(false, a[p], b[p])) == R[p]
3174///
3175/// And F is pure, so that we can move and reduce F.
3176/// So select(M, F(a0, a1, ..., a_n), F(b0, b1, ..., b_n)) is equal to
3177/// F(select(M, a0, b0), select(M, a1, b1), ...) ignoring poison.
3178///
3179/// Let's consider the poison value now. ArithBase.td says:
3180///
3181/// Unless otherwise stated, the operations within this dialect
3182/// propagate poison values, i.e., if any of its inputs are poison, then the
3183/// output is poison.
3184///
3185/// So most F, F(poison) = poison
3186///
3187/// Then for any position p, if a[p] is poison
3188///
3189/// if M[p] == true, L[p] == f(a[p]) == poison == F(select(true, a[p], b[p])) == R[p]
3190/// if M[p] == false, L[p] == f(b[p]) == F(select(false, a[p], b[p])) == R[p]
3191/// if M[p] == P, L[p] == poison == F(select(P, a, b)) == R[p]
3192///
3193/// Then the transformation doesn't change poison propagation.
3194///
3195/// For F which doesn't meet the assumption, this transformation may change the
3196/// poison propagation, but the language allows us to replace poison with a undef
3197/// or any value with the same type. So this is still valid transformation.
3198struct SelectCommonElementwiseOp : public OpRewritePattern<arith::SelectOp> {
3199 using Base::Base;
3200
3201 LogicalResult matchAndRewrite(arith::SelectOp op,
3202 PatternRewriter &rewriter) const override {
3203 Operation *lhs = op.getTrueValue().getDefiningOp();
3204 Operation *rhs = op.getFalseValue().getDefiningOp();
3205 if (!lhs || !rhs || lhs == rhs)
3206 return failure();
3207
3208 // Do not hoist a select through selects: rewriting
3209 // select(c, select(d, a, b), select(d, x, y)) into
3210 // select(d, select(c, a, x), select(c, b, y)) matches again with c and d
3211 // swapped, so the greedy rewriter never converges.
3212 if (isa<arith::SelectOp>(Val: lhs))
3213 return failure();
3214
3215 for (Operation *producer : {lhs, rhs}) {
3216 // Avoid sinking loop invariant code into the loop.
3217 //
3218 // TODO: handle the cases by introducing more CFG/Loop analysis.
3219 if (producer->getBlock() != op->getBlock())
3220 return failure();
3221
3222 // `isEquivalentTo` doesn't catch all cases, so we conservatively match
3223 // the cases we know how to convert
3224 if (producer->getNumResults() != 1 || !producer->hasOneUse() ||
3225 producer->getNumRegions() != 0 || producer->getNumSuccessors() != 0 ||
3226 !producer->hasTrait<OpTrait::Elementwise>() || !isPure(op: producer))
3227 return failure();
3228 }
3229
3230 // Check if the lhs and the rhs are the same kind of op, have the
3231 // same structure of inputs, types, and attributes.
3232 if (!OperationEquivalence::isEquivalentTo(
3233 lhs, rhs, checkEquivalent: OperationEquivalence::ignoreValueEquivalence, markEquivalent: nullptr,
3234 flags: OperationEquivalence::IgnoreLocations))
3235 return failure();
3236
3237 DenseMap<std::pair<Value, Value>, Value> selected;
3238 SmallVector<Operation *> createdSelects;
3239 SmallVector<Value> operands;
3240 Type conditionType = op.getCondition().getType();
3241 for (auto [a, b] : llvm::zip(t: lhs->getOperands(), u: rhs->getOperands())) {
3242 // For select(c, F(a, k), F(b, k)), we should transform it into
3243 // F(select(c, a, b), k) instead of F(select(c, a, b), select(c, k, k))
3244 if (a == b) {
3245 operands.push_back(Elt: a);
3246 continue;
3247 }
3248 if (!conditionType.isSignlessInteger(width: 1) &&
3249 (!isa<VectorType, TensorType>(Val: a.getType()) ||
3250 cast<ShapedType>(Val: a.getType()).clone(elementType: rewriter.getI1Type()) !=
3251 conditionType)) {
3252
3253 for (Operation *created : llvm::reverse(C&: createdSelects))
3254 rewriter.eraseOp(op: created);
3255 return failure();
3256 }
3257
3258 // Always creating a new pair would not merging equivalent results.
3259 auto [it, inserted] = selected.try_emplace(Key: std::make_pair(x&: a, y&: b));
3260 if (inserted) {
3261 auto select = arith::SelectOp::create(builder&: rewriter, location: op.getLoc(),
3262 condition: op.getCondition(), true_value: a, false_value: b);
3263 it->second = select;
3264 createdSelects.push_back(Elt: select);
3265 }
3266 operands.push_back(Elt: it->second);
3267 }
3268
3269 Operation *common = rewriter.clone(op&: *lhs);
3270 common->setOperands(operands);
3271 rewriter.replaceOp(op, newValues: common->getResults());
3272 return success();
3273 }
3274};
3275
3276void arith::SelectOp::getCanonicalizationPatterns(RewritePatternSet &results,
3277 MLIRContext *context) {
3278 results.add<RedundantSelectFalse, RedundantSelectTrue, SelectNotCond,
3279 SelectI1ToNot, SelectCmpISgeToMaxSI, SelectCmpISgeToMinSI,
3280 SelectCmpISgtToMaxSI, SelectCmpISgtToMinSI, SelectCmpISleToMaxSI,
3281 SelectCmpISleToMinSI, SelectCmpISltToMaxSI, SelectCmpISltToMinSI,
3282 SelectCmpIUgeToMaxUI, SelectCmpIUgeToMinUI, SelectCmpIUgtToMaxUI,
3283 SelectCmpIUgtToMinUI, SelectCmpIUleToMaxUI, SelectCmpIUleToMinUI,
3284 SelectCmpIUltToMaxUI, SelectCmpIUltToMinUI, SelectToExtUI>(
3285 arg&: context);
3286 results.add<SelectCommonElementwiseOp>(arg&: context);
3287}
3288
3289OpFoldResult arith::SelectOp::fold(FoldAdaptor adaptor) {
3290 Value trueVal = getTrueValue();
3291 Value falseVal = getFalseValue();
3292 if (trueVal == falseVal)
3293 return trueVal;
3294
3295 Value condition = getCondition();
3296
3297 // select true, %0, %1 => %0
3298 if (matchPattern(attr: adaptor.getCondition(), pattern: m_One()))
3299 return trueVal;
3300
3301 // select false, %0, %1 => %1
3302 if (matchPattern(attr: adaptor.getCondition(), pattern: m_Zero()))
3303 return falseVal;
3304
3305 // If either operand is fully poisoned, return the other.
3306 if (matchPattern(attr: adaptor.getTrueValue(), pattern: ub::m_Poison()))
3307 return falseVal;
3308
3309 if (matchPattern(attr: adaptor.getFalseValue(), pattern: ub::m_Poison()))
3310 return trueVal;
3311
3312 // select %x, true, false => %x
3313 if (getType().isSignlessInteger(width: 1) &&
3314 matchPattern(attr: adaptor.getTrueValue(), pattern: m_One()) &&
3315 matchPattern(attr: adaptor.getFalseValue(), pattern: m_Zero()))
3316 return condition;
3317
3318 if (auto cmp = condition.getDefiningOp<arith::CmpIOp>()) {
3319 auto pred = cmp.getPredicate();
3320 if (pred == arith::CmpIPredicate::eq || pred == arith::CmpIPredicate::ne) {
3321 auto cmpLhs = cmp.getLhs();
3322 auto cmpRhs = cmp.getRhs();
3323
3324 // %0 = arith.cmpi eq, %arg0, %arg1
3325 // %1 = arith.select %0, %arg0, %arg1 => %arg1
3326
3327 // %0 = arith.cmpi ne, %arg0, %arg1
3328 // %1 = arith.select %0, %arg0, %arg1 => %arg0
3329
3330 if ((cmpLhs == trueVal && cmpRhs == falseVal) ||
3331 (cmpRhs == trueVal && cmpLhs == falseVal))
3332 return pred == arith::CmpIPredicate::ne ? trueVal : falseVal;
3333 }
3334 }
3335
3336 // Constant-fold constant operands over non-splat constant condition.
3337 // select %cst_vec, %cst0, %cst1 => %cst2
3338 if (auto cond =
3339 dyn_cast_if_present<DenseElementsAttr>(Val: adaptor.getCondition())) {
3340 // DenseElementsAttr by construction always has a static shape.
3341 assert(cond.getType().hasStaticShape() &&
3342 "DenseElementsAttr must have static shape");
3343 if (auto lhs =
3344 dyn_cast_if_present<DenseElementsAttr>(Val: adaptor.getTrueValue())) {
3345 if (auto rhs =
3346 dyn_cast_if_present<DenseElementsAttr>(Val: adaptor.getFalseValue())) {
3347 SmallVector<Attribute> results;
3348 results.reserve(N: static_cast<size_t>(cond.getNumElements()));
3349 auto condVals = llvm::make_range(x: cond.value_begin<BoolAttr>(),
3350 y: cond.value_end<BoolAttr>());
3351 auto lhsVals = llvm::make_range(x: lhs.value_begin<Attribute>(),
3352 y: lhs.value_end<Attribute>());
3353 auto rhsVals = llvm::make_range(x: rhs.value_begin<Attribute>(),
3354 y: rhs.value_end<Attribute>());
3355
3356 for (auto [condVal, lhsVal, rhsVal] :
3357 llvm::zip_equal(t&: condVals, u&: lhsVals, args&: rhsVals))
3358 results.push_back(Elt: condVal.getValue() ? lhsVal : rhsVal);
3359
3360 return DenseElementsAttr::get(type: lhs.getType(), values: results);
3361 }
3362 }
3363 }
3364
3365 return nullptr;
3366}
3367
3368ParseResult SelectOp::parse(OpAsmParser &parser, OperationState &result) {
3369 Type conditionType, resultType;
3370 SmallVector<OpAsmParser::UnresolvedOperand, 3> operands;
3371 if (parser.parseOperandList(result&: operands, /*requiredOperandCount=*/3) ||
3372 parser.parseOptionalAttrDict(result&: result.attributes) ||
3373 parser.parseColonType(result&: resultType))
3374 return failure();
3375
3376 // Check for the explicit condition type if this is a masked tensor or vector.
3377 if (succeeded(Result: parser.parseOptionalComma())) {
3378 conditionType = resultType;
3379 if (parser.parseType(result&: resultType))
3380 return failure();
3381 } else {
3382 conditionType = parser.getBuilder().getI1Type();
3383 }
3384
3385 result.addTypes(newTypes: resultType);
3386 return parser.resolveOperands(operands,
3387 types: {conditionType, resultType, resultType},
3388 loc: parser.getNameLoc(), result&: result.operands);
3389}
3390
3391void arith::SelectOp::print(OpAsmPrinter &p) {
3392 p << " " << getOperands();
3393 p.printOptionalAttrDict(attrs: (*this)->getDiscardableAttrDictionary().getValue());
3394 p << " : ";
3395 if (ShapedType condType = dyn_cast<ShapedType>(Val: getCondition().getType()))
3396 p << condType << ", ";
3397 p << getType();
3398}
3399
3400LogicalResult arith::SelectOp::verify() {
3401 Type conditionType = getCondition().getType();
3402 if (conditionType.isSignlessInteger(width: 1))
3403 return success();
3404
3405 // If the result type is a vector or tensor, the type can be a mask with the
3406 // same elements.
3407 Type resultType = getType();
3408 if (!llvm::isa<TensorType, VectorType>(Val: resultType))
3409 return emitOpError() << "expected condition to be a signless i1, but got "
3410 << conditionType;
3411 Type shapedConditionType = getI1SameShape(type: resultType);
3412 if (conditionType != shapedConditionType) {
3413 return emitOpError() << "expected condition type to have the same shape "
3414 "as the result type, expected "
3415 << shapedConditionType << ", but got "
3416 << conditionType;
3417 }
3418 return success();
3419}
3420//===----------------------------------------------------------------------===//
3421// ShLIOp
3422//===----------------------------------------------------------------------===//
3423
3424OpFoldResult arith::ShLIOp::fold(FoldAdaptor adaptor) {
3425 // TODO: shli(x, c) -> poison when c is out of range (c >= bit width). An
3426 // out-of-range shift amount is undefined behaviour and could fold to poison,
3427 // but that would make the arith dialect depend on the ub dialect to
3428 // materialize ub.poison; left out for now.
3429
3430 // shli(x, 0) -> x
3431 if (matchPattern(attr: adaptor.getRhs(), pattern: m_Zero()))
3432 return getLhs();
3433 // shli(0, x) -> 0. An out-of-range shift amount yields poison, so refining
3434 // it to 0 is valid.
3435 if (matchPattern(attr: adaptor.getLhs(), pattern: m_Zero()))
3436 return getLhs();
3437 // Don't fold if shifting more or equal than the bit width.
3438 bool bounded = false;
3439 auto result = constFoldBinaryOp<IntegerAttr>(
3440 operands: adaptor.getOperands(), calculate: [&](const APInt &a, const APInt &b) {
3441 bounded = b.ult(RHS: b.getBitWidth());
3442 return a.shl(ShiftAmt: b);
3443 });
3444 return bounded ? result : Attribute();
3445}
3446
3447//===----------------------------------------------------------------------===//
3448// ShRUIOp
3449//===----------------------------------------------------------------------===//
3450
3451OpFoldResult arith::ShRUIOp::fold(FoldAdaptor adaptor) {
3452 // TODO: shrui(x, c) -> poison when c is out of range (c >= bit width). An
3453 // out-of-range shift amount is undefined behaviour and could fold to poison,
3454 // but that would make the arith dialect depend on the ub dialect to
3455 // materialize ub.poison; left out for now.
3456
3457 // shrui(x, 0) -> x
3458 if (matchPattern(attr: adaptor.getRhs(), pattern: m_Zero()))
3459 return getLhs();
3460 // shrui(0, x) -> 0. An out-of-range shift amount yields poison, so refining
3461 // it to 0 is valid.
3462 if (matchPattern(attr: adaptor.getLhs(), pattern: m_Zero()))
3463 return getLhs();
3464 // shrui(x, x) -> 0. For any in-range shift amount v < bitwidth, v >> v == 0
3465 // (v < 2^v); out-of-range amounts yield poison, so 0 is a valid refinement.
3466 if (getLhs() == getRhs())
3467 return getIntegerAttrOfType(type: getType(), value: 0);
3468 // Don't fold if shifting more or equal than the bit width.
3469 bool bounded = false;
3470 auto result = constFoldBinaryOp<IntegerAttr>(
3471 operands: adaptor.getOperands(), calculate: [&](const APInt &a, const APInt &b) {
3472 bounded = b.ult(RHS: b.getBitWidth());
3473 return a.lshr(ShiftAmt: b);
3474 });
3475 return bounded ? result : Attribute();
3476}
3477
3478//===----------------------------------------------------------------------===//
3479// ShRSIOp
3480//===----------------------------------------------------------------------===//
3481
3482OpFoldResult arith::ShRSIOp::fold(FoldAdaptor adaptor) {
3483 // TODO: shrsi(x, c) -> poison when c is out of range (c >= bit width). An
3484 // out-of-range shift amount is undefined behaviour and could fold to poison,
3485 // but that would make the arith dialect depend on the ub dialect to
3486 // materialize ub.poison; left out for now.
3487
3488 // shrsi(x, 0) -> x
3489 if (matchPattern(attr: adaptor.getRhs(), pattern: m_Zero()))
3490 return getLhs();
3491 // shrsi(0, x) -> 0. An out-of-range shift amount yields poison, so refining
3492 // it to 0 is valid.
3493 if (matchPattern(attr: adaptor.getLhs(), pattern: m_Zero()))
3494 return getLhs();
3495 // shrsi(x, x) -> 0. For any in-range shift amount v < bitwidth, v is a small
3496 // non-negative value and v >> v == 0; out-of-range amounts yield poison.
3497 if (getLhs() == getRhs())
3498 return getIntegerAttrOfType(type: getType(), value: 0);
3499 // shrsi(-1, x) -> -1. Arithmetic shift of all-ones is all-ones for any
3500 // in-range amount; out-of-range amounts yield poison.
3501 if (APInt val;
3502 matchPattern(attr: adaptor.getLhs(), pattern: m_ConstantInt(bind_value: &val)) && val.isAllOnes())
3503 return getLhs();
3504 // Don't fold if shifting more or equal than the bit width.
3505 bool bounded = false;
3506 auto result = constFoldBinaryOp<IntegerAttr>(
3507 operands: adaptor.getOperands(), calculate: [&](const APInt &a, const APInt &b) {
3508 bounded = b.ult(RHS: b.getBitWidth());
3509 return a.ashr(ShiftAmt: b);
3510 });
3511 return bounded ? result : Attribute();
3512}
3513
3514//===----------------------------------------------------------------------===//
3515// Atomic Enum
3516//===----------------------------------------------------------------------===//
3517
3518/// Returns the identity value attribute associated with an AtomicRMWKind op.
3519TypedAttr mlir::arith::getIdentityValueAttr(AtomicRMWKind kind, Type resultType,
3520 OpBuilder &builder, Location loc,
3521 bool useOnlyFiniteValue) {
3522 switch (kind) {
3523 case AtomicRMWKind::maximumf: {
3524 const llvm::fltSemantics &semantic =
3525 llvm::cast<FloatType>(Val&: resultType).getFloatSemantics();
3526 APFloat identity = useOnlyFiniteValue
3527 ? APFloat::getLargest(Sem: semantic, /*Negative=*/true)
3528 : APFloat::getInf(Sem: semantic, /*Negative=*/true);
3529 return builder.getFloatAttr(type: resultType, value: identity);
3530 }
3531 case AtomicRMWKind::maxnumf: {
3532 const llvm::fltSemantics &semantic =
3533 llvm::cast<FloatType>(Val&: resultType).getFloatSemantics();
3534 APFloat identity = useOnlyFiniteValue
3535 ? APFloat::getLargest(Sem: semantic, /*Negative=*/true)
3536 : APFloat::getNaN(Sem: semantic, /*Negative=*/true);
3537 return builder.getFloatAttr(type: resultType, value: identity);
3538 }
3539 case AtomicRMWKind::addf:
3540 case AtomicRMWKind::addi:
3541 case AtomicRMWKind::maxu:
3542 case AtomicRMWKind::ori:
3543 case AtomicRMWKind::xori:
3544 return builder.getZeroAttr(type: resultType);
3545 case AtomicRMWKind::andi:
3546 return builder.getIntegerAttr(
3547 type: resultType,
3548 value: APInt::getAllOnes(numBits: llvm::cast<IntegerType>(Val&: resultType).getWidth()));
3549 case AtomicRMWKind::maxs:
3550 return builder.getIntegerAttr(
3551 type: resultType, value: APInt::getSignedMinValue(
3552 numBits: llvm::cast<IntegerType>(Val&: resultType).getWidth()));
3553 case AtomicRMWKind::minimumf: {
3554 const llvm::fltSemantics &semantic =
3555 llvm::cast<FloatType>(Val&: resultType).getFloatSemantics();
3556 APFloat identity = useOnlyFiniteValue
3557 ? APFloat::getLargest(Sem: semantic, /*Negative=*/false)
3558 : APFloat::getInf(Sem: semantic, /*Negative=*/false);
3559
3560 return builder.getFloatAttr(type: resultType, value: identity);
3561 }
3562 case AtomicRMWKind::minnumf: {
3563 const llvm::fltSemantics &semantic =
3564 llvm::cast<FloatType>(Val&: resultType).getFloatSemantics();
3565 APFloat identity = useOnlyFiniteValue
3566 ? APFloat::getLargest(Sem: semantic, /*Negative=*/false)
3567 : APFloat::getNaN(Sem: semantic, /*Negative=*/false);
3568 return builder.getFloatAttr(type: resultType, value: identity);
3569 }
3570 case AtomicRMWKind::mins:
3571 return builder.getIntegerAttr(
3572 type: resultType, value: APInt::getSignedMaxValue(
3573 numBits: llvm::cast<IntegerType>(Val&: resultType).getWidth()));
3574 case AtomicRMWKind::minu:
3575 return builder.getIntegerAttr(
3576 type: resultType,
3577 value: APInt::getMaxValue(numBits: llvm::cast<IntegerType>(Val&: resultType).getWidth()));
3578 case AtomicRMWKind::muli:
3579 return builder.getIntegerAttr(type: resultType, value: 1);
3580 case AtomicRMWKind::mulf:
3581 return builder.getFloatAttr(type: resultType, value: 1);
3582 // `assign` is not a reduction and has no identity element.
3583 case AtomicRMWKind::assign:
3584 break;
3585 }
3586 (void)emitOptionalError(loc, args: "Reduction operation type not supported");
3587 return nullptr;
3588}
3589
3590/// Returns the identity numeric value of the given op.
3591std::optional<TypedAttr> mlir::arith::getNeutralElement(Operation *op) {
3592 std::optional<AtomicRMWKind> maybeKind =
3593 llvm::TypeSwitch<Operation *, std::optional<AtomicRMWKind>>(op)
3594 // Floating-point operations.
3595 .Case(caseFn: [](arith::AddFOp op) { return AtomicRMWKind::addf; })
3596 .Case(caseFn: [](arith::MulFOp op) { return AtomicRMWKind::mulf; })
3597 .Case(caseFn: [](arith::MaximumFOp op) { return AtomicRMWKind::maximumf; })
3598 .Case(caseFn: [](arith::MinimumFOp op) { return AtomicRMWKind::minimumf; })
3599 .Case(caseFn: [](arith::MaxNumFOp op) { return AtomicRMWKind::maxnumf; })
3600 .Case(caseFn: [](arith::MinNumFOp op) { return AtomicRMWKind::minnumf; })
3601 // Integer operations.
3602 .Case(caseFn: [](arith::AddIOp op) { return AtomicRMWKind::addi; })
3603 .Case(caseFn: [](arith::OrIOp op) { return AtomicRMWKind::ori; })
3604 .Case(caseFn: [](arith::XOrIOp op) { return AtomicRMWKind::xori; })
3605 .Case(caseFn: [](arith::AndIOp op) { return AtomicRMWKind::andi; })
3606 .Case(caseFn: [](arith::MaxUIOp op) { return AtomicRMWKind::maxu; })
3607 .Case(caseFn: [](arith::MinUIOp op) { return AtomicRMWKind::minu; })
3608 .Case(caseFn: [](arith::MaxSIOp op) { return AtomicRMWKind::maxs; })
3609 .Case(caseFn: [](arith::MinSIOp op) { return AtomicRMWKind::mins; })
3610 .Case(caseFn: [](arith::MulIOp op) { return AtomicRMWKind::muli; })
3611 .Default(std::nullopt);
3612 if (!maybeKind) {
3613 return std::nullopt;
3614 }
3615
3616 bool useOnlyFiniteValue = false;
3617 auto fmfOpInterface = dyn_cast<ArithFastMathInterface>(Val: op);
3618 if (fmfOpInterface) {
3619 arith::FastMathFlagsAttr fmfAttr = fmfOpInterface.getFastMathFlagsAttr();
3620 useOnlyFiniteValue =
3621 bitEnumContainsAny(bits: fmfAttr.getValue(), bit: arith::FastMathFlags::ninf);
3622 if (*maybeKind == AtomicRMWKind::maxnumf ||
3623 *maybeKind == AtomicRMWKind::minnumf) {
3624 if (bitEnumContainsAny(bits: fmfAttr.getValue(), bit: arith::FastMathFlags::nnan)) {
3625 *maybeKind = *maybeKind == AtomicRMWKind::maxnumf
3626 ? AtomicRMWKind::maximumf
3627 : AtomicRMWKind::minimumf;
3628 } else {
3629 useOnlyFiniteValue = false;
3630 }
3631 }
3632 }
3633
3634 // Builder only used as helper for attribute creation.
3635 OpBuilder b(op->getContext());
3636 Type resultType = op->getResult(idx: 0).getType();
3637
3638 return getIdentityValueAttr(kind: *maybeKind, resultType, builder&: b, loc: op->getLoc(),
3639 useOnlyFiniteValue);
3640}
3641
3642/// Returns the identity value associated with an AtomicRMWKind op.
3643Value mlir::arith::getIdentityValue(AtomicRMWKind op, Type resultType,
3644 OpBuilder &builder, Location loc,
3645 bool useOnlyFiniteValue) {
3646 if (auto attr = getIdentityValueAttr(kind: op, resultType, builder, loc,
3647 useOnlyFiniteValue))
3648 return arith::ConstantOp::create(builder, location: loc, value: attr);
3649 return {};
3650}
3651
3652/// Return the value obtained by applying the reduction operation kind
3653/// associated with a binary AtomicRMWKind op to `lhs` and `rhs`.
3654Value mlir::arith::getReductionOp(AtomicRMWKind op, OpBuilder &builder,
3655 Location loc, Value lhs, Value rhs) {
3656 switch (op) {
3657 case AtomicRMWKind::addf:
3658 return arith::AddFOp::create(builder, location: loc, lhs, rhs);
3659 case AtomicRMWKind::addi:
3660 return arith::AddIOp::create(builder, location: loc, lhs, rhs);
3661 case AtomicRMWKind::mulf:
3662 return arith::MulFOp::create(builder, location: loc, lhs, rhs);
3663 case AtomicRMWKind::muli:
3664 return arith::MulIOp::create(builder, location: loc, lhs, rhs);
3665 case AtomicRMWKind::maximumf:
3666 return arith::MaximumFOp::create(builder, location: loc, lhs, rhs);
3667 case AtomicRMWKind::minimumf:
3668 return arith::MinimumFOp::create(builder, location: loc, lhs, rhs);
3669 case AtomicRMWKind::maxnumf:
3670 return arith::MaxNumFOp::create(builder, location: loc, lhs, rhs);
3671 case AtomicRMWKind::minnumf:
3672 return arith::MinNumFOp::create(builder, location: loc, lhs, rhs);
3673 case AtomicRMWKind::maxs:
3674 return arith::MaxSIOp::create(builder, location: loc, lhs, rhs);
3675 case AtomicRMWKind::mins:
3676 return arith::MinSIOp::create(builder, location: loc, lhs, rhs);
3677 case AtomicRMWKind::maxu:
3678 return arith::MaxUIOp::create(builder, location: loc, lhs, rhs);
3679 case AtomicRMWKind::minu:
3680 return arith::MinUIOp::create(builder, location: loc, lhs, rhs);
3681 case AtomicRMWKind::ori:
3682 return arith::OrIOp::create(builder, location: loc, lhs, rhs);
3683 case AtomicRMWKind::andi:
3684 return arith::AndIOp::create(builder, location: loc, lhs, rhs);
3685 case AtomicRMWKind::xori:
3686 return arith::XOrIOp::create(builder, location: loc, lhs, rhs);
3687 // `assign` is not a reduction and has no corresponding binary operation.
3688 case AtomicRMWKind::assign:
3689 break;
3690 }
3691 (void)emitOptionalError(loc, args: "Reduction operation type not supported");
3692 return nullptr;
3693}
3694
3695//===----------------------------------------------------------------------===//
3696// TableGen'd op method definitions
3697//===----------------------------------------------------------------------===//
3698
3699#define GET_OP_CLASSES
3700#include "mlir/Dialect/Arith/IR/ArithOps.cpp.inc"
3701
3702//===----------------------------------------------------------------------===//
3703// TableGen'd enum attribute definitions
3704//===----------------------------------------------------------------------===//
3705
3706#include "mlir/Dialect/Arith/IR/ArithOpsEnums.cpp.inc"
3707