1//===- ExpandPatterns.cpp - Code to expand various math operations. -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements expansion of various math operations.
10//
11//===----------------------------------------------------------------------===//
12
13#include "mlir/Dialect/Arith/IR/Arith.h"
14#include "mlir/Dialect/Math/IR/Math.h"
15#include "mlir/Dialect/Math/Transforms/Passes.h"
16#include "mlir/Dialect/Tensor/IR/Tensor.h"
17#include "mlir/IR/Builders.h"
18#include "mlir/IR/Matchers.h"
19#include "mlir/IR/TypeUtilities.h"
20#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
21
22using namespace mlir;
23
24namespace mlir::math {
25#define GEN_PASS_DEF_MATHEXPANDOPSPASS
26#include "mlir/Dialect/Math/Transforms/Passes.h.inc"
27} // namespace mlir::math
28
29/// Returns true if the type is an unranked shaped type (e.g., tensor<*xf32>).
30/// Unranked types can't be expanded because tensor.splat requires ranked types.
31static bool isUnrankedShaped(Type type) {
32 auto shapedTy = dyn_cast<ShapedType>(Val&: type);
33 return shapedTy && !shapedTy.hasRank();
34}
35
36/// Create a float constant. For dynamically-shaped tensors, creates a scalar
37/// constant and uses tensor.dim + tensor.splat to broadcast to the target
38/// shape. The optional `dynamicShapeRef` provides the runtime dimension sizes.
39static Value createFloatConst(Location loc, Type type, APFloat value,
40 OpBuilder &b, Value dynamicShapeRef = Value()) {
41 bool losesInfo = false;
42 auto eltType = getElementTypeOrSelf(type);
43 // Convert double to the given `FloatType` with round-to-nearest-ties-to-even.
44 value.convert(ToSemantics: cast<FloatType>(Val&: eltType).getFloatSemantics(),
45 RM: APFloat::rmNearestTiesToEven, losesInfo: &losesInfo);
46 auto attr = b.getFloatAttr(type: eltType, value);
47 if (auto shapedTy = dyn_cast<ShapedType>(Val&: type)) {
48 if (shapedTy.hasStaticShape())
49 return arith::ConstantOp::create(builder&: b, location: loc,
50 value: DenseElementsAttr::get(type: shapedTy, values: attr));
51
52 // Dynamic shape: create scalar constant and splat to the target shape.
53 Value scalar = arith::ConstantOp::create(builder&: b, location: loc, result: eltType, value: attr);
54 SmallVector<Value> dynamicSizes;
55 for (int64_t i = 0; i < shapedTy.getRank(); ++i) {
56 if (shapedTy.isDynamicDim(idx: i))
57 dynamicSizes.push_back(
58 Elt: tensor::DimOp::create(builder&: b, location: loc, source: dynamicShapeRef, index: i));
59 }
60 return tensor::SplatOp::create(builder&: b, location: loc, aggregate: type, input: scalar, dynamicSizes);
61 }
62
63 return arith::ConstantOp::create(builder&: b, location: loc, value: attr);
64}
65
66static Value createFloatConst(Location loc, Type type, double value,
67 OpBuilder &b, Value dynamicShapeRef = Value()) {
68 return createFloatConst(loc, type, value: APFloat(value), b, dynamicShapeRef);
69}
70
71/// Create an integer constant. For dynamically-shaped tensors, creates a scalar
72/// constant and uses tensor.dim + tensor.splat to broadcast to the target
73/// shape. The optional `dynamicShapeRef` provides the runtime dimension sizes.
74static Value createIntConst(Location loc, Type type, int64_t value,
75 OpBuilder &b, Value dynamicShapeRef = Value()) {
76 auto eltType = getElementTypeOrSelf(type);
77 auto attr = b.getIntegerAttr(type: eltType, value);
78 if (auto shapedTy = dyn_cast<ShapedType>(Val&: type)) {
79 if (shapedTy.hasStaticShape())
80 return arith::ConstantOp::create(builder&: b, location: loc,
81 value: DenseElementsAttr::get(type: shapedTy, values: attr));
82
83 // Dynamic shape: create scalar constant and splat to the target shape.
84 Value scalar = arith::ConstantOp::create(builder&: b, location: loc, result: eltType, value: attr);
85 SmallVector<Value> dynamicSizes;
86 for (int64_t i = 0; i < shapedTy.getRank(); ++i) {
87 if (shapedTy.isDynamicDim(idx: i))
88 dynamicSizes.push_back(
89 Elt: tensor::DimOp::create(builder&: b, location: loc, source: dynamicShapeRef, index: i));
90 }
91 return tensor::SplatOp::create(builder&: b, location: loc, aggregate: type, input: scalar, dynamicSizes);
92 }
93
94 return arith::ConstantOp::create(builder&: b, location: loc, value: attr);
95}
96
97static Value createTruncatedFPValue(Value operand, ImplicitLocOpBuilder &b) {
98 Type opType = operand.getType();
99 Type i64Ty = b.getI64Type();
100 if (auto shapedTy = dyn_cast<ShapedType>(Val&: opType))
101 i64Ty = shapedTy.clone(elementType: i64Ty);
102 Value fixedConvert = arith::FPToSIOp::create(builder&: b, out: i64Ty, in: operand);
103 Value fpFixedConvert = arith::SIToFPOp::create(builder&: b, out: opType, in: fixedConvert);
104 // The truncation does not preserve the sign when the truncated
105 // value is -0. So here the sign is copied again.
106 return math::CopySignOp::create(builder&: b, lhs: fpFixedConvert, rhs: operand);
107}
108
109// sinhf(float x) -> (exp(x) - exp(-x)) / 2
110static LogicalResult convertSinhOp(math::SinhOp op, PatternRewriter &rewriter) {
111 ImplicitLocOpBuilder b(op->getLoc(), rewriter);
112 Value operand = op.getOperand();
113 Type opType = operand.getType();
114
115 if (isUnrankedShaped(type: opType))
116 return failure();
117
118 Value exp = math::ExpOp::create(builder&: b, operand);
119 Value neg = arith::NegFOp::create(builder&: b, operand);
120 Value nexp = math::ExpOp::create(builder&: b, operand: neg);
121 Value sub = arith::SubFOp::create(builder&: b, lhs: exp, rhs: nexp);
122 Value half = createFloatConst(loc: op->getLoc(), type: opType, value: 0.5, b&: rewriter, dynamicShapeRef: operand);
123 Value res = arith::MulFOp::create(builder&: b, lhs: sub, rhs: half);
124 rewriter.replaceOp(op, newValues: res);
125 return success();
126}
127
128// coshf(float x) -> (exp(x) + exp(-x)) / 2
129static LogicalResult convertCoshOp(math::CoshOp op, PatternRewriter &rewriter) {
130 ImplicitLocOpBuilder b(op->getLoc(), rewriter);
131 Value operand = op.getOperand();
132 Type opType = operand.getType();
133
134 if (isUnrankedShaped(type: opType))
135 return failure();
136
137 Value exp = math::ExpOp::create(builder&: b, operand);
138 Value neg = arith::NegFOp::create(builder&: b, operand);
139 Value nexp = math::ExpOp::create(builder&: b, operand: neg);
140 Value add = arith::AddFOp::create(builder&: b, lhs: exp, rhs: nexp);
141 Value half = createFloatConst(loc: op->getLoc(), type: opType, value: 0.5, b&: rewriter, dynamicShapeRef: operand);
142 Value res = arith::MulFOp::create(builder&: b, lhs: add, rhs: half);
143 rewriter.replaceOp(op, newValues: res);
144 return success();
145}
146
147/// Expands tanh op into
148/// 1-exp^{-2x} / 1+exp^{-2x}
149/// To avoid overflow we exploit the reflection symmetry `tanh(-x) = -tanh(x)`.
150/// We compute a "signs" value which is -1 if input is negative and +1 if input
151/// is positive. Then multiply the input by this value, guaranteeing that the
152/// result is positive, which also guarantees `exp^{-2x * sign(x)}` is in (0,
153/// 1]. Expand the computation on the input `x * sign(x)`, then multiply the
154/// result by `sign(x)` to retain sign of the real result.
155static LogicalResult convertTanhOp(math::TanhOp op, PatternRewriter &rewriter) {
156 Value operand = op.getOperand();
157 auto floatType = operand.getType();
158
159 if (isUnrankedShaped(type: floatType))
160 return failure();
161
162 Location loc = op.getLoc();
163 Value zero = createFloatConst(loc, type: floatType, value: 0.0, b&: rewriter, dynamicShapeRef: operand);
164 Value one = createFloatConst(loc, type: floatType, value: 1.0, b&: rewriter, dynamicShapeRef: operand);
165 Value negTwo = createFloatConst(loc, type: floatType, value: -2.0, b&: rewriter, dynamicShapeRef: operand);
166
167 // Compute sign(x) = cast<float_type>(x < 0) * (-2) + 1
168 Value isNegative = arith::CmpFOp::create(
169 builder&: rewriter, location: loc, predicate: arith::CmpFPredicate::OLT, lhs: operand, rhs: zero);
170 Value isNegativeFloat =
171 arith::UIToFPOp::create(builder&: rewriter, location: loc, out: floatType, in: isNegative);
172 Value isNegativeTimesNegTwo =
173 arith::MulFOp::create(builder&: rewriter, location: loc, lhs: isNegativeFloat, rhs: negTwo);
174 Value sign = arith::AddFOp::create(builder&: rewriter, location: loc, lhs: isNegativeTimesNegTwo, rhs: one);
175
176 // Normalize input to positive value: y = sign(x) * x
177 Value positiveX = arith::MulFOp::create(builder&: rewriter, location: loc, lhs: sign, rhs: operand);
178
179 // Decompose on normalized input
180 Value negDoubledX = arith::MulFOp::create(builder&: rewriter, location: loc, lhs: negTwo, rhs: positiveX);
181 Value exp2x = math::ExpOp::create(builder&: rewriter, location: loc, operand: negDoubledX);
182 Value dividend = arith::SubFOp::create(builder&: rewriter, location: loc, lhs: one, rhs: exp2x);
183 Value divisor = arith::AddFOp::create(builder&: rewriter, location: loc, lhs: one, rhs: exp2x);
184 Value positiveRes = arith::DivFOp::create(builder&: rewriter, location: loc, lhs: dividend, rhs: divisor);
185
186 // Multiply result by sign(x) to retain signs from negative inputs
187 rewriter.replaceOpWithNewOp<arith::MulFOp>(op, args&: sign, args&: positiveRes);
188
189 return success();
190}
191
192// Converts math.tan to math.sin, math.cos, and arith.divf.
193static LogicalResult convertTanOp(math::TanOp op, PatternRewriter &rewriter) {
194 ImplicitLocOpBuilder b(op->getLoc(), rewriter);
195 Value operand = op.getOperand();
196 Type type = operand.getType();
197 Value sin = math::SinOp::create(builder&: b, result: type, operand);
198 Value cos = math::CosOp::create(builder&: b, result: type, operand);
199 Value div = arith::DivFOp::create(builder&: b, type, lhs: sin, rhs: cos);
200 rewriter.replaceOp(op, newValues: div);
201 return success();
202}
203
204// asinh(float x) -> log(x + sqrt(x**2 + 1))
205static LogicalResult convertAsinhOp(math::AsinhOp op,
206 PatternRewriter &rewriter) {
207 ImplicitLocOpBuilder b(op->getLoc(), rewriter);
208 Value operand = op.getOperand();
209 Type opType = operand.getType();
210
211 if (isUnrankedShaped(type: opType))
212 return failure();
213
214 Value one = createFloatConst(loc: op->getLoc(), type: opType, value: 1.0, b&: rewriter, dynamicShapeRef: operand);
215 Value fma = math::FmaOp::create(builder&: b, a: operand, b: operand, c: one);
216 Value sqrt = math::SqrtOp::create(builder&: b, operand: fma);
217 Value add = arith::AddFOp::create(builder&: b, lhs: operand, rhs: sqrt);
218 Value res = math::LogOp::create(builder&: b, operand: add);
219 rewriter.replaceOp(op, newValues: res);
220 return success();
221}
222
223// acosh(float x) -> log(x + sqrt(x**2 - 1))
224static LogicalResult convertAcoshOp(math::AcoshOp op,
225 PatternRewriter &rewriter) {
226 ImplicitLocOpBuilder b(op->getLoc(), rewriter);
227 Value operand = op.getOperand();
228 Type opType = operand.getType();
229
230 if (isUnrankedShaped(type: opType))
231 return failure();
232
233 Value negOne =
234 createFloatConst(loc: op->getLoc(), type: opType, value: -1.0, b&: rewriter, dynamicShapeRef: operand);
235 Value fma = math::FmaOp::create(builder&: b, a: operand, b: operand, c: negOne);
236 Value sqrt = math::SqrtOp::create(builder&: b, operand: fma);
237 Value add = arith::AddFOp::create(builder&: b, lhs: operand, rhs: sqrt);
238 Value res = math::LogOp::create(builder&: b, operand: add);
239 rewriter.replaceOp(op, newValues: res);
240 return success();
241}
242
243// atanh(float x) -> log((1 + x) / (1 - x)) / 2
244static LogicalResult convertAtanhOp(math::AtanhOp op,
245 PatternRewriter &rewriter) {
246 ImplicitLocOpBuilder b(op->getLoc(), rewriter);
247 Value operand = op.getOperand();
248 Type opType = operand.getType();
249
250 if (isUnrankedShaped(type: opType))
251 return failure();
252
253 Value one = createFloatConst(loc: op->getLoc(), type: opType, value: 1.0, b&: rewriter, dynamicShapeRef: operand);
254 Value add = arith::AddFOp::create(builder&: b, lhs: operand, rhs: one);
255 Value neg = arith::NegFOp::create(builder&: b, operand);
256 Value sub = arith::AddFOp::create(builder&: b, lhs: neg, rhs: one);
257 Value div = arith::DivFOp::create(builder&: b, lhs: add, rhs: sub);
258 Value log = math::LogOp::create(builder&: b, operand: div);
259 Value half = createFloatConst(loc: op->getLoc(), type: opType, value: 0.5, b&: rewriter, dynamicShapeRef: operand);
260 Value res = arith::MulFOp::create(builder&: b, lhs: log, rhs: half);
261 rewriter.replaceOp(op, newValues: res);
262 return success();
263}
264
265static LogicalResult convertFmaFOp(math::FmaOp op, PatternRewriter &rewriter) {
266 ImplicitLocOpBuilder b(op->getLoc(), rewriter);
267 Value operandA = op.getOperand(i: 0);
268 Value operandB = op.getOperand(i: 1);
269 Value operandC = op.getOperand(i: 2);
270 Type type = op.getType();
271 Value mult = arith::MulFOp::create(builder&: b, type, lhs: operandA, rhs: operandB);
272 Value add = arith::AddFOp::create(builder&: b, type, lhs: mult, rhs: operandC);
273 rewriter.replaceOp(op, newValues: add);
274 return success();
275}
276
277// Converts a ceilf() function to the following:
278// ceilf(float x) ->
279// y = (float)(int) x
280// if (x > y) then incr = 1 else incr = 0
281// y = y + incr <= replace this op with the ceilf op.
282static LogicalResult convertCeilOp(math::CeilOp op, PatternRewriter &rewriter) {
283 ImplicitLocOpBuilder b(op->getLoc(), rewriter);
284 Value operand = op.getOperand();
285 Type opType = operand.getType();
286
287 if (isUnrankedShaped(type: opType))
288 return failure();
289
290 Type operandETy = getElementTypeOrSelf(type: opType);
291 FloatType floatTy = llvm::dyn_cast<FloatType>(Val&: operandETy);
292 const llvm::fltSemantics &semantics = floatTy.getFloatSemantics();
293
294 unsigned bitWidth = floatTy.getWidth();
295 unsigned mantissaWidth = floatTy.getFPMantissaWidth() - 1;
296 const int bias = (&semantics == &APFloat::Float8E8M0FNU())
297 ? -semantics.minExponent
298 : -(semantics.minExponent - 1);
299 bool hasNegativeZeroNaNEncoding =
300 (semantics.nanEncoding == llvm::fltNanEncoding::NegativeZero);
301
302 Type iTy = rewriter.getIntegerType(width: bitWidth);
303 if (auto shapedTy = dyn_cast<ShapedType>(Val&: opType))
304 iTy = shapedTy.clone(elementType: iTy);
305
306 // For IEEE-like floating-point formats with an unbiased exponent ≥
307 // `mantissaWidth` falls into one of these categories:
308 // - a large finite value (|x| ≥ 2^mantissaWidth), where all representable
309 // numbers are already integral, or
310 // - a special value (NaN or ±Inf), which also satisfies this exponent
311 // condition.
312 // For all such cases, `ceilf(x)` is defined to return `x` directly.
313 Value operandBitcast = arith::BitcastOp::create(builder&: b, out: iTy, in: operand);
314 Value cMask = createIntConst(
315 loc: op->getLoc(), type: iTy, value: static_cast<int64_t>((1ull << (bitWidth - 1)) - 1), b,
316 dynamicShapeRef: operand);
317 Value unsignedBits = arith::AndIOp::create(builder&: b, lhs: operandBitcast, rhs: cMask);
318 Value cThreshold = createIntConst(
319 loc: op->getLoc(), type: iTy,
320 value: static_cast<int64_t>((uint64_t(bias + mantissaWidth)) << mantissaWidth),
321 b, dynamicShapeRef: operand);
322 Value isLargeExp = arith::CmpIOp::create(builder&: b, predicate: arith::CmpIPredicate::uge,
323 lhs: unsignedBits, rhs: cThreshold);
324 Value isSpecialValOrLargeVal = isLargeExp;
325
326 // In FNUZ-suffixed floating point, NaN is represented by a sign bit of 1 and
327 // all 0s in the exponent and mantissa, therefore requires an explicit check.
328 if (hasNegativeZeroNaNEncoding) {
329 Value cNegZeroBits = createIntConst(
330 loc: op->getLoc(), type: iTy, value: static_cast<int64_t>(1ull << (bitWidth - 1)), b,
331 dynamicShapeRef: operand);
332 Value isNegZeroEncoding = arith::CmpIOp::create(
333 builder&: b, predicate: arith::CmpIPredicate::eq, lhs: operandBitcast, rhs: cNegZeroBits);
334 isSpecialValOrLargeVal =
335 arith::OrIOp::create(builder&: b, lhs: isLargeExp, rhs: isNegZeroEncoding);
336 }
337
338 Value fpFixedConvert = createTruncatedFPValue(operand, b);
339
340 // Creating constants for later use.
341 Value zero = createFloatConst(loc: op->getLoc(), type: opType, value: 0.00, b&: rewriter, dynamicShapeRef: operand);
342 Value one = createFloatConst(loc: op->getLoc(), type: opType, value: 1.00, b&: rewriter, dynamicShapeRef: operand);
343
344 Value gtCheck = arith::CmpFOp::create(builder&: b, predicate: arith::CmpFPredicate::OGT, lhs: operand,
345 rhs: fpFixedConvert);
346 Value incrValue =
347 arith::SelectOp::create(builder&: b, location: op->getLoc(), condition: gtCheck, true_value: one, false_value: zero);
348
349 Value add = arith::AddFOp::create(builder&: b, type: opType, lhs: fpFixedConvert, rhs: incrValue);
350 Value ret = arith::SelectOp::create(builder&: b, condition: isSpecialValOrLargeVal, true_value: operand, false_value: add);
351 rewriter.replaceOp(op, newValues: ret);
352 return success();
353}
354
355// Convert `math.fpowi` to a series of `arith.mulf` operations.
356// If the power is negative, we divide one by the result.
357// If both the base and power are zero, the result is 1.
358// In the case of non constant power, we convert the operation to `math.powf`.
359static LogicalResult convertFPowIOp(math::FPowIOp op,
360 PatternRewriter &rewriter) {
361 ImplicitLocOpBuilder b(op->getLoc(), rewriter);
362 Value base = op.getOperand(i: 0);
363 Value power = op.getOperand(i: 1);
364 Type baseType = base.getType();
365
366 auto convertFPowItoPowf = [&]() -> LogicalResult {
367 Value castPowerToFp =
368 arith::SIToFPOp::create(builder&: rewriter, location: op.getLoc(), out: baseType, in: power);
369 Value res = math::PowFOp::create(builder&: rewriter, location: op.getLoc(), result: baseType, lhs: base,
370 rhs: castPowerToFp);
371 rewriter.replaceOp(op, newValues: res);
372 return success();
373 };
374
375 Attribute cstAttr;
376 if (!matchPattern(value: power, pattern: m_Constant(bind_value: &cstAttr)))
377 return convertFPowItoPowf();
378
379 APInt value;
380 if (!matchPattern(attr: cstAttr, pattern: m_ConstantInt(bind_value: &value)))
381 return convertFPowItoPowf();
382
383 int64_t powerInt = value.getSExtValue();
384 bool isNegative = powerInt < 0;
385 int64_t absPower = std::abs(i: powerInt);
386 Value one = createFloatConst(loc: op->getLoc(), type: baseType, value: 1.00, b&: rewriter);
387 Value res = createFloatConst(loc: op->getLoc(), type: baseType, value: 1.00, b&: rewriter);
388
389 while (absPower > 0) {
390 if (absPower & 1)
391 res = arith::MulFOp::create(builder&: b, type: baseType, lhs: base, rhs: res);
392 absPower >>= 1;
393 base = arith::MulFOp::create(builder&: b, type: baseType, lhs: base, rhs: base);
394 }
395
396 // A zero `res` makes this a division by a signed zero, which yields the
397 // correctly signed infinity.
398 if (isNegative)
399 res = arith::DivFOp::create(builder&: b, type: baseType, lhs: one, rhs: res);
400
401 rewriter.replaceOp(op, newValues: res);
402 return success();
403}
404
405// Converts Powf(float a, float b) (meaning a^b) to exp^(b * ln(a))
406// Some special cases where b is constant are handled separately:
407// when b == 0, or |b| == 0.5, 1.0, or 2.0.
408static LogicalResult convertPowfOp(math::PowFOp op, PatternRewriter &rewriter) {
409 ImplicitLocOpBuilder b(op->getLoc(), rewriter);
410 Value operandA = op.getOperand(i: 0);
411 Value operandB = op.getOperand(i: 1);
412 auto typeA = operandA.getType();
413 auto typeB = operandB.getType();
414
415 auto &sem =
416 cast<mlir::FloatType>(Val: getElementTypeOrSelf(type: typeB)).getFloatSemantics();
417 APFloat valueB(sem);
418 auto mulf = [&](Value x, Value y) -> Value {
419 return arith::MulFOp::create(builder&: b, lhs: x, rhs: y);
420 };
421 if (matchPattern(value: operandB, pattern: m_ConstantFloat(bind_value: &valueB))) {
422 if (valueB.isZero()) {
423 // a^0 -> 1
424 Value one = createFloatConst(loc: op->getLoc(), type: typeA, value: 1.0, b&: rewriter);
425 rewriter.replaceOp(op, newValues: one);
426 return success();
427 }
428 if (valueB.isExactlyValue(V: 1.0)) {
429 // a^1 -> a
430 rewriter.replaceOp(op, newValues: operandA);
431 return success();
432 }
433 if (valueB.isExactlyValue(V: -1.0)) {
434 // a^(-1) -> 1 / a
435 Value one = createFloatConst(loc: op->getLoc(), type: typeA, value: 1.0, b&: rewriter);
436 Value div = arith::DivFOp::create(builder&: b, lhs: one, rhs: operandA);
437 rewriter.replaceOp(op, newValues: div);
438 return success();
439 }
440 if (valueB.isExactlyValue(V: 0.5)) {
441 // a^(1/2) -> sqrt(a)
442 Value sqrt = math::SqrtOp::create(builder&: b, operand: operandA);
443 rewriter.replaceOp(op, newValues: sqrt);
444 return success();
445 }
446 if (valueB.isExactlyValue(V: -0.5)) {
447 // a^(-1/2) -> 1 / sqrt(a)
448 Value rsqrt = math::RsqrtOp::create(builder&: b, operand: operandA);
449 rewriter.replaceOp(op, newValues: rsqrt);
450 return success();
451 }
452 if (valueB.isExactlyValue(V: 2.0)) {
453 // a^2 -> a * a
454 rewriter.replaceOp(op, newValues: mulf(operandA, operandA));
455 return success();
456 }
457 if (valueB.isExactlyValue(V: -2.0)) {
458 // a^(-2) -> 1 / (a * a)
459 Value one =
460 createFloatConst(loc: op->getLoc(), type: operandA.getType(), value: 1.0, b&: rewriter);
461 Value div = arith::DivFOp::create(builder&: b, lhs: one, rhs: mulf(operandA, operandA));
462 rewriter.replaceOp(op, newValues: div);
463 return success();
464 }
465 if (valueB.isExactlyValue(V: 3.0)) {
466 rewriter.replaceOp(op, newValues: mulf(mulf(operandA, operandA), operandA));
467 return success();
468 }
469 }
470
471 Value logA = math::LogOp::create(builder&: b, operand: operandA);
472 Value mult = arith::MulFOp::create(builder&: b, lhs: operandB, rhs: logA);
473 Value expResult = math::ExpOp::create(builder&: b, operand: mult);
474 rewriter.replaceOp(op, newValues: expResult);
475 return success();
476}
477
478// exp2f(float x) -> exp(x * ln(2))
479// Proof: Let's say 2^x = y
480// ln(2^x) = ln(y)
481// x * ln(2) = ln(y) => e ^(x*ln(2)) = y
482static LogicalResult convertExp2fOp(math::Exp2Op op,
483 PatternRewriter &rewriter) {
484 ImplicitLocOpBuilder b(op->getLoc(), rewriter);
485 Value operand = op.getOperand();
486 Type opType = operand.getType();
487
488 if (isUnrankedShaped(type: opType))
489 return failure();
490
491 Value ln2 =
492 createFloatConst(loc: op->getLoc(), type: opType, value: llvm::numbers::ln2, b, dynamicShapeRef: operand);
493 Value mult = arith::MulFOp::create(builder&: b, type: opType, lhs: operand, rhs: ln2);
494 Value exp = math::ExpOp::create(builder&: b, location: op->getLoc(), operand: mult);
495 rewriter.replaceOp(op, newValues: exp);
496 return success();
497}
498
499static LogicalResult convertRoundF64Op(math::RoundOp op,
500 PatternRewriter &rewriter) {
501 Location loc = op.getLoc();
502 ImplicitLocOpBuilder b(loc, rewriter);
503 Value operand = op.getOperand();
504 Type opType = operand.getType();
505 Type opEType = getElementTypeOrSelf(type: opType);
506
507 if (!opEType.isF64() || isUnrankedShaped(type: opType))
508 return failure();
509
510 Type intType = b.getI64Type();
511 if (auto shapedType = dyn_cast<ShapedType>(Val&: opType))
512 intType = shapedType.clone(elementType: intType);
513
514 auto intConst = [&](int64_t value) {
515 return createIntConst(loc, type: intType, value, b, dynamicShapeRef: operand);
516 };
517
518 Value zero = intConst(0);
519 Value one = intConst(1);
520 Value allOnes = intConst(-1);
521 Value cFractionBits = intConst(52);
522 Value cBias = intConst(1023);
523 Value cRoundShiftBase = intConst(51);
524 Value cShiftMask = intConst(63);
525 Value magnitudeMask = intConst(0x7fffffffffffffffLL);
526 Value halfBits = intConst(0x3fe0000000000000LL);
527 Value oneBits = intConst(0x3ff0000000000000LL);
528
529 Value bits = arith::BitcastOp::create(builder&: b, out: intType, in: operand);
530 Value magnitude = arith::AndIOp::create(builder&: b, lhs: bits, rhs: magnitudeMask);
531 Value sign = arith::XOrIOp::create(builder&: b, lhs: bits, rhs: magnitude);
532 Value exponentField = arith::ShRUIOp::create(builder&: b, lhs: magnitude, rhs: cFractionBits);
533 Value exponent = arith::SubIOp::create(builder&: b, lhs: exponentField, rhs: cBias);
534 Value roundShift = arith::SubIOp::create(builder&: b, lhs: cRoundShiftBase, rhs: exponent);
535 roundShift = arith::AndIOp::create(builder&: b, lhs: roundShift, rhs: cShiftMask);
536 Value roundBit = arith::ShLIOp::create(builder&: b, lhs: one, rhs: roundShift);
537 Value mask =
538 arith::SubIOp::create(builder&: b, lhs: arith::ShLIOp::create(builder&: b, lhs: roundBit, rhs: one), rhs: one);
539 Value invertedMask = arith::XOrIOp::create(builder&: b, lhs: mask, rhs: allOnes);
540 Value incremented = arith::AddIOp::create(builder&: b, lhs: magnitude, rhs: roundBit);
541 Value roundedMagnitude = arith::AndIOp::create(builder&: b, lhs: incremented, rhs: invertedMask);
542 Value atLeastHalf =
543 arith::CmpIOp::create(builder&: b, predicate: arith::CmpIPredicate::uge, lhs: magnitude, rhs: halfBits);
544 Value smallMagnitude = arith::SelectOp::create(builder&: b, condition: atLeastHalf, true_value: oneBits, false_value: zero);
545 Value isSmall =
546 arith::CmpIOp::create(builder&: b, predicate: arith::CmpIPredicate::slt, lhs: exponent, rhs: zero);
547 Value resultMagnitude =
548 arith::SelectOp::create(builder&: b, condition: isSmall, true_value: smallMagnitude, false_value: roundedMagnitude);
549 Value isLarge = arith::CmpIOp::create(builder&: b, predicate: arith::CmpIPredicate::sge, lhs: exponent,
550 rhs: cFractionBits);
551 resultMagnitude =
552 arith::SelectOp::create(builder&: b, condition: isLarge, true_value: magnitude, false_value: resultMagnitude);
553 Value resultBits = arith::OrIOp::create(builder&: b, lhs: resultMagnitude, rhs: sign);
554 rewriter.replaceOp(op, newOp: arith::BitcastOp::create(builder&: b, out: opType, in: resultBits));
555 return success();
556}
557
558static LogicalResult convertRoundOp(math::RoundOp op,
559 PatternRewriter &rewriter) {
560 Location loc = op.getLoc();
561 ImplicitLocOpBuilder b(loc, rewriter);
562 Value operand = op.getOperand();
563 Type opType = operand.getType();
564 Type opEType = getElementTypeOrSelf(type: opType);
565
566 if (isUnrankedShaped(type: opType))
567 return failure();
568
569 if (!opEType.isF32()) {
570 return rewriter.notifyMatchFailure(arg&: op, msg: "not a round of f32.");
571 }
572
573 Type i32Ty = b.getI32Type();
574 if (auto shapedTy = dyn_cast<ShapedType>(Val&: opType))
575 i32Ty = shapedTy.clone(elementType: i32Ty);
576
577 Value half = createFloatConst(loc, type: opType, value: 0.5, b, dynamicShapeRef: operand);
578 Value c23 = createIntConst(loc, type: i32Ty, value: 23, b, dynamicShapeRef: operand);
579 Value c127 = createIntConst(loc, type: i32Ty, value: 127, b, dynamicShapeRef: operand);
580 Value expMask = createIntConst(loc, type: i32Ty, value: (1 << 8) - 1, b, dynamicShapeRef: operand);
581
582 Value incrValue = math::CopySignOp::create(builder&: b, lhs: half, rhs: operand);
583 Value add = arith::AddFOp::create(builder&: b, type: opType, lhs: operand, rhs: incrValue);
584 Value fpFixedConvert = createTruncatedFPValue(operand: add, b);
585
586 // There are three cases where adding 0.5 to the value and truncating by
587 // converting to an i64 does not result in the correct behavior:
588 //
589 // 1. Special values: +-inf and +-nan
590 // Casting these special values to i64 has undefined behavior. To identify
591 // these values, we use the fact that these values are the only float
592 // values with the maximum possible biased exponent.
593 //
594 // 2. Large values: 2^23 <= |x| <= INT_64_MAX
595 // Adding 0.5 to a float larger than or equal to 2^23 results in precision
596 // errors that sometimes round the value up and sometimes round the value
597 // down. For example:
598 // 8388608.0 + 0.5 = 8388608.0
599 // 8388609.0 + 0.5 = 8388610.0
600 //
601 // 3. Very large values: |x| > INT_64_MAX
602 // Casting to i64 a value greater than the max i64 value will overflow the
603 // i64 leading to wrong outputs.
604 //
605 // All three cases satisfy the property `biasedExp >= 23`.
606 Value operandBitcast = arith::BitcastOp::create(builder&: b, out: i32Ty, in: operand);
607 Value operandExp = arith::AndIOp::create(
608 builder&: b, lhs: arith::ShRUIOp::create(builder&: b, lhs: operandBitcast, rhs: c23), rhs: expMask);
609 Value operandBiasedExp = arith::SubIOp::create(builder&: b, lhs: operandExp, rhs: c127);
610 Value isSpecialValOrLargeVal = arith::CmpIOp::create(
611 builder&: b, predicate: arith::CmpIPredicate::sge, lhs: operandBiasedExp, rhs: c23);
612
613 Value result = arith::SelectOp::create(builder&: b, condition: isSpecialValOrLargeVal, true_value: operand,
614 false_value: fpFixedConvert);
615 rewriter.replaceOp(op, newValues: result);
616 return success();
617}
618
619// Converts math.ctlz to scf and arith operations. This is done
620// by performing a binary search on the bits.
621static LogicalResult convertCtlzOp(math::CountLeadingZerosOp op,
622 PatternRewriter &rewriter) {
623 auto operand = op.getOperand();
624 auto operandTy = operand.getType();
625 auto eTy = getElementTypeOrSelf(type: operandTy);
626 Location loc = op.getLoc();
627
628 if (isUnrankedShaped(type: operandTy))
629 return failure();
630
631 // Only expand for integer or float element types (index has no fixed bitwidth).
632 if (!eTy.isIntOrFloat()) {
633 return rewriter.notifyMatchFailure(arg&: op, msg: "ctlz expansion only supports int or float types");
634 }
635
636 int32_t bitwidth = eTy.getIntOrFloatBitWidth();
637 if (bitwidth > 64)
638 return failure();
639
640 uint64_t allbits = -1;
641 if (bitwidth < 64) {
642 allbits = allbits >> (64 - bitwidth);
643 }
644
645 Value x = operand;
646 Value count = createIntConst(loc, type: operandTy, value: 0, b&: rewriter, dynamicShapeRef: operand);
647 for (int32_t bw = bitwidth; bw > 1; bw = bw / 2) {
648 auto half = bw / 2;
649 auto bits = createIntConst(loc, type: operandTy, value: half, b&: rewriter, dynamicShapeRef: operand);
650 auto mask =
651 createIntConst(loc, type: operandTy, value: allbits >> half, b&: rewriter, dynamicShapeRef: operand);
652
653 Value pred = arith::CmpIOp::create(builder&: rewriter, location: loc, predicate: arith::CmpIPredicate::ule,
654 lhs: x, rhs: mask);
655 Value add = arith::AddIOp::create(builder&: rewriter, location: loc, lhs: count, rhs: bits);
656 Value shift = arith::ShLIOp::create(builder&: rewriter, location: loc, lhs: x, rhs: bits);
657
658 x = arith::SelectOp::create(builder&: rewriter, location: loc, condition: pred, true_value: shift, false_value: x);
659 count = arith::SelectOp::create(builder&: rewriter, location: loc, condition: pred, true_value: add, false_value: count);
660 }
661
662 Value zero = createIntConst(loc, type: operandTy, value: 0, b&: rewriter, dynamicShapeRef: operand);
663 Value pred = arith::CmpIOp::create(builder&: rewriter, location: loc, predicate: arith::CmpIPredicate::eq,
664 lhs: operand, rhs: zero);
665
666 Value bwval = createIntConst(loc, type: operandTy, value: bitwidth, b&: rewriter, dynamicShapeRef: operand);
667 Value sel = arith::SelectOp::create(builder&: rewriter, location: loc, condition: pred, true_value: bwval, false_value: count);
668 rewriter.replaceOp(op, newValues: sel);
669 return success();
670}
671
672// Convert `math.roundeven` into `math.round` + arith ops
673static LogicalResult convertRoundEvenOp(math::RoundEvenOp op,
674 PatternRewriter &rewriter) {
675 Location loc = op.getLoc();
676 ImplicitLocOpBuilder b(loc, rewriter);
677 auto operand = op.getOperand();
678 Type operandTy = operand.getType();
679 Type resultTy = op.getType();
680 Type operandETy = getElementTypeOrSelf(type: operandTy);
681 Type resultETy = getElementTypeOrSelf(type: resultTy);
682
683 if (isUnrankedShaped(type: operandTy))
684 return failure();
685
686 if (!isa<FloatType>(Val: operandETy) || !isa<FloatType>(Val: resultETy)) {
687 return rewriter.notifyMatchFailure(arg&: op, msg: "not a roundeven of f16 or f32.");
688 }
689
690 Type fTy = operandTy;
691 Type iTy = rewriter.getIntegerType(width: operandETy.getIntOrFloatBitWidth());
692 if (auto shapedTy = dyn_cast<ShapedType>(Val&: fTy)) {
693 iTy = shapedTy.clone(elementType: iTy);
694 }
695
696 unsigned bitWidth = operandETy.getIntOrFloatBitWidth();
697 // The width returned by getFPMantissaWidth includes the integer bit.
698 unsigned mantissaWidth =
699 llvm::cast<FloatType>(Val&: operandETy).getFPMantissaWidth() - 1;
700 unsigned exponentWidth = bitWidth - mantissaWidth - 1;
701
702 // The names of the variables correspond to f32.
703 // f64: 1 bit sign | 11 bits exponent | 52 bits mantissa.
704 // f32: 1 bit sign | 8 bits exponent | 23 bits mantissa.
705 // f16: 1 bit sign | 5 bits exponent | 10 bits mantissa.
706 Value c1Float = createFloatConst(loc, type: fTy, value: 1.0, b, dynamicShapeRef: operand);
707 Value c0 = createIntConst(loc, type: iTy, value: 0, b, dynamicShapeRef: operand);
708 Value c1 = createIntConst(loc, type: iTy, value: 1, b, dynamicShapeRef: operand);
709 Value cNeg1 = createIntConst(loc, type: iTy, value: -1, b, dynamicShapeRef: operand);
710 Value c23 = createIntConst(loc, type: iTy, value: mantissaWidth, b, dynamicShapeRef: operand);
711 Value c31 = createIntConst(loc, type: iTy, value: bitWidth - 1, b, dynamicShapeRef: operand);
712 Value c127 =
713 createIntConst(loc, type: iTy, value: (1ull << (exponentWidth - 1)) - 1, b, dynamicShapeRef: operand);
714 Value c2To22 =
715 createIntConst(loc, type: iTy, value: 1ull << (mantissaWidth - 1), b, dynamicShapeRef: operand);
716 Value c23Mask =
717 createIntConst(loc, type: iTy, value: (1ull << mantissaWidth) - 1, b, dynamicShapeRef: operand);
718 Value expMask =
719 createIntConst(loc, type: iTy, value: (1ull << exponentWidth) - 1, b, dynamicShapeRef: operand);
720
721 Value operandBitcast = arith::BitcastOp::create(builder&: b, out: iTy, in: operand);
722 Value round = math::RoundOp::create(builder&: b, operand);
723 Value roundBitcast = arith::BitcastOp::create(builder&: b, out: iTy, in: round);
724
725 // Get biased exponents for operand and round(operand)
726 Value operandExp = arith::AndIOp::create(
727 builder&: b, lhs: arith::ShRUIOp::create(builder&: b, lhs: operandBitcast, rhs: c23), rhs: expMask);
728 Value operandBiasedExp = arith::SubIOp::create(builder&: b, lhs: operandExp, rhs: c127);
729 Value roundExp = arith::AndIOp::create(
730 builder&: b, lhs: arith::ShRUIOp::create(builder&: b, lhs: roundBitcast, rhs: c23), rhs: expMask);
731 Value roundBiasedExp = arith::SubIOp::create(builder&: b, lhs: roundExp, rhs: c127);
732
733 auto safeShiftRight = [&](Value x, Value shift) -> Value {
734 // Clamp shift to valid range [0, bitwidth - 1] to avoid undefined behavior
735 Value clampedShift = arith::MaxSIOp::create(builder&: b, lhs: shift, rhs: c0);
736 clampedShift = arith::MinSIOp::create(builder&: b, lhs: clampedShift, rhs: c31);
737 return arith::ShRUIOp::create(builder&: b, lhs: x, rhs: clampedShift);
738 };
739
740 auto maskMantissa = [&](Value mantissa,
741 Value mantissaMaskRightShift) -> Value {
742 Value shiftedMantissaMask = safeShiftRight(c23Mask, mantissaMaskRightShift);
743 return arith::AndIOp::create(builder&: b, lhs: mantissa, rhs: shiftedMantissaMask);
744 };
745
746 // A whole number `x`, such that `|x| != 1`, is even if the mantissa, ignoring
747 // the leftmost `clamp(biasedExp - 1, 0, 23)` bits, is zero. Large numbers
748 // with `biasedExp > 23` (numbers where there is not enough precision to store
749 // decimals) are always even, and they satisfy the even condition trivially
750 // since the mantissa without all its bits is zero. The even condition
751 // is also true for +-0, since they have `biasedExp = -127` and the entire
752 // mantissa is zero. The case of +-1 has to be handled separately. Here
753 // we identify these values by noting that +-1 are the only whole numbers with
754 // `biasedExp == 0`.
755 //
756 // The special values +-inf and +-nan also satisfy the same property that
757 // whole non-unit even numbers satisfy. In particular, the special values have
758 // `biasedExp > 23`, so they get treated as large numbers with no room for
759 // decimals, which are always even.
760 Value roundBiasedExpEq0 =
761 arith::CmpIOp::create(builder&: b, predicate: arith::CmpIPredicate::eq, lhs: roundBiasedExp, rhs: c0);
762 Value roundBiasedExpMinus1 = arith::SubIOp::create(builder&: b, lhs: roundBiasedExp, rhs: c1);
763 Value roundMaskedMantissa = maskMantissa(roundBitcast, roundBiasedExpMinus1);
764 Value roundIsNotEvenOrSpecialVal = arith::CmpIOp::create(
765 builder&: b, predicate: arith::CmpIPredicate::ne, lhs: roundMaskedMantissa, rhs: c0);
766 roundIsNotEvenOrSpecialVal =
767 arith::OrIOp::create(builder&: b, lhs: roundIsNotEvenOrSpecialVal, rhs: roundBiasedExpEq0);
768
769 // A value `x` with `0 <= biasedExp < 23`, is halfway between two consecutive
770 // integers if the bit at index `biasedExp` starting from the left in the
771 // mantissa is 1 and all the bits to the right are zero. Values with
772 // `biasedExp >= 23` don't have decimals, so they are never halfway. The
773 // values +-0.5 are the only halfway values that have `biasedExp == -1 < 0`,
774 // so these are handled separately. In particular, if `biasedExp == -1`, the
775 // value is halfway if the entire mantissa is zero.
776 Value operandBiasedExpEqNeg1 = arith::CmpIOp::create(
777 builder&: b, predicate: arith::CmpIPredicate::eq, lhs: operandBiasedExp, rhs: cNeg1);
778 Value expectedOperandMaskedMantissa = arith::SelectOp::create(
779 builder&: b, condition: operandBiasedExpEqNeg1, true_value: c0, false_value: safeShiftRight(c2To22, operandBiasedExp));
780 Value operandMaskedMantissa = maskMantissa(operandBitcast, operandBiasedExp);
781 Value operandIsHalfway =
782 arith::CmpIOp::create(builder&: b, predicate: arith::CmpIPredicate::eq, lhs: operandMaskedMantissa,
783 rhs: expectedOperandMaskedMantissa);
784 // Ensure `biasedExp` is in the valid range for half values.
785 Value operandBiasedExpGeNeg1 = arith::CmpIOp::create(
786 builder&: b, predicate: arith::CmpIPredicate::sge, lhs: operandBiasedExp, rhs: cNeg1);
787 Value operandBiasedExpLt23 = arith::CmpIOp::create(
788 builder&: b, predicate: arith::CmpIPredicate::slt, lhs: operandBiasedExp, rhs: c23);
789 operandIsHalfway =
790 arith::AndIOp::create(builder&: b, lhs: operandIsHalfway, rhs: operandBiasedExpLt23);
791 operandIsHalfway =
792 arith::AndIOp::create(builder&: b, lhs: operandIsHalfway, rhs: operandBiasedExpGeNeg1);
793
794 // Adjust rounded operand with `round(operand) - sign(operand)` to correct the
795 // case where `round` rounded in the opposite direction of `roundeven`.
796 Value sign = math::CopySignOp::create(builder&: b, lhs: c1Float, rhs: operand);
797 Value roundShifted = arith::SubFOp::create(builder&: b, lhs: round, rhs: sign);
798 // If the rounded value is even or a special value, we default to the behavior
799 // of `math.round`.
800 Value needsShift =
801 arith::AndIOp::create(builder&: b, lhs: roundIsNotEvenOrSpecialVal, rhs: operandIsHalfway);
802 Value result = arith::SelectOp::create(builder&: b, condition: needsShift, true_value: roundShifted, false_value: round);
803 // The `x - sign` adjustment does not preserve the sign when we are adjusting
804 // the value -1 to -0. So here the sign is copied again to ensure that -0.5 is
805 // rounded to -0.0.
806 result = math::CopySignOp::create(builder&: b, lhs: result, rhs: operand);
807 rewriter.replaceOp(op, newValues: result);
808 return success();
809}
810
811// Convert `math.rsqrt` into `arith.divf` + `math.sqrt`
812static LogicalResult convertRsqrtOp(math::RsqrtOp op,
813 PatternRewriter &rewriter) {
814 auto operand = op.getOperand();
815 auto operandTy = operand.getType();
816
817 if (isUnrankedShaped(type: operandTy))
818 return failure();
819
820 auto eTy = getElementTypeOrSelf(type: operandTy);
821 if (!isa<FloatType>(Val: eTy))
822 return failure();
823
824 Location loc = op->getLoc();
825 auto constOneFloat = createFloatConst(loc, type: operandTy, value: 1.0, b&: rewriter, dynamicShapeRef: operand);
826 auto sqrtOp = math::SqrtOp::create(builder&: rewriter, location: loc, operand);
827 rewriter.replaceOpWithNewOp<arith::DivFOp>(op, args&: constOneFloat, args&: sqrtOp);
828 return success();
829}
830
831// Convert `math.clampf` into `arith.minimumf` + `arith.maximumf`
832static LogicalResult convertClampfOp(math::ClampFOp op,
833 PatternRewriter &rewriter) {
834 auto minOp = arith::MinimumFOp::create(builder&: rewriter, location: op.getLoc(), lhs: op.getValue(),
835 rhs: op.getMax(), fastmath: op.getFastmath());
836 rewriter.replaceOpWithNewOp<arith::MaximumFOp>(op, args&: minOp, args: op.getMin(),
837 args: op.getFastmath());
838 return success();
839}
840
841void mlir::math::populateExpansionPatterns(RewritePatternSet &patterns,
842 ArrayRef<StringRef> opMnemonics) {
843 auto filter = [&](StringRef name) {
844 // This should be a static assert and `consume_front` take a twine, but none
845 // is currently possible. TODO: augment `StringRef::consume_front` and make
846 // `getDialectNamespace` use `std::string_view`.
847 assert("math" == MathDialect::getDialectNamespace());
848 name.consume_front(Prefix: "math.");
849 return opMnemonics.empty() || (llvm::count(Range&: opMnemonics, Element: name) > 0);
850 };
851 if (filter(CountLeadingZerosOp::getOperationName()))
852 patterns.add(implFn: convertCtlzOp);
853 if (filter(SinhOp::getOperationName()))
854 patterns.add(implFn: convertSinhOp);
855 if (filter(CoshOp::getOperationName()))
856 patterns.add(implFn: convertCoshOp);
857 if (filter(TanOp::getOperationName()))
858 patterns.add(implFn: convertTanOp);
859 if (filter(TanhOp::getOperationName()))
860 patterns.add(implFn: convertTanhOp);
861 if (filter(AsinhOp::getOperationName()))
862 patterns.add(implFn: convertAsinhOp);
863 if (filter(AcoshOp::getOperationName()))
864 patterns.add(implFn: convertAcoshOp);
865 if (filter(AtanhOp::getOperationName()))
866 patterns.add(implFn: convertAtanhOp);
867 if (filter(FmaOp::getOperationName()))
868 patterns.add(implFn: convertFmaFOp);
869 if (filter(CeilOp::getOperationName()))
870 patterns.add(implFn: convertCeilOp);
871 if (filter(Exp2Op::getOperationName()))
872 patterns.add(implFn: convertExp2fOp);
873 if (filter(PowFOp::getOperationName()))
874 patterns.add(implFn: convertPowfOp);
875 if (filter(FPowIOp::getOperationName()))
876 patterns.add(implFn: convertFPowIOp);
877 if (filter(RoundOp::getOperationName()))
878 patterns.add(implFn: convertRoundOp);
879 if (filter(RoundEvenOp::getOperationName()))
880 patterns.add(implFn: convertRoundEvenOp);
881 if (filter(RsqrtOp::getOperationName()))
882 patterns.add(implFn: convertRsqrtOp);
883 if (filter(ClampFOp::getOperationName()))
884 patterns.add(implFn: convertClampfOp);
885}
886
887//===----------------------------------------------------------------------===//
888// MathExpandOpsPass pass
889//===----------------------------------------------------------------------===//
890namespace {
891struct MathExpandOpsPass final
892 : math::impl::MathExpandOpsPassBase<MathExpandOpsPass> {
893 using MathExpandOpsPassBase::MathExpandOpsPassBase;
894
895 void runOnOperation() override {
896 RewritePatternSet patterns(&getContext());
897 SmallVector<StringRef> mnemonics =
898 llvm::to_vector_of<StringRef>(Range&: opMnemonics);
899 math::populateExpansionPatterns(patterns, opMnemonics: mnemonics);
900 if (expandRoundF64 &&
901 (mnemonics.empty() || llvm::count(Range&: mnemonics, Element: "round") > 0))
902 patterns.add(implFn: convertRoundF64Op);
903 if (failed(Result: applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns))))
904 return signalPassFailure();
905 }
906};
907} // namespace
908