| 1 | //===- ExpandPatterns.cpp - Code to expand various math operations. -------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file implements expansion of various math operations. |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 14 | #include "mlir/Dialect/Math/IR/Math.h" |
| 15 | #include "mlir/Dialect/Math/Transforms/Passes.h" |
| 16 | #include "mlir/Dialect/Tensor/IR/Tensor.h" |
| 17 | #include "mlir/IR/Builders.h" |
| 18 | #include "mlir/IR/Matchers.h" |
| 19 | #include "mlir/IR/TypeUtilities.h" |
| 20 | #include "mlir/Transforms/GreedyPatternRewriteDriver.h" |
| 21 | |
| 22 | using namespace mlir; |
| 23 | |
| 24 | namespace mlir::math { |
| 25 | #define GEN_PASS_DEF_MATHEXPANDOPSPASS |
| 26 | #include "mlir/Dialect/Math/Transforms/Passes.h.inc" |
| 27 | } // namespace mlir::math |
| 28 | |
| 29 | /// Returns true if the type is an unranked shaped type (e.g., tensor<*xf32>). |
| 30 | /// Unranked types can't be expanded because tensor.splat requires ranked types. |
| 31 | static bool isUnrankedShaped(Type type) { |
| 32 | auto shapedTy = dyn_cast<ShapedType>(Val&: type); |
| 33 | return shapedTy && !shapedTy.hasRank(); |
| 34 | } |
| 35 | |
| 36 | /// Create a float constant. For dynamically-shaped tensors, creates a scalar |
| 37 | /// constant and uses tensor.dim + tensor.splat to broadcast to the target |
| 38 | /// shape. The optional `dynamicShapeRef` provides the runtime dimension sizes. |
| 39 | static Value createFloatConst(Location loc, Type type, APFloat value, |
| 40 | OpBuilder &b, Value dynamicShapeRef = Value()) { |
| 41 | bool losesInfo = false; |
| 42 | auto eltType = getElementTypeOrSelf(type); |
| 43 | // Convert double to the given `FloatType` with round-to-nearest-ties-to-even. |
| 44 | value.convert(ToSemantics: cast<FloatType>(Val&: eltType).getFloatSemantics(), |
| 45 | RM: APFloat::rmNearestTiesToEven, losesInfo: &losesInfo); |
| 46 | auto attr = b.getFloatAttr(type: eltType, value); |
| 47 | if (auto shapedTy = dyn_cast<ShapedType>(Val&: type)) { |
| 48 | if (shapedTy.hasStaticShape()) |
| 49 | return arith::ConstantOp::create(builder&: b, location: loc, |
| 50 | value: DenseElementsAttr::get(type: shapedTy, values: attr)); |
| 51 | |
| 52 | // Dynamic shape: create scalar constant and splat to the target shape. |
| 53 | Value scalar = arith::ConstantOp::create(builder&: b, location: loc, result: eltType, value: attr); |
| 54 | SmallVector<Value> dynamicSizes; |
| 55 | for (int64_t i = 0; i < shapedTy.getRank(); ++i) { |
| 56 | if (shapedTy.isDynamicDim(idx: i)) |
| 57 | dynamicSizes.push_back( |
| 58 | Elt: tensor::DimOp::create(builder&: b, location: loc, source: dynamicShapeRef, index: i)); |
| 59 | } |
| 60 | return tensor::SplatOp::create(builder&: b, location: loc, aggregate: type, input: scalar, dynamicSizes); |
| 61 | } |
| 62 | |
| 63 | return arith::ConstantOp::create(builder&: b, location: loc, value: attr); |
| 64 | } |
| 65 | |
| 66 | static Value createFloatConst(Location loc, Type type, double value, |
| 67 | OpBuilder &b, Value dynamicShapeRef = Value()) { |
| 68 | return createFloatConst(loc, type, value: APFloat(value), b, dynamicShapeRef); |
| 69 | } |
| 70 | |
| 71 | /// Create an integer constant. For dynamically-shaped tensors, creates a scalar |
| 72 | /// constant and uses tensor.dim + tensor.splat to broadcast to the target |
| 73 | /// shape. The optional `dynamicShapeRef` provides the runtime dimension sizes. |
| 74 | static Value createIntConst(Location loc, Type type, int64_t value, |
| 75 | OpBuilder &b, Value dynamicShapeRef = Value()) { |
| 76 | auto eltType = getElementTypeOrSelf(type); |
| 77 | auto attr = b.getIntegerAttr(type: eltType, value); |
| 78 | if (auto shapedTy = dyn_cast<ShapedType>(Val&: type)) { |
| 79 | if (shapedTy.hasStaticShape()) |
| 80 | return arith::ConstantOp::create(builder&: b, location: loc, |
| 81 | value: DenseElementsAttr::get(type: shapedTy, values: attr)); |
| 82 | |
| 83 | // Dynamic shape: create scalar constant and splat to the target shape. |
| 84 | Value scalar = arith::ConstantOp::create(builder&: b, location: loc, result: eltType, value: attr); |
| 85 | SmallVector<Value> dynamicSizes; |
| 86 | for (int64_t i = 0; i < shapedTy.getRank(); ++i) { |
| 87 | if (shapedTy.isDynamicDim(idx: i)) |
| 88 | dynamicSizes.push_back( |
| 89 | Elt: tensor::DimOp::create(builder&: b, location: loc, source: dynamicShapeRef, index: i)); |
| 90 | } |
| 91 | return tensor::SplatOp::create(builder&: b, location: loc, aggregate: type, input: scalar, dynamicSizes); |
| 92 | } |
| 93 | |
| 94 | return arith::ConstantOp::create(builder&: b, location: loc, value: attr); |
| 95 | } |
| 96 | |
| 97 | static Value createTruncatedFPValue(Value operand, ImplicitLocOpBuilder &b) { |
| 98 | Type opType = operand.getType(); |
| 99 | Type i64Ty = b.getI64Type(); |
| 100 | if (auto shapedTy = dyn_cast<ShapedType>(Val&: opType)) |
| 101 | i64Ty = shapedTy.clone(elementType: i64Ty); |
| 102 | Value fixedConvert = arith::FPToSIOp::create(builder&: b, out: i64Ty, in: operand); |
| 103 | Value fpFixedConvert = arith::SIToFPOp::create(builder&: b, out: opType, in: fixedConvert); |
| 104 | // The truncation does not preserve the sign when the truncated |
| 105 | // value is -0. So here the sign is copied again. |
| 106 | return math::CopySignOp::create(builder&: b, lhs: fpFixedConvert, rhs: operand); |
| 107 | } |
| 108 | |
| 109 | // sinhf(float x) -> (exp(x) - exp(-x)) / 2 |
| 110 | static LogicalResult convertSinhOp(math::SinhOp op, PatternRewriter &rewriter) { |
| 111 | ImplicitLocOpBuilder b(op->getLoc(), rewriter); |
| 112 | Value operand = op.getOperand(); |
| 113 | Type opType = operand.getType(); |
| 114 | |
| 115 | if (isUnrankedShaped(type: opType)) |
| 116 | return failure(); |
| 117 | |
| 118 | Value exp = math::ExpOp::create(builder&: b, operand); |
| 119 | Value neg = arith::NegFOp::create(builder&: b, operand); |
| 120 | Value nexp = math::ExpOp::create(builder&: b, operand: neg); |
| 121 | Value sub = arith::SubFOp::create(builder&: b, lhs: exp, rhs: nexp); |
| 122 | Value half = createFloatConst(loc: op->getLoc(), type: opType, value: 0.5, b&: rewriter, dynamicShapeRef: operand); |
| 123 | Value res = arith::MulFOp::create(builder&: b, lhs: sub, rhs: half); |
| 124 | rewriter.replaceOp(op, newValues: res); |
| 125 | return success(); |
| 126 | } |
| 127 | |
| 128 | // coshf(float x) -> (exp(x) + exp(-x)) / 2 |
| 129 | static LogicalResult convertCoshOp(math::CoshOp op, PatternRewriter &rewriter) { |
| 130 | ImplicitLocOpBuilder b(op->getLoc(), rewriter); |
| 131 | Value operand = op.getOperand(); |
| 132 | Type opType = operand.getType(); |
| 133 | |
| 134 | if (isUnrankedShaped(type: opType)) |
| 135 | return failure(); |
| 136 | |
| 137 | Value exp = math::ExpOp::create(builder&: b, operand); |
| 138 | Value neg = arith::NegFOp::create(builder&: b, operand); |
| 139 | Value nexp = math::ExpOp::create(builder&: b, operand: neg); |
| 140 | Value add = arith::AddFOp::create(builder&: b, lhs: exp, rhs: nexp); |
| 141 | Value half = createFloatConst(loc: op->getLoc(), type: opType, value: 0.5, b&: rewriter, dynamicShapeRef: operand); |
| 142 | Value res = arith::MulFOp::create(builder&: b, lhs: add, rhs: half); |
| 143 | rewriter.replaceOp(op, newValues: res); |
| 144 | return success(); |
| 145 | } |
| 146 | |
| 147 | /// Expands tanh op into |
| 148 | /// 1-exp^{-2x} / 1+exp^{-2x} |
| 149 | /// To avoid overflow we exploit the reflection symmetry `tanh(-x) = -tanh(x)`. |
| 150 | /// We compute a "signs" value which is -1 if input is negative and +1 if input |
| 151 | /// is positive. Then multiply the input by this value, guaranteeing that the |
| 152 | /// result is positive, which also guarantees `exp^{-2x * sign(x)}` is in (0, |
| 153 | /// 1]. Expand the computation on the input `x * sign(x)`, then multiply the |
| 154 | /// result by `sign(x)` to retain sign of the real result. |
| 155 | static LogicalResult convertTanhOp(math::TanhOp op, PatternRewriter &rewriter) { |
| 156 | Value operand = op.getOperand(); |
| 157 | auto floatType = operand.getType(); |
| 158 | |
| 159 | if (isUnrankedShaped(type: floatType)) |
| 160 | return failure(); |
| 161 | |
| 162 | Location loc = op.getLoc(); |
| 163 | Value zero = createFloatConst(loc, type: floatType, value: 0.0, b&: rewriter, dynamicShapeRef: operand); |
| 164 | Value one = createFloatConst(loc, type: floatType, value: 1.0, b&: rewriter, dynamicShapeRef: operand); |
| 165 | Value negTwo = createFloatConst(loc, type: floatType, value: -2.0, b&: rewriter, dynamicShapeRef: operand); |
| 166 | |
| 167 | // Compute sign(x) = cast<float_type>(x < 0) * (-2) + 1 |
| 168 | Value isNegative = arith::CmpFOp::create( |
| 169 | builder&: rewriter, location: loc, predicate: arith::CmpFPredicate::OLT, lhs: operand, rhs: zero); |
| 170 | Value isNegativeFloat = |
| 171 | arith::UIToFPOp::create(builder&: rewriter, location: loc, out: floatType, in: isNegative); |
| 172 | Value isNegativeTimesNegTwo = |
| 173 | arith::MulFOp::create(builder&: rewriter, location: loc, lhs: isNegativeFloat, rhs: negTwo); |
| 174 | Value sign = arith::AddFOp::create(builder&: rewriter, location: loc, lhs: isNegativeTimesNegTwo, rhs: one); |
| 175 | |
| 176 | // Normalize input to positive value: y = sign(x) * x |
| 177 | Value positiveX = arith::MulFOp::create(builder&: rewriter, location: loc, lhs: sign, rhs: operand); |
| 178 | |
| 179 | // Decompose on normalized input |
| 180 | Value negDoubledX = arith::MulFOp::create(builder&: rewriter, location: loc, lhs: negTwo, rhs: positiveX); |
| 181 | Value exp2x = math::ExpOp::create(builder&: rewriter, location: loc, operand: negDoubledX); |
| 182 | Value dividend = arith::SubFOp::create(builder&: rewriter, location: loc, lhs: one, rhs: exp2x); |
| 183 | Value divisor = arith::AddFOp::create(builder&: rewriter, location: loc, lhs: one, rhs: exp2x); |
| 184 | Value positiveRes = arith::DivFOp::create(builder&: rewriter, location: loc, lhs: dividend, rhs: divisor); |
| 185 | |
| 186 | // Multiply result by sign(x) to retain signs from negative inputs |
| 187 | rewriter.replaceOpWithNewOp<arith::MulFOp>(op, args&: sign, args&: positiveRes); |
| 188 | |
| 189 | return success(); |
| 190 | } |
| 191 | |
| 192 | // Converts math.tan to math.sin, math.cos, and arith.divf. |
| 193 | static LogicalResult convertTanOp(math::TanOp op, PatternRewriter &rewriter) { |
| 194 | ImplicitLocOpBuilder b(op->getLoc(), rewriter); |
| 195 | Value operand = op.getOperand(); |
| 196 | Type type = operand.getType(); |
| 197 | Value sin = math::SinOp::create(builder&: b, result: type, operand); |
| 198 | Value cos = math::CosOp::create(builder&: b, result: type, operand); |
| 199 | Value div = arith::DivFOp::create(builder&: b, type, lhs: sin, rhs: cos); |
| 200 | rewriter.replaceOp(op, newValues: div); |
| 201 | return success(); |
| 202 | } |
| 203 | |
| 204 | // asinh(float x) -> log(x + sqrt(x**2 + 1)) |
| 205 | static LogicalResult convertAsinhOp(math::AsinhOp op, |
| 206 | PatternRewriter &rewriter) { |
| 207 | ImplicitLocOpBuilder b(op->getLoc(), rewriter); |
| 208 | Value operand = op.getOperand(); |
| 209 | Type opType = operand.getType(); |
| 210 | |
| 211 | if (isUnrankedShaped(type: opType)) |
| 212 | return failure(); |
| 213 | |
| 214 | Value one = createFloatConst(loc: op->getLoc(), type: opType, value: 1.0, b&: rewriter, dynamicShapeRef: operand); |
| 215 | Value fma = math::FmaOp::create(builder&: b, a: operand, b: operand, c: one); |
| 216 | Value sqrt = math::SqrtOp::create(builder&: b, operand: fma); |
| 217 | Value add = arith::AddFOp::create(builder&: b, lhs: operand, rhs: sqrt); |
| 218 | Value res = math::LogOp::create(builder&: b, operand: add); |
| 219 | rewriter.replaceOp(op, newValues: res); |
| 220 | return success(); |
| 221 | } |
| 222 | |
| 223 | // acosh(float x) -> log(x + sqrt(x**2 - 1)) |
| 224 | static LogicalResult convertAcoshOp(math::AcoshOp op, |
| 225 | PatternRewriter &rewriter) { |
| 226 | ImplicitLocOpBuilder b(op->getLoc(), rewriter); |
| 227 | Value operand = op.getOperand(); |
| 228 | Type opType = operand.getType(); |
| 229 | |
| 230 | if (isUnrankedShaped(type: opType)) |
| 231 | return failure(); |
| 232 | |
| 233 | Value negOne = |
| 234 | createFloatConst(loc: op->getLoc(), type: opType, value: -1.0, b&: rewriter, dynamicShapeRef: operand); |
| 235 | Value fma = math::FmaOp::create(builder&: b, a: operand, b: operand, c: negOne); |
| 236 | Value sqrt = math::SqrtOp::create(builder&: b, operand: fma); |
| 237 | Value add = arith::AddFOp::create(builder&: b, lhs: operand, rhs: sqrt); |
| 238 | Value res = math::LogOp::create(builder&: b, operand: add); |
| 239 | rewriter.replaceOp(op, newValues: res); |
| 240 | return success(); |
| 241 | } |
| 242 | |
| 243 | // atanh(float x) -> log((1 + x) / (1 - x)) / 2 |
| 244 | static LogicalResult convertAtanhOp(math::AtanhOp op, |
| 245 | PatternRewriter &rewriter) { |
| 246 | ImplicitLocOpBuilder b(op->getLoc(), rewriter); |
| 247 | Value operand = op.getOperand(); |
| 248 | Type opType = operand.getType(); |
| 249 | |
| 250 | if (isUnrankedShaped(type: opType)) |
| 251 | return failure(); |
| 252 | |
| 253 | Value one = createFloatConst(loc: op->getLoc(), type: opType, value: 1.0, b&: rewriter, dynamicShapeRef: operand); |
| 254 | Value add = arith::AddFOp::create(builder&: b, lhs: operand, rhs: one); |
| 255 | Value neg = arith::NegFOp::create(builder&: b, operand); |
| 256 | Value sub = arith::AddFOp::create(builder&: b, lhs: neg, rhs: one); |
| 257 | Value div = arith::DivFOp::create(builder&: b, lhs: add, rhs: sub); |
| 258 | Value log = math::LogOp::create(builder&: b, operand: div); |
| 259 | Value half = createFloatConst(loc: op->getLoc(), type: opType, value: 0.5, b&: rewriter, dynamicShapeRef: operand); |
| 260 | Value res = arith::MulFOp::create(builder&: b, lhs: log, rhs: half); |
| 261 | rewriter.replaceOp(op, newValues: res); |
| 262 | return success(); |
| 263 | } |
| 264 | |
| 265 | static LogicalResult convertFmaFOp(math::FmaOp op, PatternRewriter &rewriter) { |
| 266 | ImplicitLocOpBuilder b(op->getLoc(), rewriter); |
| 267 | Value operandA = op.getOperand(i: 0); |
| 268 | Value operandB = op.getOperand(i: 1); |
| 269 | Value operandC = op.getOperand(i: 2); |
| 270 | Type type = op.getType(); |
| 271 | Value mult = arith::MulFOp::create(builder&: b, type, lhs: operandA, rhs: operandB); |
| 272 | Value add = arith::AddFOp::create(builder&: b, type, lhs: mult, rhs: operandC); |
| 273 | rewriter.replaceOp(op, newValues: add); |
| 274 | return success(); |
| 275 | } |
| 276 | |
| 277 | // Converts a ceilf() function to the following: |
| 278 | // ceilf(float x) -> |
| 279 | // y = (float)(int) x |
| 280 | // if (x > y) then incr = 1 else incr = 0 |
| 281 | // y = y + incr <= replace this op with the ceilf op. |
| 282 | static LogicalResult convertCeilOp(math::CeilOp op, PatternRewriter &rewriter) { |
| 283 | ImplicitLocOpBuilder b(op->getLoc(), rewriter); |
| 284 | Value operand = op.getOperand(); |
| 285 | Type opType = operand.getType(); |
| 286 | |
| 287 | if (isUnrankedShaped(type: opType)) |
| 288 | return failure(); |
| 289 | |
| 290 | Type operandETy = getElementTypeOrSelf(type: opType); |
| 291 | FloatType floatTy = llvm::dyn_cast<FloatType>(Val&: operandETy); |
| 292 | const llvm::fltSemantics &semantics = floatTy.getFloatSemantics(); |
| 293 | |
| 294 | unsigned bitWidth = floatTy.getWidth(); |
| 295 | unsigned mantissaWidth = floatTy.getFPMantissaWidth() - 1; |
| 296 | const int bias = (&semantics == &APFloat::Float8E8M0FNU()) |
| 297 | ? -semantics.minExponent |
| 298 | : -(semantics.minExponent - 1); |
| 299 | bool hasNegativeZeroNaNEncoding = |
| 300 | (semantics.nanEncoding == llvm::fltNanEncoding::NegativeZero); |
| 301 | |
| 302 | Type iTy = rewriter.getIntegerType(width: bitWidth); |
| 303 | if (auto shapedTy = dyn_cast<ShapedType>(Val&: opType)) |
| 304 | iTy = shapedTy.clone(elementType: iTy); |
| 305 | |
| 306 | // For IEEE-like floating-point formats with an unbiased exponent ≥ |
| 307 | // `mantissaWidth` falls into one of these categories: |
| 308 | // - a large finite value (|x| ≥ 2^mantissaWidth), where all representable |
| 309 | // numbers are already integral, or |
| 310 | // - a special value (NaN or ±Inf), which also satisfies this exponent |
| 311 | // condition. |
| 312 | // For all such cases, `ceilf(x)` is defined to return `x` directly. |
| 313 | Value operandBitcast = arith::BitcastOp::create(builder&: b, out: iTy, in: operand); |
| 314 | Value cMask = createIntConst( |
| 315 | loc: op->getLoc(), type: iTy, value: static_cast<int64_t>((1ull << (bitWidth - 1)) - 1), b, |
| 316 | dynamicShapeRef: operand); |
| 317 | Value unsignedBits = arith::AndIOp::create(builder&: b, lhs: operandBitcast, rhs: cMask); |
| 318 | Value cThreshold = createIntConst( |
| 319 | loc: op->getLoc(), type: iTy, |
| 320 | value: static_cast<int64_t>((uint64_t(bias + mantissaWidth)) << mantissaWidth), |
| 321 | b, dynamicShapeRef: operand); |
| 322 | Value isLargeExp = arith::CmpIOp::create(builder&: b, predicate: arith::CmpIPredicate::uge, |
| 323 | lhs: unsignedBits, rhs: cThreshold); |
| 324 | Value isSpecialValOrLargeVal = isLargeExp; |
| 325 | |
| 326 | // In FNUZ-suffixed floating point, NaN is represented by a sign bit of 1 and |
| 327 | // all 0s in the exponent and mantissa, therefore requires an explicit check. |
| 328 | if (hasNegativeZeroNaNEncoding) { |
| 329 | Value cNegZeroBits = createIntConst( |
| 330 | loc: op->getLoc(), type: iTy, value: static_cast<int64_t>(1ull << (bitWidth - 1)), b, |
| 331 | dynamicShapeRef: operand); |
| 332 | Value isNegZeroEncoding = arith::CmpIOp::create( |
| 333 | builder&: b, predicate: arith::CmpIPredicate::eq, lhs: operandBitcast, rhs: cNegZeroBits); |
| 334 | isSpecialValOrLargeVal = |
| 335 | arith::OrIOp::create(builder&: b, lhs: isLargeExp, rhs: isNegZeroEncoding); |
| 336 | } |
| 337 | |
| 338 | Value fpFixedConvert = createTruncatedFPValue(operand, b); |
| 339 | |
| 340 | // Creating constants for later use. |
| 341 | Value zero = createFloatConst(loc: op->getLoc(), type: opType, value: 0.00, b&: rewriter, dynamicShapeRef: operand); |
| 342 | Value one = createFloatConst(loc: op->getLoc(), type: opType, value: 1.00, b&: rewriter, dynamicShapeRef: operand); |
| 343 | |
| 344 | Value gtCheck = arith::CmpFOp::create(builder&: b, predicate: arith::CmpFPredicate::OGT, lhs: operand, |
| 345 | rhs: fpFixedConvert); |
| 346 | Value incrValue = |
| 347 | arith::SelectOp::create(builder&: b, location: op->getLoc(), condition: gtCheck, true_value: one, false_value: zero); |
| 348 | |
| 349 | Value add = arith::AddFOp::create(builder&: b, type: opType, lhs: fpFixedConvert, rhs: incrValue); |
| 350 | Value ret = arith::SelectOp::create(builder&: b, condition: isSpecialValOrLargeVal, true_value: operand, false_value: add); |
| 351 | rewriter.replaceOp(op, newValues: ret); |
| 352 | return success(); |
| 353 | } |
| 354 | |
| 355 | // Convert `math.fpowi` to a series of `arith.mulf` operations. |
| 356 | // If the power is negative, we divide one by the result. |
| 357 | // If both the base and power are zero, the result is 1. |
| 358 | // In the case of non constant power, we convert the operation to `math.powf`. |
| 359 | static LogicalResult convertFPowIOp(math::FPowIOp op, |
| 360 | PatternRewriter &rewriter) { |
| 361 | ImplicitLocOpBuilder b(op->getLoc(), rewriter); |
| 362 | Value base = op.getOperand(i: 0); |
| 363 | Value power = op.getOperand(i: 1); |
| 364 | Type baseType = base.getType(); |
| 365 | |
| 366 | auto convertFPowItoPowf = [&]() -> LogicalResult { |
| 367 | Value castPowerToFp = |
| 368 | arith::SIToFPOp::create(builder&: rewriter, location: op.getLoc(), out: baseType, in: power); |
| 369 | Value res = math::PowFOp::create(builder&: rewriter, location: op.getLoc(), result: baseType, lhs: base, |
| 370 | rhs: castPowerToFp); |
| 371 | rewriter.replaceOp(op, newValues: res); |
| 372 | return success(); |
| 373 | }; |
| 374 | |
| 375 | Attribute cstAttr; |
| 376 | if (!matchPattern(value: power, pattern: m_Constant(bind_value: &cstAttr))) |
| 377 | return convertFPowItoPowf(); |
| 378 | |
| 379 | APInt value; |
| 380 | if (!matchPattern(attr: cstAttr, pattern: m_ConstantInt(bind_value: &value))) |
| 381 | return convertFPowItoPowf(); |
| 382 | |
| 383 | int64_t powerInt = value.getSExtValue(); |
| 384 | bool isNegative = powerInt < 0; |
| 385 | int64_t absPower = std::abs(i: powerInt); |
| 386 | Value one = createFloatConst(loc: op->getLoc(), type: baseType, value: 1.00, b&: rewriter); |
| 387 | Value res = createFloatConst(loc: op->getLoc(), type: baseType, value: 1.00, b&: rewriter); |
| 388 | |
| 389 | while (absPower > 0) { |
| 390 | if (absPower & 1) |
| 391 | res = arith::MulFOp::create(builder&: b, type: baseType, lhs: base, rhs: res); |
| 392 | absPower >>= 1; |
| 393 | base = arith::MulFOp::create(builder&: b, type: baseType, lhs: base, rhs: base); |
| 394 | } |
| 395 | |
| 396 | // A zero `res` makes this a division by a signed zero, which yields the |
| 397 | // correctly signed infinity. |
| 398 | if (isNegative) |
| 399 | res = arith::DivFOp::create(builder&: b, type: baseType, lhs: one, rhs: res); |
| 400 | |
| 401 | rewriter.replaceOp(op, newValues: res); |
| 402 | return success(); |
| 403 | } |
| 404 | |
| 405 | // Converts Powf(float a, float b) (meaning a^b) to exp^(b * ln(a)) |
| 406 | // Some special cases where b is constant are handled separately: |
| 407 | // when b == 0, or |b| == 0.5, 1.0, or 2.0. |
| 408 | static LogicalResult convertPowfOp(math::PowFOp op, PatternRewriter &rewriter) { |
| 409 | ImplicitLocOpBuilder b(op->getLoc(), rewriter); |
| 410 | Value operandA = op.getOperand(i: 0); |
| 411 | Value operandB = op.getOperand(i: 1); |
| 412 | auto typeA = operandA.getType(); |
| 413 | auto typeB = operandB.getType(); |
| 414 | |
| 415 | auto &sem = |
| 416 | cast<mlir::FloatType>(Val: getElementTypeOrSelf(type: typeB)).getFloatSemantics(); |
| 417 | APFloat valueB(sem); |
| 418 | auto mulf = [&](Value x, Value y) -> Value { |
| 419 | return arith::MulFOp::create(builder&: b, lhs: x, rhs: y); |
| 420 | }; |
| 421 | if (matchPattern(value: operandB, pattern: m_ConstantFloat(bind_value: &valueB))) { |
| 422 | if (valueB.isZero()) { |
| 423 | // a^0 -> 1 |
| 424 | Value one = createFloatConst(loc: op->getLoc(), type: typeA, value: 1.0, b&: rewriter); |
| 425 | rewriter.replaceOp(op, newValues: one); |
| 426 | return success(); |
| 427 | } |
| 428 | if (valueB.isExactlyValue(V: 1.0)) { |
| 429 | // a^1 -> a |
| 430 | rewriter.replaceOp(op, newValues: operandA); |
| 431 | return success(); |
| 432 | } |
| 433 | if (valueB.isExactlyValue(V: -1.0)) { |
| 434 | // a^(-1) -> 1 / a |
| 435 | Value one = createFloatConst(loc: op->getLoc(), type: typeA, value: 1.0, b&: rewriter); |
| 436 | Value div = arith::DivFOp::create(builder&: b, lhs: one, rhs: operandA); |
| 437 | rewriter.replaceOp(op, newValues: div); |
| 438 | return success(); |
| 439 | } |
| 440 | if (valueB.isExactlyValue(V: 0.5)) { |
| 441 | // a^(1/2) -> sqrt(a) |
| 442 | Value sqrt = math::SqrtOp::create(builder&: b, operand: operandA); |
| 443 | rewriter.replaceOp(op, newValues: sqrt); |
| 444 | return success(); |
| 445 | } |
| 446 | if (valueB.isExactlyValue(V: -0.5)) { |
| 447 | // a^(-1/2) -> 1 / sqrt(a) |
| 448 | Value rsqrt = math::RsqrtOp::create(builder&: b, operand: operandA); |
| 449 | rewriter.replaceOp(op, newValues: rsqrt); |
| 450 | return success(); |
| 451 | } |
| 452 | if (valueB.isExactlyValue(V: 2.0)) { |
| 453 | // a^2 -> a * a |
| 454 | rewriter.replaceOp(op, newValues: mulf(operandA, operandA)); |
| 455 | return success(); |
| 456 | } |
| 457 | if (valueB.isExactlyValue(V: -2.0)) { |
| 458 | // a^(-2) -> 1 / (a * a) |
| 459 | Value one = |
| 460 | createFloatConst(loc: op->getLoc(), type: operandA.getType(), value: 1.0, b&: rewriter); |
| 461 | Value div = arith::DivFOp::create(builder&: b, lhs: one, rhs: mulf(operandA, operandA)); |
| 462 | rewriter.replaceOp(op, newValues: div); |
| 463 | return success(); |
| 464 | } |
| 465 | if (valueB.isExactlyValue(V: 3.0)) { |
| 466 | rewriter.replaceOp(op, newValues: mulf(mulf(operandA, operandA), operandA)); |
| 467 | return success(); |
| 468 | } |
| 469 | } |
| 470 | |
| 471 | Value logA = math::LogOp::create(builder&: b, operand: operandA); |
| 472 | Value mult = arith::MulFOp::create(builder&: b, lhs: operandB, rhs: logA); |
| 473 | Value expResult = math::ExpOp::create(builder&: b, operand: mult); |
| 474 | rewriter.replaceOp(op, newValues: expResult); |
| 475 | return success(); |
| 476 | } |
| 477 | |
| 478 | // exp2f(float x) -> exp(x * ln(2)) |
| 479 | // Proof: Let's say 2^x = y |
| 480 | // ln(2^x) = ln(y) |
| 481 | // x * ln(2) = ln(y) => e ^(x*ln(2)) = y |
| 482 | static LogicalResult convertExp2fOp(math::Exp2Op op, |
| 483 | PatternRewriter &rewriter) { |
| 484 | ImplicitLocOpBuilder b(op->getLoc(), rewriter); |
| 485 | Value operand = op.getOperand(); |
| 486 | Type opType = operand.getType(); |
| 487 | |
| 488 | if (isUnrankedShaped(type: opType)) |
| 489 | return failure(); |
| 490 | |
| 491 | Value ln2 = |
| 492 | createFloatConst(loc: op->getLoc(), type: opType, value: llvm::numbers::ln2, b, dynamicShapeRef: operand); |
| 493 | Value mult = arith::MulFOp::create(builder&: b, type: opType, lhs: operand, rhs: ln2); |
| 494 | Value exp = math::ExpOp::create(builder&: b, location: op->getLoc(), operand: mult); |
| 495 | rewriter.replaceOp(op, newValues: exp); |
| 496 | return success(); |
| 497 | } |
| 498 | |
| 499 | static LogicalResult convertRoundF64Op(math::RoundOp op, |
| 500 | PatternRewriter &rewriter) { |
| 501 | Location loc = op.getLoc(); |
| 502 | ImplicitLocOpBuilder b(loc, rewriter); |
| 503 | Value operand = op.getOperand(); |
| 504 | Type opType = operand.getType(); |
| 505 | Type opEType = getElementTypeOrSelf(type: opType); |
| 506 | |
| 507 | if (!opEType.isF64() || isUnrankedShaped(type: opType)) |
| 508 | return failure(); |
| 509 | |
| 510 | Type intType = b.getI64Type(); |
| 511 | if (auto shapedType = dyn_cast<ShapedType>(Val&: opType)) |
| 512 | intType = shapedType.clone(elementType: intType); |
| 513 | |
| 514 | auto intConst = [&](int64_t value) { |
| 515 | return createIntConst(loc, type: intType, value, b, dynamicShapeRef: operand); |
| 516 | }; |
| 517 | |
| 518 | Value zero = intConst(0); |
| 519 | Value one = intConst(1); |
| 520 | Value allOnes = intConst(-1); |
| 521 | Value cFractionBits = intConst(52); |
| 522 | Value cBias = intConst(1023); |
| 523 | Value cRoundShiftBase = intConst(51); |
| 524 | Value cShiftMask = intConst(63); |
| 525 | Value magnitudeMask = intConst(0x7fffffffffffffffLL); |
| 526 | Value halfBits = intConst(0x3fe0000000000000LL); |
| 527 | Value oneBits = intConst(0x3ff0000000000000LL); |
| 528 | |
| 529 | Value bits = arith::BitcastOp::create(builder&: b, out: intType, in: operand); |
| 530 | Value magnitude = arith::AndIOp::create(builder&: b, lhs: bits, rhs: magnitudeMask); |
| 531 | Value sign = arith::XOrIOp::create(builder&: b, lhs: bits, rhs: magnitude); |
| 532 | Value exponentField = arith::ShRUIOp::create(builder&: b, lhs: magnitude, rhs: cFractionBits); |
| 533 | Value exponent = arith::SubIOp::create(builder&: b, lhs: exponentField, rhs: cBias); |
| 534 | Value roundShift = arith::SubIOp::create(builder&: b, lhs: cRoundShiftBase, rhs: exponent); |
| 535 | roundShift = arith::AndIOp::create(builder&: b, lhs: roundShift, rhs: cShiftMask); |
| 536 | Value roundBit = arith::ShLIOp::create(builder&: b, lhs: one, rhs: roundShift); |
| 537 | Value mask = |
| 538 | arith::SubIOp::create(builder&: b, lhs: arith::ShLIOp::create(builder&: b, lhs: roundBit, rhs: one), rhs: one); |
| 539 | Value invertedMask = arith::XOrIOp::create(builder&: b, lhs: mask, rhs: allOnes); |
| 540 | Value incremented = arith::AddIOp::create(builder&: b, lhs: magnitude, rhs: roundBit); |
| 541 | Value roundedMagnitude = arith::AndIOp::create(builder&: b, lhs: incremented, rhs: invertedMask); |
| 542 | Value atLeastHalf = |
| 543 | arith::CmpIOp::create(builder&: b, predicate: arith::CmpIPredicate::uge, lhs: magnitude, rhs: halfBits); |
| 544 | Value smallMagnitude = arith::SelectOp::create(builder&: b, condition: atLeastHalf, true_value: oneBits, false_value: zero); |
| 545 | Value isSmall = |
| 546 | arith::CmpIOp::create(builder&: b, predicate: arith::CmpIPredicate::slt, lhs: exponent, rhs: zero); |
| 547 | Value resultMagnitude = |
| 548 | arith::SelectOp::create(builder&: b, condition: isSmall, true_value: smallMagnitude, false_value: roundedMagnitude); |
| 549 | Value isLarge = arith::CmpIOp::create(builder&: b, predicate: arith::CmpIPredicate::sge, lhs: exponent, |
| 550 | rhs: cFractionBits); |
| 551 | resultMagnitude = |
| 552 | arith::SelectOp::create(builder&: b, condition: isLarge, true_value: magnitude, false_value: resultMagnitude); |
| 553 | Value resultBits = arith::OrIOp::create(builder&: b, lhs: resultMagnitude, rhs: sign); |
| 554 | rewriter.replaceOp(op, newOp: arith::BitcastOp::create(builder&: b, out: opType, in: resultBits)); |
| 555 | return success(); |
| 556 | } |
| 557 | |
| 558 | static LogicalResult convertRoundOp(math::RoundOp op, |
| 559 | PatternRewriter &rewriter) { |
| 560 | Location loc = op.getLoc(); |
| 561 | ImplicitLocOpBuilder b(loc, rewriter); |
| 562 | Value operand = op.getOperand(); |
| 563 | Type opType = operand.getType(); |
| 564 | Type opEType = getElementTypeOrSelf(type: opType); |
| 565 | |
| 566 | if (isUnrankedShaped(type: opType)) |
| 567 | return failure(); |
| 568 | |
| 569 | if (!opEType.isF32()) { |
| 570 | return rewriter.notifyMatchFailure(arg&: op, msg: "not a round of f32." ); |
| 571 | } |
| 572 | |
| 573 | Type i32Ty = b.getI32Type(); |
| 574 | if (auto shapedTy = dyn_cast<ShapedType>(Val&: opType)) |
| 575 | i32Ty = shapedTy.clone(elementType: i32Ty); |
| 576 | |
| 577 | Value half = createFloatConst(loc, type: opType, value: 0.5, b, dynamicShapeRef: operand); |
| 578 | Value c23 = createIntConst(loc, type: i32Ty, value: 23, b, dynamicShapeRef: operand); |
| 579 | Value c127 = createIntConst(loc, type: i32Ty, value: 127, b, dynamicShapeRef: operand); |
| 580 | Value expMask = createIntConst(loc, type: i32Ty, value: (1 << 8) - 1, b, dynamicShapeRef: operand); |
| 581 | |
| 582 | Value incrValue = math::CopySignOp::create(builder&: b, lhs: half, rhs: operand); |
| 583 | Value add = arith::AddFOp::create(builder&: b, type: opType, lhs: operand, rhs: incrValue); |
| 584 | Value fpFixedConvert = createTruncatedFPValue(operand: add, b); |
| 585 | |
| 586 | // There are three cases where adding 0.5 to the value and truncating by |
| 587 | // converting to an i64 does not result in the correct behavior: |
| 588 | // |
| 589 | // 1. Special values: +-inf and +-nan |
| 590 | // Casting these special values to i64 has undefined behavior. To identify |
| 591 | // these values, we use the fact that these values are the only float |
| 592 | // values with the maximum possible biased exponent. |
| 593 | // |
| 594 | // 2. Large values: 2^23 <= |x| <= INT_64_MAX |
| 595 | // Adding 0.5 to a float larger than or equal to 2^23 results in precision |
| 596 | // errors that sometimes round the value up and sometimes round the value |
| 597 | // down. For example: |
| 598 | // 8388608.0 + 0.5 = 8388608.0 |
| 599 | // 8388609.0 + 0.5 = 8388610.0 |
| 600 | // |
| 601 | // 3. Very large values: |x| > INT_64_MAX |
| 602 | // Casting to i64 a value greater than the max i64 value will overflow the |
| 603 | // i64 leading to wrong outputs. |
| 604 | // |
| 605 | // All three cases satisfy the property `biasedExp >= 23`. |
| 606 | Value operandBitcast = arith::BitcastOp::create(builder&: b, out: i32Ty, in: operand); |
| 607 | Value operandExp = arith::AndIOp::create( |
| 608 | builder&: b, lhs: arith::ShRUIOp::create(builder&: b, lhs: operandBitcast, rhs: c23), rhs: expMask); |
| 609 | Value operandBiasedExp = arith::SubIOp::create(builder&: b, lhs: operandExp, rhs: c127); |
| 610 | Value isSpecialValOrLargeVal = arith::CmpIOp::create( |
| 611 | builder&: b, predicate: arith::CmpIPredicate::sge, lhs: operandBiasedExp, rhs: c23); |
| 612 | |
| 613 | Value result = arith::SelectOp::create(builder&: b, condition: isSpecialValOrLargeVal, true_value: operand, |
| 614 | false_value: fpFixedConvert); |
| 615 | rewriter.replaceOp(op, newValues: result); |
| 616 | return success(); |
| 617 | } |
| 618 | |
| 619 | // Converts math.ctlz to scf and arith operations. This is done |
| 620 | // by performing a binary search on the bits. |
| 621 | static LogicalResult convertCtlzOp(math::CountLeadingZerosOp op, |
| 622 | PatternRewriter &rewriter) { |
| 623 | auto operand = op.getOperand(); |
| 624 | auto operandTy = operand.getType(); |
| 625 | auto eTy = getElementTypeOrSelf(type: operandTy); |
| 626 | Location loc = op.getLoc(); |
| 627 | |
| 628 | if (isUnrankedShaped(type: operandTy)) |
| 629 | return failure(); |
| 630 | |
| 631 | // Only expand for integer or float element types (index has no fixed bitwidth). |
| 632 | if (!eTy.isIntOrFloat()) { |
| 633 | return rewriter.notifyMatchFailure(arg&: op, msg: "ctlz expansion only supports int or float types" ); |
| 634 | } |
| 635 | |
| 636 | int32_t bitwidth = eTy.getIntOrFloatBitWidth(); |
| 637 | if (bitwidth > 64) |
| 638 | return failure(); |
| 639 | |
| 640 | uint64_t allbits = -1; |
| 641 | if (bitwidth < 64) { |
| 642 | allbits = allbits >> (64 - bitwidth); |
| 643 | } |
| 644 | |
| 645 | Value x = operand; |
| 646 | Value count = createIntConst(loc, type: operandTy, value: 0, b&: rewriter, dynamicShapeRef: operand); |
| 647 | for (int32_t bw = bitwidth; bw > 1; bw = bw / 2) { |
| 648 | auto half = bw / 2; |
| 649 | auto bits = createIntConst(loc, type: operandTy, value: half, b&: rewriter, dynamicShapeRef: operand); |
| 650 | auto mask = |
| 651 | createIntConst(loc, type: operandTy, value: allbits >> half, b&: rewriter, dynamicShapeRef: operand); |
| 652 | |
| 653 | Value pred = arith::CmpIOp::create(builder&: rewriter, location: loc, predicate: arith::CmpIPredicate::ule, |
| 654 | lhs: x, rhs: mask); |
| 655 | Value add = arith::AddIOp::create(builder&: rewriter, location: loc, lhs: count, rhs: bits); |
| 656 | Value shift = arith::ShLIOp::create(builder&: rewriter, location: loc, lhs: x, rhs: bits); |
| 657 | |
| 658 | x = arith::SelectOp::create(builder&: rewriter, location: loc, condition: pred, true_value: shift, false_value: x); |
| 659 | count = arith::SelectOp::create(builder&: rewriter, location: loc, condition: pred, true_value: add, false_value: count); |
| 660 | } |
| 661 | |
| 662 | Value zero = createIntConst(loc, type: operandTy, value: 0, b&: rewriter, dynamicShapeRef: operand); |
| 663 | Value pred = arith::CmpIOp::create(builder&: rewriter, location: loc, predicate: arith::CmpIPredicate::eq, |
| 664 | lhs: operand, rhs: zero); |
| 665 | |
| 666 | Value bwval = createIntConst(loc, type: operandTy, value: bitwidth, b&: rewriter, dynamicShapeRef: operand); |
| 667 | Value sel = arith::SelectOp::create(builder&: rewriter, location: loc, condition: pred, true_value: bwval, false_value: count); |
| 668 | rewriter.replaceOp(op, newValues: sel); |
| 669 | return success(); |
| 670 | } |
| 671 | |
| 672 | // Convert `math.roundeven` into `math.round` + arith ops |
| 673 | static LogicalResult convertRoundEvenOp(math::RoundEvenOp op, |
| 674 | PatternRewriter &rewriter) { |
| 675 | Location loc = op.getLoc(); |
| 676 | ImplicitLocOpBuilder b(loc, rewriter); |
| 677 | auto operand = op.getOperand(); |
| 678 | Type operandTy = operand.getType(); |
| 679 | Type resultTy = op.getType(); |
| 680 | Type operandETy = getElementTypeOrSelf(type: operandTy); |
| 681 | Type resultETy = getElementTypeOrSelf(type: resultTy); |
| 682 | |
| 683 | if (isUnrankedShaped(type: operandTy)) |
| 684 | return failure(); |
| 685 | |
| 686 | if (!isa<FloatType>(Val: operandETy) || !isa<FloatType>(Val: resultETy)) { |
| 687 | return rewriter.notifyMatchFailure(arg&: op, msg: "not a roundeven of f16 or f32." ); |
| 688 | } |
| 689 | |
| 690 | Type fTy = operandTy; |
| 691 | Type iTy = rewriter.getIntegerType(width: operandETy.getIntOrFloatBitWidth()); |
| 692 | if (auto shapedTy = dyn_cast<ShapedType>(Val&: fTy)) { |
| 693 | iTy = shapedTy.clone(elementType: iTy); |
| 694 | } |
| 695 | |
| 696 | unsigned bitWidth = operandETy.getIntOrFloatBitWidth(); |
| 697 | // The width returned by getFPMantissaWidth includes the integer bit. |
| 698 | unsigned mantissaWidth = |
| 699 | llvm::cast<FloatType>(Val&: operandETy).getFPMantissaWidth() - 1; |
| 700 | unsigned exponentWidth = bitWidth - mantissaWidth - 1; |
| 701 | |
| 702 | // The names of the variables correspond to f32. |
| 703 | // f64: 1 bit sign | 11 bits exponent | 52 bits mantissa. |
| 704 | // f32: 1 bit sign | 8 bits exponent | 23 bits mantissa. |
| 705 | // f16: 1 bit sign | 5 bits exponent | 10 bits mantissa. |
| 706 | Value c1Float = createFloatConst(loc, type: fTy, value: 1.0, b, dynamicShapeRef: operand); |
| 707 | Value c0 = createIntConst(loc, type: iTy, value: 0, b, dynamicShapeRef: operand); |
| 708 | Value c1 = createIntConst(loc, type: iTy, value: 1, b, dynamicShapeRef: operand); |
| 709 | Value cNeg1 = createIntConst(loc, type: iTy, value: -1, b, dynamicShapeRef: operand); |
| 710 | Value c23 = createIntConst(loc, type: iTy, value: mantissaWidth, b, dynamicShapeRef: operand); |
| 711 | Value c31 = createIntConst(loc, type: iTy, value: bitWidth - 1, b, dynamicShapeRef: operand); |
| 712 | Value c127 = |
| 713 | createIntConst(loc, type: iTy, value: (1ull << (exponentWidth - 1)) - 1, b, dynamicShapeRef: operand); |
| 714 | Value c2To22 = |
| 715 | createIntConst(loc, type: iTy, value: 1ull << (mantissaWidth - 1), b, dynamicShapeRef: operand); |
| 716 | Value c23Mask = |
| 717 | createIntConst(loc, type: iTy, value: (1ull << mantissaWidth) - 1, b, dynamicShapeRef: operand); |
| 718 | Value expMask = |
| 719 | createIntConst(loc, type: iTy, value: (1ull << exponentWidth) - 1, b, dynamicShapeRef: operand); |
| 720 | |
| 721 | Value operandBitcast = arith::BitcastOp::create(builder&: b, out: iTy, in: operand); |
| 722 | Value round = math::RoundOp::create(builder&: b, operand); |
| 723 | Value roundBitcast = arith::BitcastOp::create(builder&: b, out: iTy, in: round); |
| 724 | |
| 725 | // Get biased exponents for operand and round(operand) |
| 726 | Value operandExp = arith::AndIOp::create( |
| 727 | builder&: b, lhs: arith::ShRUIOp::create(builder&: b, lhs: operandBitcast, rhs: c23), rhs: expMask); |
| 728 | Value operandBiasedExp = arith::SubIOp::create(builder&: b, lhs: operandExp, rhs: c127); |
| 729 | Value roundExp = arith::AndIOp::create( |
| 730 | builder&: b, lhs: arith::ShRUIOp::create(builder&: b, lhs: roundBitcast, rhs: c23), rhs: expMask); |
| 731 | Value roundBiasedExp = arith::SubIOp::create(builder&: b, lhs: roundExp, rhs: c127); |
| 732 | |
| 733 | auto safeShiftRight = [&](Value x, Value shift) -> Value { |
| 734 | // Clamp shift to valid range [0, bitwidth - 1] to avoid undefined behavior |
| 735 | Value clampedShift = arith::MaxSIOp::create(builder&: b, lhs: shift, rhs: c0); |
| 736 | clampedShift = arith::MinSIOp::create(builder&: b, lhs: clampedShift, rhs: c31); |
| 737 | return arith::ShRUIOp::create(builder&: b, lhs: x, rhs: clampedShift); |
| 738 | }; |
| 739 | |
| 740 | auto maskMantissa = [&](Value mantissa, |
| 741 | Value mantissaMaskRightShift) -> Value { |
| 742 | Value shiftedMantissaMask = safeShiftRight(c23Mask, mantissaMaskRightShift); |
| 743 | return arith::AndIOp::create(builder&: b, lhs: mantissa, rhs: shiftedMantissaMask); |
| 744 | }; |
| 745 | |
| 746 | // A whole number `x`, such that `|x| != 1`, is even if the mantissa, ignoring |
| 747 | // the leftmost `clamp(biasedExp - 1, 0, 23)` bits, is zero. Large numbers |
| 748 | // with `biasedExp > 23` (numbers where there is not enough precision to store |
| 749 | // decimals) are always even, and they satisfy the even condition trivially |
| 750 | // since the mantissa without all its bits is zero. The even condition |
| 751 | // is also true for +-0, since they have `biasedExp = -127` and the entire |
| 752 | // mantissa is zero. The case of +-1 has to be handled separately. Here |
| 753 | // we identify these values by noting that +-1 are the only whole numbers with |
| 754 | // `biasedExp == 0`. |
| 755 | // |
| 756 | // The special values +-inf and +-nan also satisfy the same property that |
| 757 | // whole non-unit even numbers satisfy. In particular, the special values have |
| 758 | // `biasedExp > 23`, so they get treated as large numbers with no room for |
| 759 | // decimals, which are always even. |
| 760 | Value roundBiasedExpEq0 = |
| 761 | arith::CmpIOp::create(builder&: b, predicate: arith::CmpIPredicate::eq, lhs: roundBiasedExp, rhs: c0); |
| 762 | Value roundBiasedExpMinus1 = arith::SubIOp::create(builder&: b, lhs: roundBiasedExp, rhs: c1); |
| 763 | Value roundMaskedMantissa = maskMantissa(roundBitcast, roundBiasedExpMinus1); |
| 764 | Value roundIsNotEvenOrSpecialVal = arith::CmpIOp::create( |
| 765 | builder&: b, predicate: arith::CmpIPredicate::ne, lhs: roundMaskedMantissa, rhs: c0); |
| 766 | roundIsNotEvenOrSpecialVal = |
| 767 | arith::OrIOp::create(builder&: b, lhs: roundIsNotEvenOrSpecialVal, rhs: roundBiasedExpEq0); |
| 768 | |
| 769 | // A value `x` with `0 <= biasedExp < 23`, is halfway between two consecutive |
| 770 | // integers if the bit at index `biasedExp` starting from the left in the |
| 771 | // mantissa is 1 and all the bits to the right are zero. Values with |
| 772 | // `biasedExp >= 23` don't have decimals, so they are never halfway. The |
| 773 | // values +-0.5 are the only halfway values that have `biasedExp == -1 < 0`, |
| 774 | // so these are handled separately. In particular, if `biasedExp == -1`, the |
| 775 | // value is halfway if the entire mantissa is zero. |
| 776 | Value operandBiasedExpEqNeg1 = arith::CmpIOp::create( |
| 777 | builder&: b, predicate: arith::CmpIPredicate::eq, lhs: operandBiasedExp, rhs: cNeg1); |
| 778 | Value expectedOperandMaskedMantissa = arith::SelectOp::create( |
| 779 | builder&: b, condition: operandBiasedExpEqNeg1, true_value: c0, false_value: safeShiftRight(c2To22, operandBiasedExp)); |
| 780 | Value operandMaskedMantissa = maskMantissa(operandBitcast, operandBiasedExp); |
| 781 | Value operandIsHalfway = |
| 782 | arith::CmpIOp::create(builder&: b, predicate: arith::CmpIPredicate::eq, lhs: operandMaskedMantissa, |
| 783 | rhs: expectedOperandMaskedMantissa); |
| 784 | // Ensure `biasedExp` is in the valid range for half values. |
| 785 | Value operandBiasedExpGeNeg1 = arith::CmpIOp::create( |
| 786 | builder&: b, predicate: arith::CmpIPredicate::sge, lhs: operandBiasedExp, rhs: cNeg1); |
| 787 | Value operandBiasedExpLt23 = arith::CmpIOp::create( |
| 788 | builder&: b, predicate: arith::CmpIPredicate::slt, lhs: operandBiasedExp, rhs: c23); |
| 789 | operandIsHalfway = |
| 790 | arith::AndIOp::create(builder&: b, lhs: operandIsHalfway, rhs: operandBiasedExpLt23); |
| 791 | operandIsHalfway = |
| 792 | arith::AndIOp::create(builder&: b, lhs: operandIsHalfway, rhs: operandBiasedExpGeNeg1); |
| 793 | |
| 794 | // Adjust rounded operand with `round(operand) - sign(operand)` to correct the |
| 795 | // case where `round` rounded in the opposite direction of `roundeven`. |
| 796 | Value sign = math::CopySignOp::create(builder&: b, lhs: c1Float, rhs: operand); |
| 797 | Value roundShifted = arith::SubFOp::create(builder&: b, lhs: round, rhs: sign); |
| 798 | // If the rounded value is even or a special value, we default to the behavior |
| 799 | // of `math.round`. |
| 800 | Value needsShift = |
| 801 | arith::AndIOp::create(builder&: b, lhs: roundIsNotEvenOrSpecialVal, rhs: operandIsHalfway); |
| 802 | Value result = arith::SelectOp::create(builder&: b, condition: needsShift, true_value: roundShifted, false_value: round); |
| 803 | // The `x - sign` adjustment does not preserve the sign when we are adjusting |
| 804 | // the value -1 to -0. So here the sign is copied again to ensure that -0.5 is |
| 805 | // rounded to -0.0. |
| 806 | result = math::CopySignOp::create(builder&: b, lhs: result, rhs: operand); |
| 807 | rewriter.replaceOp(op, newValues: result); |
| 808 | return success(); |
| 809 | } |
| 810 | |
| 811 | // Convert `math.rsqrt` into `arith.divf` + `math.sqrt` |
| 812 | static LogicalResult convertRsqrtOp(math::RsqrtOp op, |
| 813 | PatternRewriter &rewriter) { |
| 814 | auto operand = op.getOperand(); |
| 815 | auto operandTy = operand.getType(); |
| 816 | |
| 817 | if (isUnrankedShaped(type: operandTy)) |
| 818 | return failure(); |
| 819 | |
| 820 | auto eTy = getElementTypeOrSelf(type: operandTy); |
| 821 | if (!isa<FloatType>(Val: eTy)) |
| 822 | return failure(); |
| 823 | |
| 824 | Location loc = op->getLoc(); |
| 825 | auto constOneFloat = createFloatConst(loc, type: operandTy, value: 1.0, b&: rewriter, dynamicShapeRef: operand); |
| 826 | auto sqrtOp = math::SqrtOp::create(builder&: rewriter, location: loc, operand); |
| 827 | rewriter.replaceOpWithNewOp<arith::DivFOp>(op, args&: constOneFloat, args&: sqrtOp); |
| 828 | return success(); |
| 829 | } |
| 830 | |
| 831 | // Convert `math.clampf` into `arith.minimumf` + `arith.maximumf` |
| 832 | static LogicalResult convertClampfOp(math::ClampFOp op, |
| 833 | PatternRewriter &rewriter) { |
| 834 | auto minOp = arith::MinimumFOp::create(builder&: rewriter, location: op.getLoc(), lhs: op.getValue(), |
| 835 | rhs: op.getMax(), fastmath: op.getFastmath()); |
| 836 | rewriter.replaceOpWithNewOp<arith::MaximumFOp>(op, args&: minOp, args: op.getMin(), |
| 837 | args: op.getFastmath()); |
| 838 | return success(); |
| 839 | } |
| 840 | |
| 841 | void mlir::math::populateExpansionPatterns(RewritePatternSet &patterns, |
| 842 | ArrayRef<StringRef> opMnemonics) { |
| 843 | auto filter = [&](StringRef name) { |
| 844 | // This should be a static assert and `consume_front` take a twine, but none |
| 845 | // is currently possible. TODO: augment `StringRef::consume_front` and make |
| 846 | // `getDialectNamespace` use `std::string_view`. |
| 847 | assert("math" == MathDialect::getDialectNamespace()); |
| 848 | name.consume_front(Prefix: "math." ); |
| 849 | return opMnemonics.empty() || (llvm::count(Range&: opMnemonics, Element: name) > 0); |
| 850 | }; |
| 851 | if (filter(CountLeadingZerosOp::getOperationName())) |
| 852 | patterns.add(implFn: convertCtlzOp); |
| 853 | if (filter(SinhOp::getOperationName())) |
| 854 | patterns.add(implFn: convertSinhOp); |
| 855 | if (filter(CoshOp::getOperationName())) |
| 856 | patterns.add(implFn: convertCoshOp); |
| 857 | if (filter(TanOp::getOperationName())) |
| 858 | patterns.add(implFn: convertTanOp); |
| 859 | if (filter(TanhOp::getOperationName())) |
| 860 | patterns.add(implFn: convertTanhOp); |
| 861 | if (filter(AsinhOp::getOperationName())) |
| 862 | patterns.add(implFn: convertAsinhOp); |
| 863 | if (filter(AcoshOp::getOperationName())) |
| 864 | patterns.add(implFn: convertAcoshOp); |
| 865 | if (filter(AtanhOp::getOperationName())) |
| 866 | patterns.add(implFn: convertAtanhOp); |
| 867 | if (filter(FmaOp::getOperationName())) |
| 868 | patterns.add(implFn: convertFmaFOp); |
| 869 | if (filter(CeilOp::getOperationName())) |
| 870 | patterns.add(implFn: convertCeilOp); |
| 871 | if (filter(Exp2Op::getOperationName())) |
| 872 | patterns.add(implFn: convertExp2fOp); |
| 873 | if (filter(PowFOp::getOperationName())) |
| 874 | patterns.add(implFn: convertPowfOp); |
| 875 | if (filter(FPowIOp::getOperationName())) |
| 876 | patterns.add(implFn: convertFPowIOp); |
| 877 | if (filter(RoundOp::getOperationName())) |
| 878 | patterns.add(implFn: convertRoundOp); |
| 879 | if (filter(RoundEvenOp::getOperationName())) |
| 880 | patterns.add(implFn: convertRoundEvenOp); |
| 881 | if (filter(RsqrtOp::getOperationName())) |
| 882 | patterns.add(implFn: convertRsqrtOp); |
| 883 | if (filter(ClampFOp::getOperationName())) |
| 884 | patterns.add(implFn: convertClampfOp); |
| 885 | } |
| 886 | |
| 887 | //===----------------------------------------------------------------------===// |
| 888 | // MathExpandOpsPass pass |
| 889 | //===----------------------------------------------------------------------===// |
| 890 | namespace { |
| 891 | struct MathExpandOpsPass final |
| 892 | : math::impl::MathExpandOpsPassBase<MathExpandOpsPass> { |
| 893 | using MathExpandOpsPassBase::MathExpandOpsPassBase; |
| 894 | |
| 895 | void runOnOperation() override { |
| 896 | RewritePatternSet patterns(&getContext()); |
| 897 | SmallVector<StringRef> mnemonics = |
| 898 | llvm::to_vector_of<StringRef>(Range&: opMnemonics); |
| 899 | math::populateExpansionPatterns(patterns, opMnemonics: mnemonics); |
| 900 | if (expandRoundF64 && |
| 901 | (mnemonics.empty() || llvm::count(Range&: mnemonics, Element: "round" ) > 0)) |
| 902 | patterns.add(implFn: convertRoundF64Op); |
| 903 | if (failed(Result: applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns)))) |
| 904 | return signalPassFailure(); |
| 905 | } |
| 906 | }; |
| 907 | } // namespace |
| 908 | |