| 1 | //===- ComplexToStandard.cpp - conversion from Complex to Standard dialect ===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/Conversion/ComplexToStandard/ComplexToStandard.h" |
| 10 | |
| 11 | #include "mlir/Conversion/ComplexCommon/DivisionConverter.h" |
| 12 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 13 | #include "mlir/Dialect/Complex/IR/Complex.h" |
| 14 | #include "mlir/Dialect/Math/IR/Math.h" |
| 15 | #include "mlir/IR/PatternMatch.h" |
| 16 | #include "mlir/Transforms/DialectConversion.h" |
| 17 | #include <type_traits> |
| 18 | |
| 19 | namespace mlir { |
| 20 | #define GEN_PASS_DEF_CONVERTCOMPLEXTOSTANDARDPASS |
| 21 | #include "mlir/Conversion/Passes.h.inc" |
| 22 | } // namespace mlir |
| 23 | |
| 24 | using namespace mlir; |
| 25 | |
| 26 | namespace { |
| 27 | |
| 28 | enum class AbsFn { abs, sqrt, rsqrt }; |
| 29 | |
| 30 | // Returns the absolute value, its square root or its reciprocal square root. |
| 31 | Value computeAbs(Value real, Value imag, arith::FastMathFlags fmf, |
| 32 | ImplicitLocOpBuilder &b, AbsFn fn = AbsFn::abs) { |
| 33 | Value one = arith::ConstantOp::create(builder&: b, result: real.getType(), |
| 34 | value: b.getFloatAttr(type: real.getType(), value: 1.0)); |
| 35 | |
| 36 | Value absReal = math::AbsFOp::create(builder&: b, operand: real, fastmath: fmf); |
| 37 | Value absImag = math::AbsFOp::create(builder&: b, operand: imag, fastmath: fmf); |
| 38 | |
| 39 | Value max = arith::MaximumFOp::create(builder&: b, lhs: absReal, rhs: absImag, fastmath: fmf); |
| 40 | Value min = arith::MinimumFOp::create(builder&: b, lhs: absReal, rhs: absImag, fastmath: fmf); |
| 41 | |
| 42 | // The lowering below requires NaNs and infinities to work correctly. |
| 43 | arith::FastMathFlags fmfWithNaNInf = arith::bitEnumClear( |
| 44 | bits: fmf, bit: arith::FastMathFlags::nnan | arith::FastMathFlags::ninf); |
| 45 | Value ratio = arith::DivFOp::create(builder&: b, lhs: min, rhs: max, fastmath: fmfWithNaNInf); |
| 46 | Value ratioSq = arith::MulFOp::create(builder&: b, lhs: ratio, rhs: ratio, fastmath: fmfWithNaNInf); |
| 47 | Value ratioSqPlusOne = arith::AddFOp::create(builder&: b, lhs: ratioSq, rhs: one, fastmath: fmfWithNaNInf); |
| 48 | Value result; |
| 49 | |
| 50 | if (fn == AbsFn::rsqrt) { |
| 51 | ratioSqPlusOne = math::RsqrtOp::create(builder&: b, operand: ratioSqPlusOne, fastmath: fmfWithNaNInf); |
| 52 | min = math::RsqrtOp::create(builder&: b, operand: min, fastmath: fmfWithNaNInf); |
| 53 | max = math::RsqrtOp::create(builder&: b, operand: max, fastmath: fmfWithNaNInf); |
| 54 | } |
| 55 | |
| 56 | if (fn == AbsFn::sqrt) { |
| 57 | Value quarter = arith::ConstantOp::create( |
| 58 | builder&: b, result: real.getType(), value: b.getFloatAttr(type: real.getType(), value: 0.25)); |
| 59 | // sqrt(sqrt(a*b)) would avoid the pow, but will overflow more easily. |
| 60 | Value sqrt = math::SqrtOp::create(builder&: b, operand: max, fastmath: fmfWithNaNInf); |
| 61 | Value p025 = |
| 62 | math::PowFOp::create(builder&: b, lhs: ratioSqPlusOne, rhs: quarter, fastmath: fmfWithNaNInf); |
| 63 | result = arith::MulFOp::create(builder&: b, lhs: sqrt, rhs: p025, fastmath: fmfWithNaNInf); |
| 64 | } else { |
| 65 | Value sqrt = math::SqrtOp::create(builder&: b, operand: ratioSqPlusOne, fastmath: fmfWithNaNInf); |
| 66 | result = arith::MulFOp::create(builder&: b, lhs: max, rhs: sqrt, fastmath: fmfWithNaNInf); |
| 67 | } |
| 68 | |
| 69 | Value isNaN = arith::CmpFOp::create(builder&: b, predicate: arith::CmpFPredicate::UNO, lhs: result, |
| 70 | rhs: result, fastmath: fmfWithNaNInf); |
| 71 | return arith::SelectOp::create(builder&: b, condition: isNaN, true_value: min, false_value: result); |
| 72 | } |
| 73 | |
| 74 | struct AbsOpConversion : public OpConversionPattern<complex::AbsOp> { |
| 75 | using OpConversionPattern<complex::AbsOp>::OpConversionPattern; |
| 76 | |
| 77 | LogicalResult |
| 78 | matchAndRewrite(complex::AbsOp op, OpAdaptor adaptor, |
| 79 | ConversionPatternRewriter &rewriter) const override { |
| 80 | ImplicitLocOpBuilder b(op.getLoc(), rewriter); |
| 81 | |
| 82 | arith::FastMathFlags fmf = op.getFastMathFlagsAttr().getValue(); |
| 83 | |
| 84 | Value real = complex::ReOp::create(builder&: b, complex: adaptor.getComplex()); |
| 85 | Value imag = complex::ImOp::create(builder&: b, complex: adaptor.getComplex()); |
| 86 | rewriter.replaceOp(op, newValues: computeAbs(real, imag, fmf, b)); |
| 87 | |
| 88 | return success(); |
| 89 | } |
| 90 | }; |
| 91 | |
| 92 | // atan2(y,x) = -i * log((x + i * y)/sqrt(x**2+y**2)) |
| 93 | struct Atan2OpConversion : public OpConversionPattern<complex::Atan2Op> { |
| 94 | using OpConversionPattern<complex::Atan2Op>::OpConversionPattern; |
| 95 | |
| 96 | LogicalResult |
| 97 | matchAndRewrite(complex::Atan2Op op, OpAdaptor adaptor, |
| 98 | ConversionPatternRewriter &rewriter) const override { |
| 99 | mlir::ImplicitLocOpBuilder b(op.getLoc(), rewriter); |
| 100 | |
| 101 | auto type = cast<ComplexType>(Val: op.getType()); |
| 102 | Type elementType = type.getElementType(); |
| 103 | arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr(); |
| 104 | |
| 105 | Value lhs = adaptor.getLhs(); |
| 106 | Value rhs = adaptor.getRhs(); |
| 107 | |
| 108 | Value rhsSquared = complex::MulOp::create(builder&: b, result: type, lhs: rhs, rhs, fastmath: fmf); |
| 109 | Value lhsSquared = complex::MulOp::create(builder&: b, result: type, lhs, rhs: lhs, fastmath: fmf); |
| 110 | Value rhsSquaredPlusLhsSquared = |
| 111 | complex::AddOp::create(builder&: b, result: type, lhs: rhsSquared, rhs: lhsSquared, fastmath: fmf); |
| 112 | Value sqrtOfRhsSquaredPlusLhsSquared = |
| 113 | complex::SqrtOp::create(builder&: b, result: type, complex: rhsSquaredPlusLhsSquared, fastmath: fmf); |
| 114 | |
| 115 | Value zero = |
| 116 | arith::ConstantOp::create(builder&: b, result: elementType, value: b.getZeroAttr(type: elementType)); |
| 117 | Value one = arith::ConstantOp::create(builder&: b, result: elementType, |
| 118 | value: b.getFloatAttr(type: elementType, value: 1)); |
| 119 | Value i = complex::CreateOp::create(builder&: b, complex: type, real: zero, imaginary: one); |
| 120 | Value iTimesLhs = complex::MulOp::create(builder&: b, lhs: i, rhs: lhs, fastmath: fmf); |
| 121 | Value rhsPlusILhs = complex::AddOp::create(builder&: b, lhs: rhs, rhs: iTimesLhs, fastmath: fmf); |
| 122 | |
| 123 | Value divResult = complex::DivOp::create( |
| 124 | builder&: b, lhs: rhsPlusILhs, rhs: sqrtOfRhsSquaredPlusLhsSquared, fastmath: fmf); |
| 125 | Value logResult = complex::LogOp::create(builder&: b, complex: divResult, fastmath: fmf); |
| 126 | |
| 127 | Value negativeOne = arith::ConstantOp::create( |
| 128 | builder&: b, result: elementType, value: b.getFloatAttr(type: elementType, value: -1)); |
| 129 | Value negativeI = complex::CreateOp::create(builder&: b, complex: type, real: zero, imaginary: negativeOne); |
| 130 | |
| 131 | rewriter.replaceOpWithNewOp<complex::MulOp>(op, args&: negativeI, args&: logResult, args&: fmf); |
| 132 | return success(); |
| 133 | } |
| 134 | }; |
| 135 | |
| 136 | template <typename ComparisonOp, arith::CmpFPredicate p> |
| 137 | struct ComparisonOpConversion : public OpConversionPattern<ComparisonOp> { |
| 138 | using OpConversionPattern<ComparisonOp>::OpConversionPattern; |
| 139 | using ResultCombiner = |
| 140 | std::conditional_t<std::is_same<ComparisonOp, complex::EqualOp>::value, |
| 141 | arith::AndIOp, arith::OrIOp>; |
| 142 | |
| 143 | LogicalResult |
| 144 | matchAndRewrite(ComparisonOp op, typename ComparisonOp::Adaptor adaptor, |
| 145 | ConversionPatternRewriter &rewriter) const override { |
| 146 | auto loc = op.getLoc(); |
| 147 | auto type = cast<ComplexType>(adaptor.getLhs().getType()).getElementType(); |
| 148 | |
| 149 | Value realLhs = |
| 150 | complex::ReOp::create(rewriter, loc, type, adaptor.getLhs()); |
| 151 | Value imagLhs = |
| 152 | complex::ImOp::create(rewriter, loc, type, adaptor.getLhs()); |
| 153 | Value realRhs = |
| 154 | complex::ReOp::create(rewriter, loc, type, adaptor.getRhs()); |
| 155 | Value imagRhs = |
| 156 | complex::ImOp::create(rewriter, loc, type, adaptor.getRhs()); |
| 157 | Value realComparison = |
| 158 | arith::CmpFOp::create(rewriter, loc, p, realLhs, realRhs); |
| 159 | Value imagComparison = |
| 160 | arith::CmpFOp::create(rewriter, loc, p, imagLhs, imagRhs); |
| 161 | |
| 162 | rewriter.replaceOpWithNewOp<ResultCombiner>(op, realComparison, |
| 163 | imagComparison); |
| 164 | return success(); |
| 165 | } |
| 166 | }; |
| 167 | |
| 168 | // Default conversion which applies the BinaryStandardOp separately on the real |
| 169 | // and imaginary parts. Can for example be used for complex::AddOp and |
| 170 | // complex::SubOp. |
| 171 | template <typename BinaryComplexOp, typename BinaryStandardOp> |
| 172 | struct BinaryComplexOpConversion : public OpConversionPattern<BinaryComplexOp> { |
| 173 | using OpConversionPattern<BinaryComplexOp>::OpConversionPattern; |
| 174 | |
| 175 | LogicalResult |
| 176 | matchAndRewrite(BinaryComplexOp op, typename BinaryComplexOp::Adaptor adaptor, |
| 177 | ConversionPatternRewriter &rewriter) const override { |
| 178 | auto type = cast<ComplexType>(adaptor.getLhs().getType()); |
| 179 | auto elementType = cast<FloatType>(type.getElementType()); |
| 180 | mlir::ImplicitLocOpBuilder b(op.getLoc(), rewriter); |
| 181 | arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr(); |
| 182 | |
| 183 | Value realLhs = complex::ReOp::create(b, elementType, adaptor.getLhs()); |
| 184 | Value realRhs = complex::ReOp::create(b, elementType, adaptor.getRhs()); |
| 185 | Value resultReal = BinaryStandardOp::create(b, elementType, realLhs, |
| 186 | realRhs, fmf.getValue()); |
| 187 | Value imagLhs = complex::ImOp::create(b, elementType, adaptor.getLhs()); |
| 188 | Value imagRhs = complex::ImOp::create(b, elementType, adaptor.getRhs()); |
| 189 | Value resultImag = BinaryStandardOp::create(b, elementType, imagLhs, |
| 190 | imagRhs, fmf.getValue()); |
| 191 | rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, resultReal, |
| 192 | resultImag); |
| 193 | return success(); |
| 194 | } |
| 195 | }; |
| 196 | |
| 197 | template <typename TrigonometricOp> |
| 198 | struct TrigonometricOpConversion : public OpConversionPattern<TrigonometricOp> { |
| 199 | using OpAdaptor = typename OpConversionPattern<TrigonometricOp>::OpAdaptor; |
| 200 | |
| 201 | using OpConversionPattern<TrigonometricOp>::OpConversionPattern; |
| 202 | |
| 203 | LogicalResult |
| 204 | matchAndRewrite(TrigonometricOp op, OpAdaptor adaptor, |
| 205 | ConversionPatternRewriter &rewriter) const override { |
| 206 | auto loc = op.getLoc(); |
| 207 | auto type = cast<ComplexType>(adaptor.getComplex().getType()); |
| 208 | auto elementType = cast<FloatType>(type.getElementType()); |
| 209 | arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr(); |
| 210 | |
| 211 | Value real = |
| 212 | complex::ReOp::create(rewriter, loc, elementType, adaptor.getComplex()); |
| 213 | Value imag = |
| 214 | complex::ImOp::create(rewriter, loc, elementType, adaptor.getComplex()); |
| 215 | |
| 216 | // Trigonometric ops use a set of common building blocks to convert to real |
| 217 | // ops. Here we create these building blocks and call into an op-specific |
| 218 | // implementation in the subclass to combine them. |
| 219 | Value half = arith::ConstantOp::create( |
| 220 | rewriter, loc, elementType, rewriter.getFloatAttr(elementType, 0.5)); |
| 221 | Value exp = math::ExpOp::create(rewriter, loc, imag, fmf); |
| 222 | Value scaledExp = arith::MulFOp::create(rewriter, loc, half, exp, fmf); |
| 223 | Value reciprocalExp = arith::DivFOp::create(rewriter, loc, half, exp, fmf); |
| 224 | Value sin = math::SinOp::create(rewriter, loc, real, fmf); |
| 225 | Value cos = math::CosOp::create(rewriter, loc, real, fmf); |
| 226 | |
| 227 | auto resultPair = |
| 228 | combine(loc, scaledExp, reciprocalExp, sin, cos, rewriter, fmf); |
| 229 | |
| 230 | rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, resultPair.first, |
| 231 | resultPair.second); |
| 232 | return success(); |
| 233 | } |
| 234 | |
| 235 | virtual std::pair<Value, Value> |
| 236 | combine(Location loc, Value scaledExp, Value reciprocalExp, Value sin, |
| 237 | Value cos, ConversionPatternRewriter &rewriter, |
| 238 | arith::FastMathFlagsAttr fmf) const = 0; |
| 239 | }; |
| 240 | |
| 241 | struct CosOpConversion : public TrigonometricOpConversion<complex::CosOp> { |
| 242 | using TrigonometricOpConversion<complex::CosOp>::TrigonometricOpConversion; |
| 243 | |
| 244 | std::pair<Value, Value> combine(Location loc, Value scaledExp, |
| 245 | Value reciprocalExp, Value sin, Value cos, |
| 246 | ConversionPatternRewriter &rewriter, |
| 247 | arith::FastMathFlagsAttr fmf) const override { |
| 248 | // Complex cosine is defined as; |
| 249 | // cos(x + iy) = 0.5 * (exp(i(x + iy)) + exp(-i(x + iy))) |
| 250 | // Plugging in: |
| 251 | // exp(i(x+iy)) = exp(-y + ix) = exp(-y)(cos(x) + i sin(x)) |
| 252 | // exp(-i(x+iy)) = exp(y + i(-x)) = exp(y)(cos(x) + i (-sin(x))) |
| 253 | // and defining t := exp(y) |
| 254 | // We get: |
| 255 | // Re(cos(x + iy)) = (0.5/t + 0.5*t) * cos x |
| 256 | // Im(cos(x + iy)) = (0.5/t - 0.5*t) * sin x |
| 257 | Value sum = |
| 258 | arith::AddFOp::create(builder&: rewriter, location: loc, lhs: reciprocalExp, rhs: scaledExp, fastmath: fmf); |
| 259 | Value resultReal = arith::MulFOp::create(builder&: rewriter, location: loc, lhs: sum, rhs: cos, fastmath: fmf); |
| 260 | Value diff = |
| 261 | arith::SubFOp::create(builder&: rewriter, location: loc, lhs: reciprocalExp, rhs: scaledExp, fastmath: fmf); |
| 262 | Value resultImag = arith::MulFOp::create(builder&: rewriter, location: loc, lhs: diff, rhs: sin, fastmath: fmf); |
| 263 | return {resultReal, resultImag}; |
| 264 | } |
| 265 | }; |
| 266 | |
| 267 | struct DivOpConversion : public OpConversionPattern<complex::DivOp> { |
| 268 | DivOpConversion(MLIRContext *context, complex::ComplexRangeFlags target) |
| 269 | : OpConversionPattern<complex::DivOp>(context), complexRange(target) {} |
| 270 | |
| 271 | using OpConversionPattern<complex::DivOp>::OpConversionPattern; |
| 272 | |
| 273 | LogicalResult |
| 274 | matchAndRewrite(complex::DivOp op, OpAdaptor adaptor, |
| 275 | ConversionPatternRewriter &rewriter) const override { |
| 276 | auto loc = op.getLoc(); |
| 277 | auto type = cast<ComplexType>(Val: adaptor.getLhs().getType()); |
| 278 | auto elementType = cast<FloatType>(Val: type.getElementType()); |
| 279 | arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr(); |
| 280 | |
| 281 | Value lhsReal = |
| 282 | complex::ReOp::create(builder&: rewriter, location: loc, real: elementType, complex: adaptor.getLhs()); |
| 283 | Value lhsImag = |
| 284 | complex::ImOp::create(builder&: rewriter, location: loc, imaginary: elementType, complex: adaptor.getLhs()); |
| 285 | Value rhsReal = |
| 286 | complex::ReOp::create(builder&: rewriter, location: loc, real: elementType, complex: adaptor.getRhs()); |
| 287 | Value rhsImag = |
| 288 | complex::ImOp::create(builder&: rewriter, location: loc, imaginary: elementType, complex: adaptor.getRhs()); |
| 289 | |
| 290 | Value resultReal, resultImag; |
| 291 | |
| 292 | if (complexRange == complex::ComplexRangeFlags::basic || |
| 293 | complexRange == complex::ComplexRangeFlags::none) { |
| 294 | mlir::complex::convertDivToStandardUsingAlgebraic( |
| 295 | rewriter, loc, lhsRe: lhsReal, lhsIm: lhsImag, rhsRe: rhsReal, rhsIm: rhsImag, fmf, resultRe: &resultReal, |
| 296 | resultIm: &resultImag); |
| 297 | } else if (complexRange == complex::ComplexRangeFlags::improved) { |
| 298 | mlir::complex::convertDivToStandardUsingRangeReduction( |
| 299 | rewriter, loc, lhsRe: lhsReal, lhsIm: lhsImag, rhsRe: rhsReal, rhsIm: rhsImag, fmf, resultRe: &resultReal, |
| 300 | resultIm: &resultImag); |
| 301 | } |
| 302 | |
| 303 | rewriter.replaceOpWithNewOp<complex::CreateOp>(op, args&: type, args&: resultReal, |
| 304 | args&: resultImag); |
| 305 | |
| 306 | return success(); |
| 307 | } |
| 308 | |
| 309 | private: |
| 310 | complex::ComplexRangeFlags complexRange; |
| 311 | }; |
| 312 | |
| 313 | struct ExpOpConversion : public OpConversionPattern<complex::ExpOp> { |
| 314 | using OpConversionPattern<complex::ExpOp>::OpConversionPattern; |
| 315 | |
| 316 | // exp(x+I*y) = exp(x)*(cos(y)+I*sin(y)) |
| 317 | // Handle special cases as StableHLO implementation does: |
| 318 | // 1. When y == +-0, set imag(exp(z)) = y |
| 319 | // 2. When exp(x) == inf, use exp(x/2)*(cos(y)+I*sin(y))*exp(x/2) |
| 320 | // Handle non-finite inputs according to C99 annex G: |
| 321 | // 3. When x == -inf and y is non-finite, return 0 + I*0. |
| 322 | // 4. When x == +inf and y is non-finite, return inf + I*NaN. |
| 323 | LogicalResult |
| 324 | matchAndRewrite(complex::ExpOp op, OpAdaptor adaptor, |
| 325 | ConversionPatternRewriter &rewriter) const override { |
| 326 | auto loc = op.getLoc(); |
| 327 | auto type = cast<ComplexType>(Val: adaptor.getComplex().getType()); |
| 328 | auto ET = cast<FloatType>(Val: type.getElementType()); |
| 329 | arith::FastMathFlags fmf = op.getFastMathFlagsAttr().getValue(); |
| 330 | const auto &floatSemantics = ET.getFloatSemantics(); |
| 331 | ImplicitLocOpBuilder b(loc, rewriter); |
| 332 | |
| 333 | Value x = complex::ReOp::create(builder&: b, real: ET, complex: adaptor.getComplex()); |
| 334 | Value y = complex::ImOp::create(builder&: b, imaginary: ET, complex: adaptor.getComplex()); |
| 335 | Value zero = arith::ConstantOp::create(builder&: b, result: ET, value: b.getZeroAttr(type: ET)); |
| 336 | Value half = arith::ConstantOp::create(builder&: b, result: ET, value: b.getFloatAttr(type: ET, value: 0.5)); |
| 337 | Value inf = arith::ConstantOp::create( |
| 338 | builder&: b, result: ET, value: b.getFloatAttr(type: ET, value: APFloat::getInf(Sem: floatSemantics))); |
| 339 | |
| 340 | // Intermediate exp may overflow. |
| 341 | arith::FastMathFlags fmfWithInf = |
| 342 | arith::bitEnumClear(bits: fmf, bit: arith::FastMathFlags::ninf); |
| 343 | Value exp = math::ExpOp::create(builder&: b, operand: x, fastmath: fmfWithInf); |
| 344 | Value xHalf = arith::MulFOp::create(builder&: b, lhs: x, rhs: half, fastmath: fmf); |
| 345 | Value expHalf = math::ExpOp::create(builder&: b, operand: xHalf, fastmath: fmf); |
| 346 | Value cos = math::CosOp::create(builder&: b, operand: y, fastmath: fmf); |
| 347 | Value sin = math::SinOp::create(builder&: b, operand: y, fastmath: fmf); |
| 348 | |
| 349 | Value expIsInf = arith::CmpFOp::create(builder&: b, predicate: arith::CmpFPredicate::OEQ, lhs: exp, |
| 350 | rhs: inf, fastmath: fmfWithInf); |
| 351 | Value yIsZero = |
| 352 | arith::CmpFOp::create(builder&: b, predicate: arith::CmpFPredicate::OEQ, lhs: y, rhs: zero); |
| 353 | |
| 354 | // Real path: select between exp(x)*cos(y) and exp(x/2)*cos(y)*exp(x/2) |
| 355 | Value realNormal = arith::MulFOp::create(builder&: b, lhs: exp, rhs: cos, fastmath: fmf); |
| 356 | Value expHalfCos = arith::MulFOp::create(builder&: b, lhs: expHalf, rhs: cos, fastmath: fmf); |
| 357 | Value realOverflow = arith::MulFOp::create(builder&: b, lhs: expHalfCos, rhs: expHalf, fastmath: fmf); |
| 358 | Value resultReal = |
| 359 | arith::SelectOp::create(builder&: b, condition: expIsInf, true_value: realOverflow, false_value: realNormal); |
| 360 | |
| 361 | // Imaginary part: if y == +-0 return +-0 else select between exp(x)*sin(y) |
| 362 | // and exp(x/2)*sin(y)*exp(x/2) |
| 363 | Value imagNormal = arith::MulFOp::create(builder&: b, lhs: exp, rhs: sin, fastmath: fmf); |
| 364 | Value expHalfSin = arith::MulFOp::create(builder&: b, lhs: expHalf, rhs: sin, fastmath: fmf); |
| 365 | Value imagOverflow = arith::MulFOp::create(builder&: b, lhs: expHalfSin, rhs: expHalf, fastmath: fmf); |
| 366 | Value imagNonZero = |
| 367 | arith::SelectOp::create(builder&: b, condition: expIsInf, true_value: imagOverflow, false_value: imagNormal); |
| 368 | Value resultImag = arith::SelectOp::create(builder&: b, condition: yIsZero, true_value: y, false_value: imagNonZero); |
| 369 | |
| 370 | // When x is +-inf and y is +-inf or NaN, exp(x) * cos(y) is inf * NaN or |
| 371 | // 0 * NaN. C99 Annex G gives exp(-inf + I*y) = +-0 +- 0*I and |
| 372 | // exp(+inf + I*y) = +-inf + NaN*I for these y. |
| 373 | if (!arith::bitEnumContainsAll(bits: fmf, bit: arith::FastMathFlags::ninf)) { |
| 374 | Value absX = math::AbsFOp::create(builder&: b, operand: x); |
| 375 | Value absY = math::AbsFOp::create(builder&: b, operand: y); |
| 376 | Value yIsNotFinite = |
| 377 | arith::CmpFOp::create(builder&: b, predicate: arith::CmpFPredicate::UGE, lhs: absY, rhs: inf); |
| 378 | Value xIsInf = |
| 379 | arith::CmpFOp::create(builder&: b, predicate: arith::CmpFPredicate::OEQ, lhs: absX, rhs: inf); |
| 380 | Value xIsNeg = |
| 381 | arith::CmpFOp::create(builder&: b, predicate: arith::CmpFPredicate::OLT, lhs: x, rhs: zero); |
| 382 | Value useExpX = arith::AndIOp::create(builder&: b, lhs: xIsInf, rhs: yIsNotFinite); |
| 383 | Value useZeroImag = arith::AndIOp::create(builder&: b, lhs: useExpX, rhs: xIsNeg); |
| 384 | resultReal = arith::SelectOp::create(builder&: b, condition: useExpX, true_value: exp, false_value: resultReal); |
| 385 | resultImag = arith::SelectOp::create(builder&: b, condition: useZeroImag, true_value: zero, false_value: resultImag); |
| 386 | } |
| 387 | |
| 388 | rewriter.replaceOpWithNewOp<complex::CreateOp>(op, args&: type, args&: resultReal, |
| 389 | args&: resultImag); |
| 390 | return success(); |
| 391 | } |
| 392 | }; |
| 393 | |
| 394 | Value evaluatePolynomial(ImplicitLocOpBuilder &b, Value arg, |
| 395 | ArrayRef<double> coefficients, |
| 396 | arith::FastMathFlagsAttr fmf) { |
| 397 | auto argType = mlir::cast<FloatType>(Val: arg.getType()); |
| 398 | Value poly = |
| 399 | arith::ConstantOp::create(builder&: b, value: b.getFloatAttr(type: argType, value: coefficients[0])); |
| 400 | for (unsigned i = 1; i < coefficients.size(); ++i) { |
| 401 | poly = math::FmaOp::create( |
| 402 | builder&: b, a: poly, b: arg, |
| 403 | c: arith::ConstantOp::create(builder&: b, value: b.getFloatAttr(type: argType, value: coefficients[i])), |
| 404 | fastmath: fmf); |
| 405 | } |
| 406 | return poly; |
| 407 | } |
| 408 | |
| 409 | struct Expm1OpConversion : public OpConversionPattern<complex::Expm1Op> { |
| 410 | using OpConversionPattern<complex::Expm1Op>::OpConversionPattern; |
| 411 | |
| 412 | // e^(a+bi)-1 = (e^a*cos(b)-1)+e^a*sin(b)i |
| 413 | // [handle inaccuracies when a and/or b are small] |
| 414 | // = ((e^a - 1) * cos(b) + cos(b) - 1) + e^a*sin(b)i |
| 415 | // = (expm1(a) * cos(b) + cosm1(b)) + e^a*sin(b)i |
| 416 | LogicalResult |
| 417 | matchAndRewrite(complex::Expm1Op op, OpAdaptor adaptor, |
| 418 | ConversionPatternRewriter &rewriter) const override { |
| 419 | auto type = op.getType(); |
| 420 | auto elemType = mlir::cast<FloatType>(Val: type.getElementType()); |
| 421 | |
| 422 | arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr(); |
| 423 | ImplicitLocOpBuilder b(op.getLoc(), rewriter); |
| 424 | Value real = complex::ReOp::create(builder&: b, complex: adaptor.getComplex()); |
| 425 | Value imag = complex::ImOp::create(builder&: b, complex: adaptor.getComplex()); |
| 426 | |
| 427 | Value zero = arith::ConstantOp::create(builder&: b, value: b.getFloatAttr(type: elemType, value: 0.0)); |
| 428 | Value one = arith::ConstantOp::create(builder&: b, value: b.getFloatAttr(type: elemType, value: 1.0)); |
| 429 | |
| 430 | Value expm1Real = math::ExpM1Op::create(builder&: b, operand: real, fastmath: fmf); |
| 431 | Value expReal = arith::AddFOp::create(builder&: b, lhs: expm1Real, rhs: one, fastmath: fmf); |
| 432 | |
| 433 | Value sinImag = math::SinOp::create(builder&: b, operand: imag, fastmath: fmf); |
| 434 | Value cosm1Imag = emitCosm1(arg: imag, fmf, b); |
| 435 | Value cosImag = arith::AddFOp::create(builder&: b, lhs: cosm1Imag, rhs: one, fastmath: fmf); |
| 436 | |
| 437 | Value realResult = arith::AddFOp::create( |
| 438 | builder&: b, lhs: arith::MulFOp::create(builder&: b, lhs: expm1Real, rhs: cosImag, fastmath: fmf), rhs: cosm1Imag, fastmath: fmf); |
| 439 | |
| 440 | Value imagIsZero = arith::CmpFOp::create(builder&: b, predicate: arith::CmpFPredicate::OEQ, lhs: imag, |
| 441 | rhs: zero, fastmath: fmf.getValue()); |
| 442 | Value imagResult = arith::SelectOp::create( |
| 443 | builder&: b, condition: imagIsZero, true_value: zero, false_value: arith::MulFOp::create(builder&: b, lhs: expReal, rhs: sinImag, fastmath: fmf)); |
| 444 | |
| 445 | rewriter.replaceOpWithNewOp<complex::CreateOp>(op, args&: type, args&: realResult, |
| 446 | args&: imagResult); |
| 447 | return success(); |
| 448 | } |
| 449 | |
| 450 | private: |
| 451 | Value emitCosm1(Value arg, arith::FastMathFlagsAttr fmf, |
| 452 | ImplicitLocOpBuilder &b) const { |
| 453 | auto argType = mlir::cast<FloatType>(Val: arg.getType()); |
| 454 | auto negHalf = arith::ConstantOp::create(builder&: b, value: b.getFloatAttr(type: argType, value: -0.5)); |
| 455 | auto one = arith::ConstantOp::create(builder&: b, value: b.getFloatAttr(type: argType, value: 1.0)); |
| 456 | |
| 457 | // Algorithm copied from cephes cosm1. |
| 458 | SmallVector<double, 7> kCoeffs{ |
| 459 | 4.7377507964246204691685E-14, -1.1470284843425359765671E-11, |
| 460 | 2.0876754287081521758361E-9, -2.7557319214999787979814E-7, |
| 461 | 2.4801587301570552304991E-5, -1.3888888888888872993737E-3, |
| 462 | 4.1666666666666666609054E-2, |
| 463 | }; |
| 464 | Value cos = math::CosOp::create(builder&: b, operand: arg, fastmath: fmf); |
| 465 | Value forLargeArg = arith::SubFOp::create(builder&: b, lhs: cos, rhs: one, fastmath: fmf); |
| 466 | |
| 467 | Value argPow2 = arith::MulFOp::create(builder&: b, lhs: arg, rhs: arg, fastmath: fmf); |
| 468 | Value argPow4 = arith::MulFOp::create(builder&: b, lhs: argPow2, rhs: argPow2, fastmath: fmf); |
| 469 | Value poly = evaluatePolynomial(b, arg: argPow2, coefficients: kCoeffs, fmf); |
| 470 | |
| 471 | auto forSmallArg = |
| 472 | arith::AddFOp::create(builder&: b, lhs: arith::MulFOp::create(builder&: b, lhs: argPow4, rhs: poly, fastmath: fmf), |
| 473 | rhs: arith::MulFOp::create(builder&: b, lhs: negHalf, rhs: argPow2, fastmath: fmf)); |
| 474 | |
| 475 | // (pi/4)^2 is approximately 0.61685 |
| 476 | Value piOver4Pow2 = |
| 477 | arith::ConstantOp::create(builder&: b, value: b.getFloatAttr(type: argType, value: 0.61685)); |
| 478 | Value cond = arith::CmpFOp::create(builder&: b, predicate: arith::CmpFPredicate::OGE, lhs: argPow2, |
| 479 | rhs: piOver4Pow2, fastmath: fmf.getValue()); |
| 480 | return arith::SelectOp::create(builder&: b, condition: cond, true_value: forLargeArg, false_value: forSmallArg); |
| 481 | } |
| 482 | }; |
| 483 | |
| 484 | struct LogOpConversion : public OpConversionPattern<complex::LogOp> { |
| 485 | using OpConversionPattern<complex::LogOp>::OpConversionPattern; |
| 486 | |
| 487 | LogicalResult |
| 488 | matchAndRewrite(complex::LogOp op, OpAdaptor adaptor, |
| 489 | ConversionPatternRewriter &rewriter) const override { |
| 490 | auto type = cast<ComplexType>(Val: adaptor.getComplex().getType()); |
| 491 | auto elementType = cast<FloatType>(Val: type.getElementType()); |
| 492 | arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr(); |
| 493 | mlir::ImplicitLocOpBuilder b(op.getLoc(), rewriter); |
| 494 | |
| 495 | Value abs = complex::AbsOp::create(builder&: b, result: elementType, complex: adaptor.getComplex(), |
| 496 | fastmath: fmf.getValue()); |
| 497 | Value resultReal = math::LogOp::create(builder&: b, result: elementType, operand: abs, fastmath: fmf.getValue()); |
| 498 | Value real = complex::ReOp::create(builder&: b, real: elementType, complex: adaptor.getComplex()); |
| 499 | Value imag = complex::ImOp::create(builder&: b, imaginary: elementType, complex: adaptor.getComplex()); |
| 500 | Value resultImag = |
| 501 | math::Atan2Op::create(builder&: b, result: elementType, lhs: imag, rhs: real, fastmath: fmf.getValue()); |
| 502 | rewriter.replaceOpWithNewOp<complex::CreateOp>(op, args&: type, args&: resultReal, |
| 503 | args&: resultImag); |
| 504 | return success(); |
| 505 | } |
| 506 | }; |
| 507 | |
| 508 | struct Log1pOpConversion : public OpConversionPattern<complex::Log1pOp> { |
| 509 | using OpConversionPattern<complex::Log1pOp>::OpConversionPattern; |
| 510 | |
| 511 | LogicalResult |
| 512 | matchAndRewrite(complex::Log1pOp op, OpAdaptor adaptor, |
| 513 | ConversionPatternRewriter &rewriter) const override { |
| 514 | auto type = cast<ComplexType>(Val: adaptor.getComplex().getType()); |
| 515 | auto elementType = cast<FloatType>(Val: type.getElementType()); |
| 516 | arith::FastMathFlags fmf = op.getFastMathFlagsAttr().getValue(); |
| 517 | mlir::ImplicitLocOpBuilder b(op.getLoc(), rewriter); |
| 518 | |
| 519 | Value real = complex::ReOp::create(builder&: b, complex: adaptor.getComplex()); |
| 520 | Value imag = complex::ImOp::create(builder&: b, complex: adaptor.getComplex()); |
| 521 | |
| 522 | Value half = arith::ConstantOp::create(builder&: b, result: elementType, |
| 523 | value: b.getFloatAttr(type: elementType, value: 0.5)); |
| 524 | Value one = arith::ConstantOp::create(builder&: b, result: elementType, |
| 525 | value: b.getFloatAttr(type: elementType, value: 1)); |
| 526 | Value realPlusOne = arith::AddFOp::create(builder&: b, lhs: real, rhs: one, fastmath: fmf); |
| 527 | Value absRealPlusOne = math::AbsFOp::create(builder&: b, operand: realPlusOne, fastmath: fmf); |
| 528 | Value absImag = math::AbsFOp::create(builder&: b, operand: imag, fastmath: fmf); |
| 529 | |
| 530 | Value maxAbs = arith::MaximumFOp::create(builder&: b, lhs: absRealPlusOne, rhs: absImag, fastmath: fmf); |
| 531 | Value minAbs = arith::MinimumFOp::create(builder&: b, lhs: absRealPlusOne, rhs: absImag, fastmath: fmf); |
| 532 | |
| 533 | Value useReal = arith::CmpFOp::create(builder&: b, predicate: arith::CmpFPredicate::OGT, |
| 534 | lhs: realPlusOne, rhs: absImag, fastmath: fmf); |
| 535 | Value maxMinusOne = arith::SubFOp::create(builder&: b, lhs: maxAbs, rhs: one, fastmath: fmf); |
| 536 | Value maxAbsOfRealPlusOneAndImagMinusOne = |
| 537 | arith::SelectOp::create(builder&: b, condition: useReal, true_value: real, false_value: maxMinusOne); |
| 538 | arith::FastMathFlags fmfWithNaNInf = arith::bitEnumClear( |
| 539 | bits: fmf, bit: arith::FastMathFlags::nnan | arith::FastMathFlags::ninf); |
| 540 | Value minMaxRatio = arith::DivFOp::create(builder&: b, lhs: minAbs, rhs: maxAbs, fastmath: fmfWithNaNInf); |
| 541 | Value logOfMaxAbsOfRealPlusOneAndImag = |
| 542 | math::Log1pOp::create(builder&: b, operand: maxAbsOfRealPlusOneAndImagMinusOne, fastmath: fmf); |
| 543 | Value logOfSqrtPart = math::Log1pOp::create( |
| 544 | builder&: b, operand: arith::MulFOp::create(builder&: b, lhs: minMaxRatio, rhs: minMaxRatio, fastmath: fmfWithNaNInf), |
| 545 | fastmath: fmfWithNaNInf); |
| 546 | Value r = arith::AddFOp::create( |
| 547 | builder&: b, lhs: arith::MulFOp::create(builder&: b, lhs: half, rhs: logOfSqrtPart, fastmath: fmfWithNaNInf), |
| 548 | rhs: logOfMaxAbsOfRealPlusOneAndImag, fastmath: fmfWithNaNInf); |
| 549 | Value resultReal = arith::SelectOp::create( |
| 550 | builder&: b, |
| 551 | condition: arith::CmpFOp::create(builder&: b, predicate: arith::CmpFPredicate::UNO, lhs: r, rhs: r, |
| 552 | fastmath: fmfWithNaNInf), |
| 553 | true_value: minAbs, false_value: r); |
| 554 | Value resultImag = math::Atan2Op::create(builder&: b, lhs: imag, rhs: realPlusOne, fastmath: fmf); |
| 555 | rewriter.replaceOpWithNewOp<complex::CreateOp>(op, args&: type, args&: resultReal, |
| 556 | args&: resultImag); |
| 557 | return success(); |
| 558 | } |
| 559 | }; |
| 560 | |
| 561 | struct MulOpConversion : public OpConversionPattern<complex::MulOp> { |
| 562 | using OpConversionPattern<complex::MulOp>::OpConversionPattern; |
| 563 | |
| 564 | LogicalResult |
| 565 | matchAndRewrite(complex::MulOp op, OpAdaptor adaptor, |
| 566 | ConversionPatternRewriter &rewriter) const override { |
| 567 | mlir::ImplicitLocOpBuilder b(op.getLoc(), rewriter); |
| 568 | auto type = cast<ComplexType>(Val: adaptor.getLhs().getType()); |
| 569 | auto elementType = cast<FloatType>(Val: type.getElementType()); |
| 570 | arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr(); |
| 571 | auto fmfValue = fmf.getValue(); |
| 572 | Value lhsReal = complex::ReOp::create(builder&: b, real: elementType, complex: adaptor.getLhs()); |
| 573 | Value lhsImag = complex::ImOp::create(builder&: b, imaginary: elementType, complex: adaptor.getLhs()); |
| 574 | Value rhsReal = complex::ReOp::create(builder&: b, real: elementType, complex: adaptor.getRhs()); |
| 575 | Value rhsImag = complex::ImOp::create(builder&: b, imaginary: elementType, complex: adaptor.getRhs()); |
| 576 | Value real; |
| 577 | Value imag; |
| 578 | if (arith::bitEnumContainsAll(bits: fmfValue, bit: arith::FastMathFlags::contract)) { |
| 579 | Value lhsImagTimesRhsImag = |
| 580 | arith::MulFOp::create(builder&: b, lhs: lhsImag, rhs: rhsImag, fastmath: fmfValue); |
| 581 | Value negLhsImagTimesRhsImag = |
| 582 | arith::NegFOp::create(builder&: b, operand: lhsImagTimesRhsImag, fastmath: fmfValue); |
| 583 | real = math::FmaOp::create(builder&: b, a: lhsReal, b: rhsReal, c: negLhsImagTimesRhsImag, |
| 584 | fastmath: fmfValue); |
| 585 | |
| 586 | Value lhsImagTimesRhsReal = |
| 587 | arith::MulFOp::create(builder&: b, lhs: lhsImag, rhs: rhsReal, fastmath: fmfValue); |
| 588 | imag = math::FmaOp::create(builder&: b, a: lhsReal, b: rhsImag, c: lhsImagTimesRhsReal, |
| 589 | fastmath: fmfValue); |
| 590 | } else { |
| 591 | Value lhsRealTimesRhsReal = |
| 592 | arith::MulFOp::create(builder&: b, lhs: lhsReal, rhs: rhsReal, fastmath: fmfValue); |
| 593 | Value lhsImagTimesRhsImag = |
| 594 | arith::MulFOp::create(builder&: b, lhs: lhsImag, rhs: rhsImag, fastmath: fmfValue); |
| 595 | Value lhsImagTimesRhsReal = |
| 596 | arith::MulFOp::create(builder&: b, lhs: lhsImag, rhs: rhsReal, fastmath: fmfValue); |
| 597 | Value lhsRealTimesRhsImag = |
| 598 | arith::MulFOp::create(builder&: b, lhs: lhsReal, rhs: rhsImag, fastmath: fmfValue); |
| 599 | |
| 600 | real = arith::SubFOp::create(builder&: b, lhs: lhsRealTimesRhsReal, rhs: lhsImagTimesRhsImag, |
| 601 | fastmath: fmfValue); |
| 602 | imag = arith::AddFOp::create(builder&: b, lhs: lhsImagTimesRhsReal, rhs: lhsRealTimesRhsImag, |
| 603 | fastmath: fmfValue); |
| 604 | } |
| 605 | rewriter.replaceOpWithNewOp<complex::CreateOp>(op, args&: type, args&: real, args&: imag); |
| 606 | return success(); |
| 607 | } |
| 608 | }; |
| 609 | |
| 610 | struct NegOpConversion : public OpConversionPattern<complex::NegOp> { |
| 611 | using OpConversionPattern<complex::NegOp>::OpConversionPattern; |
| 612 | |
| 613 | LogicalResult |
| 614 | matchAndRewrite(complex::NegOp op, OpAdaptor adaptor, |
| 615 | ConversionPatternRewriter &rewriter) const override { |
| 616 | auto loc = op.getLoc(); |
| 617 | auto type = cast<ComplexType>(Val: adaptor.getComplex().getType()); |
| 618 | auto elementType = cast<FloatType>(Val: type.getElementType()); |
| 619 | arith::FastMathFlags fmf = op.getFastMathFlagsAttr().getValue(); |
| 620 | |
| 621 | Value real = |
| 622 | complex::ReOp::create(builder&: rewriter, location: loc, real: elementType, complex: adaptor.getComplex()); |
| 623 | Value imag = |
| 624 | complex::ImOp::create(builder&: rewriter, location: loc, imaginary: elementType, complex: adaptor.getComplex()); |
| 625 | Value negReal = arith::NegFOp::create(builder&: rewriter, location: loc, operand: real, fastmath: fmf); |
| 626 | Value negImag = arith::NegFOp::create(builder&: rewriter, location: loc, operand: imag, fastmath: fmf); |
| 627 | rewriter.replaceOpWithNewOp<complex::CreateOp>(op, args&: type, args&: negReal, args&: negImag); |
| 628 | return success(); |
| 629 | } |
| 630 | }; |
| 631 | |
| 632 | struct SinOpConversion : public TrigonometricOpConversion<complex::SinOp> { |
| 633 | using TrigonometricOpConversion<complex::SinOp>::TrigonometricOpConversion; |
| 634 | |
| 635 | std::pair<Value, Value> combine(Location loc, Value scaledExp, |
| 636 | Value reciprocalExp, Value sin, Value cos, |
| 637 | ConversionPatternRewriter &rewriter, |
| 638 | arith::FastMathFlagsAttr fmf) const override { |
| 639 | // Complex sine is defined as; |
| 640 | // sin(x + iy) = -0.5i * (exp(i(x + iy)) - exp(-i(x + iy))) |
| 641 | // Plugging in: |
| 642 | // exp(i(x+iy)) = exp(-y + ix) = exp(-y)(cos(x) + i sin(x)) |
| 643 | // exp(-i(x+iy)) = exp(y + i(-x)) = exp(y)(cos(x) + i (-sin(x))) |
| 644 | // and defining t := exp(y) |
| 645 | // We get: |
| 646 | // Re(sin(x + iy)) = (0.5*t + 0.5/t) * sin x |
| 647 | // Im(sin(x + iy)) = (0.5*t - 0.5/t) * cos x |
| 648 | Value sum = |
| 649 | arith::AddFOp::create(builder&: rewriter, location: loc, lhs: scaledExp, rhs: reciprocalExp, fastmath: fmf); |
| 650 | Value resultReal = arith::MulFOp::create(builder&: rewriter, location: loc, lhs: sum, rhs: sin, fastmath: fmf); |
| 651 | Value diff = |
| 652 | arith::SubFOp::create(builder&: rewriter, location: loc, lhs: scaledExp, rhs: reciprocalExp, fastmath: fmf); |
| 653 | Value resultImag = arith::MulFOp::create(builder&: rewriter, location: loc, lhs: diff, rhs: cos, fastmath: fmf); |
| 654 | return {resultReal, resultImag}; |
| 655 | } |
| 656 | }; |
| 657 | |
| 658 | // The algorithm is listed in https://dl.acm.org/doi/pdf/10.1145/363717.363780. |
| 659 | struct SqrtOpConversion : public OpConversionPattern<complex::SqrtOp> { |
| 660 | using OpConversionPattern<complex::SqrtOp>::OpConversionPattern; |
| 661 | |
| 662 | LogicalResult |
| 663 | matchAndRewrite(complex::SqrtOp op, OpAdaptor adaptor, |
| 664 | ConversionPatternRewriter &rewriter) const override { |
| 665 | ImplicitLocOpBuilder b(op.getLoc(), rewriter); |
| 666 | |
| 667 | auto type = cast<ComplexType>(Val: op.getType()); |
| 668 | auto elementType = cast<FloatType>(Val: type.getElementType()); |
| 669 | arith::FastMathFlags fmf = op.getFastMathFlagsAttr().getValue(); |
| 670 | |
| 671 | auto cst = [&](APFloat v) { |
| 672 | return arith::ConstantOp::create(builder&: b, result: elementType, |
| 673 | value: b.getFloatAttr(type: elementType, value: v)); |
| 674 | }; |
| 675 | const auto &floatSemantics = elementType.getFloatSemantics(); |
| 676 | Value zero = cst(APFloat::getZero(Sem: floatSemantics)); |
| 677 | Value half = arith::ConstantOp::create(builder&: b, result: elementType, |
| 678 | value: b.getFloatAttr(type: elementType, value: 0.5)); |
| 679 | |
| 680 | Value real = complex::ReOp::create(builder&: b, real: elementType, complex: adaptor.getComplex()); |
| 681 | Value imag = complex::ImOp::create(builder&: b, imaginary: elementType, complex: adaptor.getComplex()); |
| 682 | Value absSqrt = computeAbs(real, imag, fmf, b, fn: AbsFn::sqrt); |
| 683 | Value argArg = math::Atan2Op::create(builder&: b, lhs: imag, rhs: real, fastmath: fmf); |
| 684 | Value sqrtArg = arith::MulFOp::create(builder&: b, lhs: argArg, rhs: half, fastmath: fmf); |
| 685 | Value cos = math::CosOp::create(builder&: b, operand: sqrtArg, fastmath: fmf); |
| 686 | Value sin = math::SinOp::create(builder&: b, operand: sqrtArg, fastmath: fmf); |
| 687 | // sin(atan2(0, inf)) = 0, sqrt(abs(inf)) = inf, but we can't multiply |
| 688 | // 0 * inf. |
| 689 | Value sinIsZero = |
| 690 | arith::CmpFOp::create(builder&: b, predicate: arith::CmpFPredicate::OEQ, lhs: sin, rhs: zero, fastmath: fmf); |
| 691 | |
| 692 | Value resultReal = arith::MulFOp::create(builder&: b, lhs: absSqrt, rhs: cos, fastmath: fmf); |
| 693 | Value resultImag = arith::SelectOp::create( |
| 694 | builder&: b, condition: sinIsZero, true_value: zero, false_value: arith::MulFOp::create(builder&: b, lhs: absSqrt, rhs: sin, fastmath: fmf)); |
| 695 | if (!arith::bitEnumContainsAll(bits: fmf, bit: arith::FastMathFlags::nnan | |
| 696 | arith::FastMathFlags::ninf)) { |
| 697 | Value inf = cst(APFloat::getInf(Sem: floatSemantics)); |
| 698 | Value negInf = cst(APFloat::getInf(Sem: floatSemantics, Negative: true)); |
| 699 | Value nan = cst(APFloat::getNaN(Sem: floatSemantics)); |
| 700 | Value absImag = math::AbsFOp::create(builder&: b, result: elementType, operand: imag, fastmath: fmf); |
| 701 | |
| 702 | Value absImagIsInf = arith::CmpFOp::create(builder&: b, predicate: arith::CmpFPredicate::OEQ, |
| 703 | lhs: absImag, rhs: inf, fastmath: fmf); |
| 704 | Value absImagIsNotInf = arith::CmpFOp::create( |
| 705 | builder&: b, predicate: arith::CmpFPredicate::ONE, lhs: absImag, rhs: inf, fastmath: fmf); |
| 706 | Value realIsInf = |
| 707 | arith::CmpFOp::create(builder&: b, predicate: arith::CmpFPredicate::OEQ, lhs: real, rhs: inf, fastmath: fmf); |
| 708 | Value realIsNegInf = arith::CmpFOp::create(builder&: b, predicate: arith::CmpFPredicate::OEQ, |
| 709 | lhs: real, rhs: negInf, fastmath: fmf); |
| 710 | |
| 711 | resultReal = arith::SelectOp::create( |
| 712 | builder&: b, condition: arith::AndIOp::create(builder&: b, lhs: realIsNegInf, rhs: absImagIsNotInf), true_value: zero, |
| 713 | false_value: resultReal); |
| 714 | resultReal = arith::SelectOp::create( |
| 715 | builder&: b, condition: arith::OrIOp::create(builder&: b, lhs: absImagIsInf, rhs: realIsInf), true_value: inf, false_value: resultReal); |
| 716 | |
| 717 | Value imagSignInf = math::CopySignOp::create(builder&: b, lhs: inf, rhs: imag, fastmath: fmf); |
| 718 | resultImag = arith::SelectOp::create( |
| 719 | builder&: b, |
| 720 | condition: arith::CmpFOp::create(builder&: b, predicate: arith::CmpFPredicate::UNO, lhs: absSqrt, rhs: absSqrt), |
| 721 | true_value: nan, false_value: resultImag); |
| 722 | resultImag = arith::SelectOp::create( |
| 723 | builder&: b, condition: arith::OrIOp::create(builder&: b, lhs: absImagIsInf, rhs: realIsNegInf), true_value: imagSignInf, |
| 724 | false_value: resultImag); |
| 725 | } |
| 726 | |
| 727 | Value resultIsZero = |
| 728 | arith::CmpFOp::create(builder&: b, predicate: arith::CmpFPredicate::OEQ, lhs: absSqrt, rhs: zero, fastmath: fmf); |
| 729 | resultReal = arith::SelectOp::create(builder&: b, condition: resultIsZero, true_value: zero, false_value: resultReal); |
| 730 | resultImag = arith::SelectOp::create(builder&: b, condition: resultIsZero, true_value: zero, false_value: resultImag); |
| 731 | |
| 732 | rewriter.replaceOpWithNewOp<complex::CreateOp>(op, args&: type, args&: resultReal, |
| 733 | args&: resultImag); |
| 734 | return success(); |
| 735 | } |
| 736 | }; |
| 737 | |
| 738 | struct SignOpConversion : public OpConversionPattern<complex::SignOp> { |
| 739 | using OpConversionPattern<complex::SignOp>::OpConversionPattern; |
| 740 | |
| 741 | LogicalResult |
| 742 | matchAndRewrite(complex::SignOp op, OpAdaptor adaptor, |
| 743 | ConversionPatternRewriter &rewriter) const override { |
| 744 | auto type = cast<ComplexType>(Val: adaptor.getComplex().getType()); |
| 745 | auto elementType = cast<FloatType>(Val: type.getElementType()); |
| 746 | mlir::ImplicitLocOpBuilder b(op.getLoc(), rewriter); |
| 747 | arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr(); |
| 748 | |
| 749 | Value real = complex::ReOp::create(builder&: b, real: elementType, complex: adaptor.getComplex()); |
| 750 | Value imag = complex::ImOp::create(builder&: b, imaginary: elementType, complex: adaptor.getComplex()); |
| 751 | Value zero = |
| 752 | arith::ConstantOp::create(builder&: b, result: elementType, value: b.getZeroAttr(type: elementType)); |
| 753 | Value realIsZero = |
| 754 | arith::CmpFOp::create(builder&: b, predicate: arith::CmpFPredicate::OEQ, lhs: real, rhs: zero); |
| 755 | Value imagIsZero = |
| 756 | arith::CmpFOp::create(builder&: b, predicate: arith::CmpFPredicate::OEQ, lhs: imag, rhs: zero); |
| 757 | Value isZero = arith::AndIOp::create(builder&: b, lhs: realIsZero, rhs: imagIsZero); |
| 758 | auto abs = |
| 759 | complex::AbsOp::create(builder&: b, result: elementType, complex: adaptor.getComplex(), fastmath: fmf); |
| 760 | Value realSign = arith::DivFOp::create(builder&: b, lhs: real, rhs: abs, fastmath: fmf); |
| 761 | Value imagSign = arith::DivFOp::create(builder&: b, lhs: imag, rhs: abs, fastmath: fmf); |
| 762 | Value sign = complex::CreateOp::create(builder&: b, complex: type, real: realSign, imaginary: imagSign); |
| 763 | rewriter.replaceOpWithNewOp<arith::SelectOp>(op, args&: isZero, |
| 764 | args: adaptor.getComplex(), args&: sign); |
| 765 | return success(); |
| 766 | } |
| 767 | }; |
| 768 | |
| 769 | template <typename Op> |
| 770 | struct TanTanhOpConversion : public OpConversionPattern<Op> { |
| 771 | using OpConversionPattern<Op>::OpConversionPattern; |
| 772 | |
| 773 | LogicalResult |
| 774 | matchAndRewrite(Op op, typename Op::Adaptor adaptor, |
| 775 | ConversionPatternRewriter &rewriter) const override { |
| 776 | ImplicitLocOpBuilder b(op.getLoc(), rewriter); |
| 777 | auto loc = op.getLoc(); |
| 778 | auto type = cast<ComplexType>(adaptor.getComplex().getType()); |
| 779 | auto elementType = cast<FloatType>(type.getElementType()); |
| 780 | arith::FastMathFlags fmf = op.getFastMathFlagsAttr().getValue(); |
| 781 | const auto &floatSemantics = elementType.getFloatSemantics(); |
| 782 | |
| 783 | Value real = |
| 784 | complex::ReOp::create(b, loc, elementType, adaptor.getComplex()); |
| 785 | Value imag = |
| 786 | complex::ImOp::create(b, loc, elementType, adaptor.getComplex()); |
| 787 | |
| 788 | if constexpr (std::is_same_v<Op, complex::TanOp>) { |
| 789 | // tan(x+yi) = -i*tanh(-y + xi) |
| 790 | std::swap(a&: real, b&: imag); |
| 791 | real = arith::NegFOp::create(builder&: b, operand: real, fastmath: fmf); |
| 792 | } |
| 793 | |
| 794 | auto cst = [&](APFloat v) { |
| 795 | return arith::ConstantOp::create(b, elementType, |
| 796 | b.getFloatAttr(elementType, v)); |
| 797 | }; |
| 798 | Value inf = cst(APFloat::getInf(Sem: floatSemantics)); |
| 799 | Value four = arith::ConstantOp::create(b, elementType, |
| 800 | b.getFloatAttr(elementType, 4.0)); |
| 801 | Value twoReal = arith::AddFOp::create(builder&: b, lhs: real, rhs: real, fastmath: fmf); |
| 802 | Value negTwoReal = arith::NegFOp::create(builder&: b, operand: twoReal, fastmath: fmf); |
| 803 | Value expTwoRealMinusOne = math::ExpM1Op::create(builder&: b, operand: twoReal, fastmath: fmf); |
| 804 | Value expNegTwoRealMinusOne = math::ExpM1Op::create(builder&: b, operand: negTwoReal, fastmath: fmf); |
| 805 | Value realNum = arith::SubFOp::create(builder&: b, lhs: expTwoRealMinusOne, |
| 806 | rhs: expNegTwoRealMinusOne, fastmath: fmf); |
| 807 | Value expProduct = arith::MulFOp::create(builder&: b, lhs: expTwoRealMinusOne, |
| 808 | rhs: expNegTwoRealMinusOne, fastmath: fmf); |
| 809 | Value expSumMinusTwo = arith::NegFOp::create(builder&: b, operand: expProduct, fastmath: fmf); |
| 810 | |
| 811 | Value cosImag = math::CosOp::create(builder&: b, operand: imag, fastmath: fmf); |
| 812 | Value cosImagSq = arith::MulFOp::create(builder&: b, lhs: cosImag, rhs: cosImag, fastmath: fmf); |
| 813 | Value twoCosTwoImagPlusOne = arith::MulFOp::create(builder&: b, lhs: cosImagSq, rhs: four, fastmath: fmf); |
| 814 | Value sinImag = math::SinOp::create(builder&: b, operand: imag, fastmath: fmf); |
| 815 | |
| 816 | Value imagNum = arith::MulFOp::create( |
| 817 | builder&: b, lhs: four, rhs: arith::MulFOp::create(builder&: b, lhs: cosImag, rhs: sinImag, fastmath: fmf), fastmath: fmf); |
| 818 | |
| 819 | Value denom = |
| 820 | arith::AddFOp::create(builder&: b, lhs: expSumMinusTwo, rhs: twoCosTwoImagPlusOne, fastmath: fmf); |
| 821 | |
| 822 | Value isInf = arith::CmpFOp::create(builder&: b, predicate: arith::CmpFPredicate::OEQ, |
| 823 | lhs: expSumMinusTwo, rhs: inf, fastmath: fmf); |
| 824 | Value negOne = arith::ConstantOp::create(b, elementType, |
| 825 | b.getFloatAttr(elementType, -1.0)); |
| 826 | Value realLimit = math::CopySignOp::create(builder&: b, lhs: negOne, rhs: real, fastmath: fmf); |
| 827 | |
| 828 | Value resultReal = arith::SelectOp::create( |
| 829 | builder&: b, condition: isInf, true_value: realLimit, false_value: arith::DivFOp::create(builder&: b, lhs: realNum, rhs: denom, fastmath: fmf)); |
| 830 | Value resultImag = arith::DivFOp::create(builder&: b, lhs: imagNum, rhs: denom, fastmath: fmf); |
| 831 | |
| 832 | if (!arith::bitEnumContainsAll(bits: fmf, bit: arith::FastMathFlags::nnan | |
| 833 | arith::FastMathFlags::ninf)) { |
| 834 | Value absReal = math::AbsFOp::create(builder&: b, operand: real, fastmath: fmf); |
| 835 | Value zero = arith::ConstantOp::create(b, elementType, |
| 836 | b.getFloatAttr(elementType, 0.0)); |
| 837 | Value nan = cst(APFloat::getNaN(Sem: floatSemantics)); |
| 838 | |
| 839 | Value absRealIsInf = arith::CmpFOp::create(builder&: b, predicate: arith::CmpFPredicate::OEQ, |
| 840 | lhs: absReal, rhs: inf, fastmath: fmf); |
| 841 | Value imagIsZero = |
| 842 | arith::CmpFOp::create(builder&: b, predicate: arith::CmpFPredicate::OEQ, lhs: imag, rhs: zero, fastmath: fmf); |
| 843 | Value absRealIsNotInf = arith::XOrIOp::create( |
| 844 | builder&: b, lhs: absRealIsInf, rhs: arith::ConstantIntOp::create(builder&: b, value: true, /*width=*/1)); |
| 845 | |
| 846 | Value imagNumIsNaN = arith::CmpFOp::create(builder&: b, predicate: arith::CmpFPredicate::UNO, |
| 847 | lhs: imagNum, rhs: imagNum, fastmath: fmf); |
| 848 | Value resultRealIsNaN = |
| 849 | arith::AndIOp::create(builder&: b, lhs: imagNumIsNaN, rhs: absRealIsNotInf); |
| 850 | Value resultImagIsZero = arith::OrIOp::create( |
| 851 | builder&: b, lhs: imagIsZero, rhs: arith::AndIOp::create(builder&: b, lhs: absRealIsInf, rhs: imagNumIsNaN)); |
| 852 | |
| 853 | resultReal = arith::SelectOp::create(builder&: b, condition: resultRealIsNaN, true_value: nan, false_value: resultReal); |
| 854 | resultImag = |
| 855 | arith::SelectOp::create(builder&: b, condition: resultImagIsZero, true_value: zero, false_value: resultImag); |
| 856 | } |
| 857 | |
| 858 | if constexpr (std::is_same_v<Op, complex::TanOp>) { |
| 859 | // tan(x+yi) = -i*tanh(-y + xi) |
| 860 | std::swap(a&: resultReal, b&: resultImag); |
| 861 | resultImag = arith::NegFOp::create(builder&: b, operand: resultImag, fastmath: fmf); |
| 862 | } |
| 863 | |
| 864 | rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, resultReal, |
| 865 | resultImag); |
| 866 | return success(); |
| 867 | } |
| 868 | }; |
| 869 | |
| 870 | struct ConjOpConversion : public OpConversionPattern<complex::ConjOp> { |
| 871 | using OpConversionPattern<complex::ConjOp>::OpConversionPattern; |
| 872 | |
| 873 | LogicalResult |
| 874 | matchAndRewrite(complex::ConjOp op, OpAdaptor adaptor, |
| 875 | ConversionPatternRewriter &rewriter) const override { |
| 876 | auto loc = op.getLoc(); |
| 877 | auto type = cast<ComplexType>(Val: adaptor.getComplex().getType()); |
| 878 | auto elementType = cast<FloatType>(Val: type.getElementType()); |
| 879 | arith::FastMathFlags fmf = op.getFastMathFlagsAttr().getValue(); |
| 880 | Value real = |
| 881 | complex::ReOp::create(builder&: rewriter, location: loc, real: elementType, complex: adaptor.getComplex()); |
| 882 | Value imag = |
| 883 | complex::ImOp::create(builder&: rewriter, location: loc, imaginary: elementType, complex: adaptor.getComplex()); |
| 884 | Value negImag = |
| 885 | arith::NegFOp::create(builder&: rewriter, location: loc, result: elementType, operand: imag, fastmath: fmf); |
| 886 | |
| 887 | rewriter.replaceOpWithNewOp<complex::CreateOp>(op, args&: type, args&: real, args&: negImag); |
| 888 | |
| 889 | return success(); |
| 890 | } |
| 891 | }; |
| 892 | |
| 893 | /// Converts lhs^y = (a+bi)^(c+di) to |
| 894 | /// (a*a+b*b)^(0.5c) * exp(-d*atan2(b,a)) * (cos(q) + i*sin(q)), |
| 895 | /// where q = c*atan2(b,a)+0.5d*ln(a*a+b*b) |
| 896 | static Value powOpConversionImpl(mlir::ImplicitLocOpBuilder &builder, |
| 897 | ComplexType type, Value lhs, Value c, Value d, |
| 898 | arith::FastMathFlags fmf) { |
| 899 | auto elementType = cast<FloatType>(Val: type.getElementType()); |
| 900 | |
| 901 | Value a = complex::ReOp::create(builder, complex: lhs); |
| 902 | Value b = complex::ImOp::create(builder, complex: lhs); |
| 903 | |
| 904 | Value abs = complex::AbsOp::create(builder, complex: lhs, fastmath: fmf); |
| 905 | Value absToC = math::PowFOp::create(builder, lhs: abs, rhs: c, fastmath: fmf); |
| 906 | |
| 907 | Value negD = arith::NegFOp::create(builder, operand: d, fastmath: fmf); |
| 908 | Value argLhs = math::Atan2Op::create(builder, lhs: b, rhs: a, fastmath: fmf); |
| 909 | Value negDArgLhs = arith::MulFOp::create(builder, lhs: negD, rhs: argLhs, fastmath: fmf); |
| 910 | Value expNegDArgLhs = math::ExpOp::create(builder, operand: negDArgLhs, fastmath: fmf); |
| 911 | |
| 912 | Value coeff = arith::MulFOp::create(builder, lhs: absToC, rhs: expNegDArgLhs, fastmath: fmf); |
| 913 | Value lnAbs = math::LogOp::create(builder, operand: abs, fastmath: fmf); |
| 914 | Value cArgLhs = arith::MulFOp::create(builder, lhs: c, rhs: argLhs, fastmath: fmf); |
| 915 | Value dLnAbs = arith::MulFOp::create(builder, lhs: d, rhs: lnAbs, fastmath: fmf); |
| 916 | Value q = arith::AddFOp::create(builder, lhs: cArgLhs, rhs: dLnAbs, fastmath: fmf); |
| 917 | Value cosQ = math::CosOp::create(builder, operand: q, fastmath: fmf); |
| 918 | Value sinQ = math::SinOp::create(builder, operand: q, fastmath: fmf); |
| 919 | |
| 920 | Value inf = arith::ConstantOp::create( |
| 921 | builder, result: elementType, |
| 922 | value: builder.getFloatAttr(type: elementType, |
| 923 | value: APFloat::getInf(Sem: elementType.getFloatSemantics()))); |
| 924 | Value zero = arith::ConstantOp::create( |
| 925 | builder, result: elementType, value: builder.getFloatAttr(type: elementType, value: 0.0)); |
| 926 | Value one = arith::ConstantOp::create(builder, result: elementType, |
| 927 | value: builder.getFloatAttr(type: elementType, value: 1.0)); |
| 928 | Value complexOne = complex::CreateOp::create(builder, complex: type, real: one, imaginary: zero); |
| 929 | Value complexZero = complex::CreateOp::create(builder, complex: type, real: zero, imaginary: zero); |
| 930 | Value complexInf = complex::CreateOp::create(builder, complex: type, real: inf, imaginary: zero); |
| 931 | |
| 932 | // Case 0: |
| 933 | // d^c is 0 if d is 0 and c > 0. 0^0 is defined to be 1.0, see |
| 934 | // Branch Cuts for Complex Elementary Functions or Much Ado About |
| 935 | // Nothing's Sign Bit, W. Kahan, Section 10. |
| 936 | Value absEqZero = |
| 937 | arith::CmpFOp::create(builder, predicate: arith::CmpFPredicate::OEQ, lhs: abs, rhs: zero, fastmath: fmf); |
| 938 | Value dEqZero = |
| 939 | arith::CmpFOp::create(builder, predicate: arith::CmpFPredicate::OEQ, lhs: d, rhs: zero, fastmath: fmf); |
| 940 | Value cEqZero = |
| 941 | arith::CmpFOp::create(builder, predicate: arith::CmpFPredicate::OEQ, lhs: c, rhs: zero, fastmath: fmf); |
| 942 | Value bEqZero = |
| 943 | arith::CmpFOp::create(builder, predicate: arith::CmpFPredicate::OEQ, lhs: b, rhs: zero, fastmath: fmf); |
| 944 | |
| 945 | Value zeroLeC = |
| 946 | arith::CmpFOp::create(builder, predicate: arith::CmpFPredicate::OLE, lhs: zero, rhs: c, fastmath: fmf); |
| 947 | Value coeffCosQ = arith::MulFOp::create(builder, lhs: coeff, rhs: cosQ, fastmath: fmf); |
| 948 | Value coeffSinQ = arith::MulFOp::create(builder, lhs: coeff, rhs: sinQ, fastmath: fmf); |
| 949 | Value complexOneOrZero = |
| 950 | arith::SelectOp::create(builder, condition: cEqZero, true_value: complexOne, false_value: complexZero); |
| 951 | Value coeffCosSin = |
| 952 | complex::CreateOp::create(builder, complex: type, real: coeffCosQ, imaginary: coeffSinQ); |
| 953 | Value cutoff0 = arith::SelectOp::create( |
| 954 | builder, |
| 955 | condition: arith::AndIOp::create( |
| 956 | builder, lhs: arith::AndIOp::create(builder, lhs: absEqZero, rhs: dEqZero), rhs: zeroLeC), |
| 957 | true_value: complexOneOrZero, false_value: coeffCosSin); |
| 958 | |
| 959 | // Case 1: |
| 960 | // x^0 is defined to be 1 for any x, see |
| 961 | // Branch Cuts for Complex Elementary Functions or Much Ado About |
| 962 | // Nothing's Sign Bit, W. Kahan, Section 10. |
| 963 | Value rhsEqZero = arith::AndIOp::create(builder, lhs: cEqZero, rhs: dEqZero); |
| 964 | Value cutoff1 = |
| 965 | arith::SelectOp::create(builder, condition: rhsEqZero, true_value: complexOne, false_value: cutoff0); |
| 966 | |
| 967 | // Case 2: |
| 968 | // 1^(c + d*i) = 1 + 0*i |
| 969 | Value lhsEqOne = arith::AndIOp::create( |
| 970 | builder, |
| 971 | lhs: arith::CmpFOp::create(builder, predicate: arith::CmpFPredicate::OEQ, lhs: a, rhs: one, fastmath: fmf), |
| 972 | rhs: bEqZero); |
| 973 | Value cutoff2 = |
| 974 | arith::SelectOp::create(builder, condition: lhsEqOne, true_value: complexOne, false_value: cutoff1); |
| 975 | |
| 976 | // Case 3: |
| 977 | // inf^(c + 0*i) = inf + 0*i, c > 0 |
| 978 | Value lhsEqInf = arith::AndIOp::create( |
| 979 | builder, |
| 980 | lhs: arith::CmpFOp::create(builder, predicate: arith::CmpFPredicate::OEQ, lhs: a, rhs: inf, fastmath: fmf), |
| 981 | rhs: bEqZero); |
| 982 | Value rhsGt0 = arith::AndIOp::create( |
| 983 | builder, lhs: dEqZero, |
| 984 | rhs: arith::CmpFOp::create(builder, predicate: arith::CmpFPredicate::OGT, lhs: c, rhs: zero, fastmath: fmf)); |
| 985 | Value cutoff3 = arith::SelectOp::create( |
| 986 | builder, condition: arith::AndIOp::create(builder, lhs: lhsEqInf, rhs: rhsGt0), true_value: complexInf, |
| 987 | false_value: cutoff2); |
| 988 | |
| 989 | // Case 4: |
| 990 | // inf^(c + 0*i) = 0 + 0*i, c < 0 |
| 991 | Value rhsLt0 = arith::AndIOp::create( |
| 992 | builder, lhs: dEqZero, |
| 993 | rhs: arith::CmpFOp::create(builder, predicate: arith::CmpFPredicate::OLT, lhs: c, rhs: zero, fastmath: fmf)); |
| 994 | Value cutoff4 = arith::SelectOp::create( |
| 995 | builder, condition: arith::AndIOp::create(builder, lhs: lhsEqInf, rhs: rhsLt0), true_value: complexZero, |
| 996 | false_value: cutoff3); |
| 997 | |
| 998 | return cutoff4; |
| 999 | } |
| 1000 | |
| 1001 | struct PowiOpConversion : public OpConversionPattern<complex::PowiOp> { |
| 1002 | using OpConversionPattern<complex::PowiOp>::OpConversionPattern; |
| 1003 | |
| 1004 | LogicalResult |
| 1005 | matchAndRewrite(complex::PowiOp op, OpAdaptor adaptor, |
| 1006 | ConversionPatternRewriter &rewriter) const override { |
| 1007 | ImplicitLocOpBuilder builder(op.getLoc(), rewriter); |
| 1008 | auto type = cast<ComplexType>(Val: op.getType()); |
| 1009 | auto elementType = cast<FloatType>(Val: type.getElementType()); |
| 1010 | |
| 1011 | Value floatExponent = |
| 1012 | arith::SIToFPOp::create(builder, out: elementType, in: adaptor.getRhs()); |
| 1013 | Value zero = arith::ConstantOp::create( |
| 1014 | builder, result: elementType, value: builder.getFloatAttr(type: elementType, value: 0.0)); |
| 1015 | Value complexExponent = |
| 1016 | complex::CreateOp::create(builder, complex: type, real: floatExponent, imaginary: zero); |
| 1017 | |
| 1018 | auto pow = complex::PowOp::create(builder, result: type, lhs: adaptor.getLhs(), |
| 1019 | rhs: complexExponent, fastmath: op.getFastmathAttr()); |
| 1020 | rewriter.replaceOp(op, newValues: pow.getResult()); |
| 1021 | return success(); |
| 1022 | } |
| 1023 | }; |
| 1024 | |
| 1025 | struct PowOpConversion : public OpConversionPattern<complex::PowOp> { |
| 1026 | using OpConversionPattern<complex::PowOp>::OpConversionPattern; |
| 1027 | |
| 1028 | LogicalResult |
| 1029 | matchAndRewrite(complex::PowOp op, OpAdaptor adaptor, |
| 1030 | ConversionPatternRewriter &rewriter) const override { |
| 1031 | mlir::ImplicitLocOpBuilder builder(op.getLoc(), rewriter); |
| 1032 | auto type = cast<ComplexType>(Val: adaptor.getLhs().getType()); |
| 1033 | auto elementType = cast<FloatType>(Val: type.getElementType()); |
| 1034 | |
| 1035 | Value c = complex::ReOp::create(builder, real: elementType, complex: adaptor.getRhs()); |
| 1036 | Value d = complex::ImOp::create(builder, imaginary: elementType, complex: adaptor.getRhs()); |
| 1037 | |
| 1038 | rewriter.replaceOp(op, newValues: {powOpConversionImpl(builder, type, lhs: adaptor.getLhs(), |
| 1039 | c, d, fmf: op.getFastmath())}); |
| 1040 | return success(); |
| 1041 | } |
| 1042 | }; |
| 1043 | |
| 1044 | struct RsqrtOpConversion : public OpConversionPattern<complex::RsqrtOp> { |
| 1045 | using OpConversionPattern<complex::RsqrtOp>::OpConversionPattern; |
| 1046 | |
| 1047 | LogicalResult |
| 1048 | matchAndRewrite(complex::RsqrtOp op, OpAdaptor adaptor, |
| 1049 | ConversionPatternRewriter &rewriter) const override { |
| 1050 | mlir::ImplicitLocOpBuilder b(op.getLoc(), rewriter); |
| 1051 | auto type = cast<ComplexType>(Val: adaptor.getComplex().getType()); |
| 1052 | auto elementType = cast<FloatType>(Val: type.getElementType()); |
| 1053 | |
| 1054 | arith::FastMathFlags fmf = op.getFastMathFlagsAttr().getValue(); |
| 1055 | |
| 1056 | auto cst = [&](APFloat v) { |
| 1057 | return arith::ConstantOp::create(builder&: b, result: elementType, |
| 1058 | value: b.getFloatAttr(type: elementType, value: v)); |
| 1059 | }; |
| 1060 | const auto &floatSemantics = elementType.getFloatSemantics(); |
| 1061 | Value zero = cst(APFloat::getZero(Sem: floatSemantics)); |
| 1062 | Value inf = cst(APFloat::getInf(Sem: floatSemantics)); |
| 1063 | Value negHalf = arith::ConstantOp::create( |
| 1064 | builder&: b, result: elementType, value: b.getFloatAttr(type: elementType, value: -0.5)); |
| 1065 | Value nan = cst(APFloat::getNaN(Sem: floatSemantics)); |
| 1066 | |
| 1067 | Value real = complex::ReOp::create(builder&: b, real: elementType, complex: adaptor.getComplex()); |
| 1068 | Value imag = complex::ImOp::create(builder&: b, imaginary: elementType, complex: adaptor.getComplex()); |
| 1069 | Value absRsqrt = computeAbs(real, imag, fmf, b, fn: AbsFn::rsqrt); |
| 1070 | Value argArg = math::Atan2Op::create(builder&: b, lhs: imag, rhs: real, fastmath: fmf); |
| 1071 | Value rsqrtArg = arith::MulFOp::create(builder&: b, lhs: argArg, rhs: negHalf, fastmath: fmf); |
| 1072 | Value cos = math::CosOp::create(builder&: b, operand: rsqrtArg, fastmath: fmf); |
| 1073 | Value sin = math::SinOp::create(builder&: b, operand: rsqrtArg, fastmath: fmf); |
| 1074 | |
| 1075 | Value resultReal = arith::MulFOp::create(builder&: b, lhs: absRsqrt, rhs: cos, fastmath: fmf); |
| 1076 | Value resultImag = arith::MulFOp::create(builder&: b, lhs: absRsqrt, rhs: sin, fastmath: fmf); |
| 1077 | |
| 1078 | if (!arith::bitEnumContainsAll(bits: fmf, bit: arith::FastMathFlags::nnan | |
| 1079 | arith::FastMathFlags::ninf)) { |
| 1080 | Value realSignedZero = math::CopySignOp::create(builder&: b, lhs: zero, rhs: real, fastmath: fmf); |
| 1081 | Value imagSignedZero = math::CopySignOp::create(builder&: b, lhs: zero, rhs: imag, fastmath: fmf); |
| 1082 | Value negImagSignedZero = arith::NegFOp::create(builder&: b, operand: imagSignedZero, fastmath: fmf); |
| 1083 | |
| 1084 | Value absReal = math::AbsFOp::create(builder&: b, operand: real, fastmath: fmf); |
| 1085 | Value absImag = math::AbsFOp::create(builder&: b, operand: imag, fastmath: fmf); |
| 1086 | |
| 1087 | Value absImagIsInf = arith::CmpFOp::create(builder&: b, predicate: arith::CmpFPredicate::OEQ, |
| 1088 | lhs: absImag, rhs: inf, fastmath: fmf); |
| 1089 | Value realIsNan = |
| 1090 | arith::CmpFOp::create(builder&: b, predicate: arith::CmpFPredicate::UNO, lhs: real, rhs: real, fastmath: fmf); |
| 1091 | Value realIsInf = arith::CmpFOp::create(builder&: b, predicate: arith::CmpFPredicate::OEQ, |
| 1092 | lhs: absReal, rhs: inf, fastmath: fmf); |
| 1093 | Value inIsNanInf = arith::AndIOp::create(builder&: b, lhs: absImagIsInf, rhs: realIsNan); |
| 1094 | |
| 1095 | Value resultIsZero = arith::OrIOp::create(builder&: b, lhs: inIsNanInf, rhs: realIsInf); |
| 1096 | |
| 1097 | resultReal = |
| 1098 | arith::SelectOp::create(builder&: b, condition: resultIsZero, true_value: realSignedZero, false_value: resultReal); |
| 1099 | resultImag = arith::SelectOp::create(builder&: b, condition: resultIsZero, true_value: negImagSignedZero, |
| 1100 | false_value: resultImag); |
| 1101 | } |
| 1102 | |
| 1103 | Value isRealZero = |
| 1104 | arith::CmpFOp::create(builder&: b, predicate: arith::CmpFPredicate::OEQ, lhs: real, rhs: zero, fastmath: fmf); |
| 1105 | Value isImagZero = |
| 1106 | arith::CmpFOp::create(builder&: b, predicate: arith::CmpFPredicate::OEQ, lhs: imag, rhs: zero, fastmath: fmf); |
| 1107 | Value isZero = arith::AndIOp::create(builder&: b, lhs: isRealZero, rhs: isImagZero); |
| 1108 | |
| 1109 | resultReal = arith::SelectOp::create(builder&: b, condition: isZero, true_value: inf, false_value: resultReal); |
| 1110 | resultImag = arith::SelectOp::create(builder&: b, condition: isZero, true_value: nan, false_value: resultImag); |
| 1111 | |
| 1112 | rewriter.replaceOpWithNewOp<complex::CreateOp>(op, args&: type, args&: resultReal, |
| 1113 | args&: resultImag); |
| 1114 | return success(); |
| 1115 | } |
| 1116 | }; |
| 1117 | |
| 1118 | struct AngleOpConversion : public OpConversionPattern<complex::AngleOp> { |
| 1119 | using OpConversionPattern<complex::AngleOp>::OpConversionPattern; |
| 1120 | |
| 1121 | LogicalResult |
| 1122 | matchAndRewrite(complex::AngleOp op, OpAdaptor adaptor, |
| 1123 | ConversionPatternRewriter &rewriter) const override { |
| 1124 | auto loc = op.getLoc(); |
| 1125 | auto type = op.getType(); |
| 1126 | arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr(); |
| 1127 | |
| 1128 | Value real = |
| 1129 | complex::ReOp::create(builder&: rewriter, location: loc, real: type, complex: adaptor.getComplex()); |
| 1130 | Value imag = |
| 1131 | complex::ImOp::create(builder&: rewriter, location: loc, imaginary: type, complex: adaptor.getComplex()); |
| 1132 | |
| 1133 | rewriter.replaceOpWithNewOp<math::Atan2Op>(op, args&: imag, args&: real, args&: fmf); |
| 1134 | |
| 1135 | return success(); |
| 1136 | } |
| 1137 | }; |
| 1138 | |
| 1139 | } // namespace |
| 1140 | |
| 1141 | void mlir::populateComplexToStandardConversionPatterns( |
| 1142 | RewritePatternSet &patterns, complex::ComplexRangeFlags complexRange) { |
| 1143 | // clang-format off |
| 1144 | patterns.add< |
| 1145 | AbsOpConversion, |
| 1146 | AngleOpConversion, |
| 1147 | Atan2OpConversion, |
| 1148 | BinaryComplexOpConversion<complex::AddOp, arith::AddFOp>, |
| 1149 | BinaryComplexOpConversion<complex::SubOp, arith::SubFOp>, |
| 1150 | ComparisonOpConversion<complex::EqualOp, arith::CmpFPredicate::OEQ>, |
| 1151 | ComparisonOpConversion<complex::NotEqualOp, arith::CmpFPredicate::UNE>, |
| 1152 | ConjOpConversion, |
| 1153 | CosOpConversion, |
| 1154 | ExpOpConversion, |
| 1155 | Expm1OpConversion, |
| 1156 | Log1pOpConversion, |
| 1157 | LogOpConversion, |
| 1158 | MulOpConversion, |
| 1159 | NegOpConversion, |
| 1160 | SignOpConversion, |
| 1161 | SinOpConversion, |
| 1162 | SqrtOpConversion, |
| 1163 | TanTanhOpConversion<complex::TanOp>, |
| 1164 | TanTanhOpConversion<complex::TanhOp>, |
| 1165 | PowiOpConversion, |
| 1166 | PowOpConversion, |
| 1167 | RsqrtOpConversion |
| 1168 | >(arg: patterns.getContext()); |
| 1169 | |
| 1170 | patterns.add<DivOpConversion>(arg: patterns.getContext(), args&: complexRange); |
| 1171 | |
| 1172 | // clang-format on |
| 1173 | } |
| 1174 | |
| 1175 | namespace { |
| 1176 | struct ConvertComplexToStandardPass |
| 1177 | : public impl::ConvertComplexToStandardPassBase< |
| 1178 | ConvertComplexToStandardPass> { |
| 1179 | using Base::Base; |
| 1180 | |
| 1181 | void runOnOperation() override; |
| 1182 | }; |
| 1183 | |
| 1184 | void ConvertComplexToStandardPass::runOnOperation() { |
| 1185 | // Convert to the Standard dialect using the converter defined above. |
| 1186 | RewritePatternSet patterns(&getContext()); |
| 1187 | populateComplexToStandardConversionPatterns(patterns, complexRange); |
| 1188 | |
| 1189 | ConversionTarget target(getContext()); |
| 1190 | target.addLegalDialect<arith::ArithDialect, math::MathDialect>(); |
| 1191 | target.addLegalOp<complex::CreateOp, complex::ImOp, complex::ReOp>(); |
| 1192 | if (failed( |
| 1193 | Result: applyPartialConversion(op: getOperation(), target, patterns: std::move(patterns)))) |
| 1194 | signalPassFailure(); |
| 1195 | } |
| 1196 | } // namespace |
| 1197 | |