| 1 | //===----------------------------------------------------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | /// |
| 9 | /// \file |
| 10 | /// Double-precision implementation of exp2. |
| 11 | /// |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
| 14 | #ifndef LLVM_LIBC_SRC___SUPPORT_MATH_EXP2_DOUBLE_EVAL_H |
| 15 | #define LLVM_LIBC_SRC___SUPPORT_MATH_EXP2_DOUBLE_EVAL_H |
| 16 | |
| 17 | #include "src/__support/CPP/bit.h" |
| 18 | #include "src/__support/FPUtil/FPBits.h" |
| 19 | #include "src/__support/FPUtil/double_double.h" |
| 20 | #include "src/__support/FPUtil/fp_bitwise.h" |
| 21 | #include "src/__support/FPUtil/multiply_add.h" |
| 22 | #include "src/__support/common.h" |
| 23 | #include "src/__support/macros/attributes.h" |
| 24 | #include "src/__support/macros/config.h" |
| 25 | #include "src/__support/macros/optimization.h" |
| 26 | #include "src/__support/macros/properties/cpu_features.h" |
| 27 | #ifdef LIBC_MATH_HAS_SKIP_ACCURATE_PASS |
| 28 | #include "src/__support/math/exp_double_utils.h" |
| 29 | #else |
| 30 | #include "src/__support/math/exp_utils.h" |
| 31 | #endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS |
| 32 | |
| 33 | namespace LIBC_NAMESPACE_DECL { |
| 34 | namespace math { |
| 35 | namespace double_eval { |
| 36 | |
| 37 | LIBC_ALWAYS_INLINE double exp2(double x) { |
| 38 | using FPBits = fputil::FPBits<double>; |
| 39 | FPBits xbits(x); |
| 40 | |
| 41 | uint64_t x_u = xbits.uintval(); |
| 42 | uint64_t x_abs = xbits.abs().uintval(); |
| 43 | |
| 44 | // Bound constants (with biased exponent and mantissa bits): |
| 45 | // - Tiny bound: nextafter(2^-54, +inf) = 0x1.0000000000001p-54 |
| 46 | // (since |x| <= 2^-54 is handled by 1.0 + x in set_exceptional) |
| 47 | constexpr uint64_t TINY_BOUND = 0x3c90'0000'0000'0001ULL; |
| 48 | // - Overflow bound: 1024.0 |
| 49 | constexpr uint64_t OVERFLOW_BOUND = 0x4090'0000'0000'0000ULL; |
| 50 | // -1024.0 |
| 51 | constexpr uint64_t NEG_OVERFLOW_BOUND = 0xc090'0000'0000'0000ULL; |
| 52 | // - Underflow bound: -1075.0 |
| 53 | constexpr uint64_t UNDERFLOW_BOUND = 0xc090'cc00'0000'0000ULL; |
| 54 | |
| 55 | // Normal range: 2^-54 <= |x| < 1024. |
| 56 | if (LIBC_UNLIKELY(x_abs - TINY_BOUND >= OVERFLOW_BOUND - TINY_BOUND)) { |
| 57 | // Exclude non-underflowing negative inputs in (-1075, -1024]. |
| 58 | if (LIBC_LIKELY(x_u < NEG_OVERFLOW_BOUND || x_u >= UNDERFLOW_BOUND)) |
| 59 | return set_exceptional(x); |
| 60 | } |
| 61 | |
| 62 | // Now -1075 < x < -2^-54 or 2^-54 < x < 1024. |
| 63 | |
| 64 | // Range reduction: |
| 65 | // Let x = k * 2^-N + dx |
| 66 | // in which: |
| 67 | // k = round(x * 2^N) |
| 68 | // hi = k >> N |
| 69 | // idx = k & (2^N - 1) |
| 70 | // then: |
| 71 | // 2^x = 2^hi * 2^(idx / 2^N) * 2^dx. |
| 72 | // |
| 73 | // Using ROUND_BIAS, ROUND_SHIFT, and ROUND_MASK (see exp_utils.h), we compute |
| 74 | // k_biased = fma(x, EXP_SCALE, ROUND_BIAS) and extract in parallel: |
| 75 | // kd = (k_biased & ROUND_MASK) - ROUND_SHIFT |
| 76 | // hi = (int32_t)(k_bits >> 19) >> (13 + N) |
| 77 | // idx = (k_bits >> 32) & (2^N - 1). |
| 78 | double k_biased = fputil::multiply_add(x, y: EXP_SCALE, z: ROUND_BIAS); |
| 79 | double kd = fputil::fp_and(a: k_biased, b: ROUND_MASK) - ROUND_SHIFT; |
| 80 | uint64_t k_bits = cpp::bit_cast<uint64_t>(from: k_biased); |
| 81 | |
| 82 | int hi = static_cast<int32_t>(k_bits >> 19) >> (13 + EXP_N_BITS); |
| 83 | unsigned idx = static_cast<unsigned>(k_bits >> 32) & EXP_MASK; |
| 84 | |
| 85 | double dx = fputil::sub_multiply(x, y: kd, z: EXP_SCALE_INV); // exact |
| 86 | |
| 87 | #ifdef LIBC_MATH_HAS_SKIP_ACCURATE_PASS |
| 88 | double u = dx * LOG_2_DD.hi; |
| 89 | return exp2_eval(x, u, hi, idx, /*is_exp2=*/true); |
| 90 | #else // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS |
| 91 | #ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE |
| 92 | DoubleDouble u = fputil::exact_mult(dx, LOG_2_DD.hi); |
| 93 | u.lo = fputil::multiply_add(dx, LOG_2_DD.lo, u.lo); |
| 94 | #else // !LIBC_TARGET_CPU_HAS_FMA_DOUBLE |
| 95 | // Without FMA, we use fixed-point splitting to avoid Veltkamp's splitting: |
| 96 | // Let: |
| 97 | // m = 2^(idx / 2^7) = m_hi + m_lo |
| 98 | // u = dx * log(2) = u_hi + u_lo |
| 99 | // |
| 100 | // In exp2_eval, we approximate 2^(idx / 2^7) * 2^dx = m * exp(u) by: |
| 101 | // m * (1 + u + u^2 * P(u)) ~ r_hi + r_lo |
| 102 | // where: |
| 103 | // r_hi = m_hi + m_hi * u_hi |
| 104 | // r_lo = m * (u_lo + u^2 * P(u)) + m_lo * (1 + u_hi) |
| 105 | // |
| 106 | // Since r_hi is in [1 - 2^-8, 2), for r_hi to be exact in double precision, |
| 107 | // we need: |
| 108 | // lsb(m_hi * u_hi) >= 2^-52. |
| 109 | // We choose: |
| 110 | // lsb(m_hi) >= 2^-22 => |m_lo| < 2^-22 |
| 111 | // lsb(u_hi) >= 2^-30 |
| 112 | // so that lsb(m_hi * u_hi) >= 2^-52, and the rounding errors from |
| 113 | // m_lo * (1 + u_hi) are bounded by 2^(-22 - 52) = 2^-74. |
| 114 | // |
| 115 | // To get lsb(u_hi) >= 2^-30 from u = dx * log(2), since |dx| <= 2^-8 and |
| 116 | // log(2) < 1, we split dx at 2^-18 and log(2) at 2^-12: |
| 117 | // dx = dx_hi + dx_lo (|dx_lo| <= 2^-19) |
| 118 | // log(2) = LOG_2_HI_12 + LOG_2_LO_12 (|LOG_2_LO_12| < 2^-13) |
| 119 | // Then: |
| 120 | // u_hi = dx_hi * LOG_2_HI_12 |
| 121 | // is an exact multiple of 2^-30, and the two remainder products in: |
| 122 | // u_lo = dx_lo * LOG_2_HI_12 + dx * LOG_2_LO_12 |
| 123 | // are both bounded by 2^-19, keeping the rounding errors of u_lo below 2^-71. |
| 124 | double dx_hi = (dx + SPLIT_18) - SPLIT_18; // lsb = 2^-18 |
| 125 | double dx_lo = dx - dx_hi; |
| 126 | double u_hi = dx_hi * LOG_2_HI_12; // exact, lsb = 2^-30 |
| 127 | double t0 = dx_lo * LOG_2_HI_12; |
| 128 | double t1 = dx * LOG_2_LO_12; |
| 129 | double u_lo = t0 + t1; |
| 130 | DoubleDouble u = {.lo: u_lo, .hi: u_hi}; |
| 131 | #endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE |
| 132 | return exp2_eval(x, u, hi, idx); |
| 133 | #endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS |
| 134 | } |
| 135 | |
| 136 | } // namespace double_eval |
| 137 | } // namespace math |
| 138 | } // namespace LIBC_NAMESPACE_DECL |
| 139 | |
| 140 | #endif // LLVM_LIBC_SRC___SUPPORT_MATH_EXP2_DOUBLE_EVAL_H |
| 141 | |