| 1 | //===----------------------------------------------------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | /// |
| 9 | /// \file |
| 10 | /// Double-precision implementation of tanh. |
| 11 | /// |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
| 14 | #ifndef LLVM_LIBC_SRC___SUPPORT_MATH_TANH_DOUBLE_EVAL_H |
| 15 | #define LLVM_LIBC_SRC___SUPPORT_MATH_TANH_DOUBLE_EVAL_H |
| 16 | |
| 17 | #include "hdr/errno_macros.h" |
| 18 | #include "hdr/fenv_macros.h" |
| 19 | #include "src/__support/CPP/bit.h" |
| 20 | #include "src/__support/FPUtil/FEnvImpl.h" |
| 21 | #include "src/__support/FPUtil/FPBits.h" |
| 22 | #include "src/__support/FPUtil/double_double.h" |
| 23 | #include "src/__support/FPUtil/except_value_utils.h" |
| 24 | #include "src/__support/FPUtil/fp_bitwise.h" |
| 25 | #include "src/__support/FPUtil/multiply_add.h" |
| 26 | #include "src/__support/FPUtil/polyeval.h" |
| 27 | #include "src/__support/common.h" |
| 28 | #include "src/__support/macros/attributes.h" |
| 29 | #include "src/__support/macros/config.h" |
| 30 | #include "src/__support/macros/optimization.h" |
| 31 | #include "src/__support/macros/properties/cpu_features.h" |
| 32 | #include "src/__support/math/exp_utils.h" |
| 33 | |
| 34 | #ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS |
| 35 | #include "src/__support/math/tanh_accurate_128.h" |
| 36 | #endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS |
| 37 | |
| 38 | namespace LIBC_NAMESPACE_DECL { |
| 39 | namespace math { |
| 40 | namespace double_eval { |
| 41 | |
| 42 | #ifdef LIBC_MATH_HAS_SKIP_ACCURATE_PASS |
| 43 | |
| 44 | // Splitting -log(2) / 2 ~ hi + lo with 35-bit-precision hi part so that |
| 45 | // kd * MLOG_2_HALF_HI is exact for |kd| < 2^18: |
| 46 | // > hi = round(-log(2) / 2, 35, RN); |
| 47 | // > lo = round(-log(2) / 2 - hi, D, RN); |
| 48 | LIBC_INLINE_VAR constexpr double MLOG_2_HALF_HI = -0x1.62e42fefcp-2; |
| 49 | LIBC_INLINE_VAR constexpr double MLOG_2_HALF_LO = 0x1.c610ca86c3899p-38; |
| 50 | |
| 51 | // Degree-3 minimax polynomials in v^2 for (cosh(v) - 1) / v^2 and |
| 52 | // (sinh(v) - v) / v^3 on [-log(2) / 4, log(2) / 4]: |
| 53 | // cosh(v) - 1 ~ v^2 * P_even(v^2) |
| 54 | // sinh(v) - v ~ v^3 * P_odd(v^2). |
| 55 | // |
| 56 | // Generated by Sollya with: |
| 57 | // > display = hexadecimal!; |
| 58 | // > prec = 500; |
| 59 | // > B = (log(2) / 4)^2; |
| 60 | // > P_even = fpminimax((cosh(sqrt(x)) - 1) / x, 3, [|D...|], [2^(-100), B]); |
| 61 | // > for i from 0 to 3 do print(coeff(P_even, i), ","); |
| 62 | // > dirtyinfnorm((cosh(sqrt(x)) - 1) - x * P_even, [0, B]); |
| 63 | // 0x1.e49e...p-55 < 2^-54. |
| 64 | LIBC_INLINE_VAR constexpr double TANH_EVEN_N0_COEFFS[4] = { |
| 65 | 0x1.fffffffffffep-2, // ~ 1/2! |
| 66 | 0x1.5555555597352p-5, // ~ 1/4! |
| 67 | 0x1.6c16bc13d272cp-10, // ~ 1/6! |
| 68 | 0x1.a06123b7cd784p-16, // ~ 1/8! |
| 69 | }; |
| 70 | |
| 71 | // > P_odd = fpminimax((sinh(sqrt(x)) / sqrt(x) - 1) / x, 3, [|D...|], |
| 72 | // [2^(-100), B]); |
| 73 | // > for i from 0 to 3 do print(coeff(P_odd, i), ","); |
| 74 | // > dirtyinfnorm((sinh(sqrt(x)) - sqrt(x)) - x * sqrt(x) * P_odd, [0, B]); |
| 75 | // 0x1.ee40...p-61 < 2^-60. |
| 76 | LIBC_INLINE_VAR constexpr double TANH_ODD_N0_COEFFS[4] = { |
| 77 | 0x1.555555555555p-3, // ~ 1/3! |
| 78 | 0x1.111111112884cp-7, // ~ 1/5! |
| 79 | 0x1.a019fdc55d34fp-13, // ~ 1/7! |
| 80 | 0x1.7211ce0221666p-19, // ~ 1/9! |
| 81 | }; |
| 82 | |
| 83 | // Tableless (N = 0) evaluation of tanh(x) = sinh(x) / cosh(x) for |
| 84 | // 2^-27 < |x| < 0x1.30fc1931f09cap+4. |
| 85 | // |
| 86 | // Range reduction: |
| 87 | // Let |x| = k * (log(2) / 2) + v, where: |
| 88 | // k = round(|x| * 2 * log2(e)) |
| 89 | // |v| <= log(2) / 4 |
| 90 | // then: |
| 91 | // e^|x| = 2^(k / 2) * e^v |
| 92 | // e^-|x| = 2^(-k / 2) * e^-v = 2^(k / 2) * 2^-k * e^-v. |
| 93 | // |
| 94 | // Factoring out 2^(k / 2) in numerator and denominator and separating |
| 95 | // e^(+/- v) = cosh(v) +/- sinh(v): |
| 96 | // tanh(|x|) = (e^v - m_m * e^-v) / (e^v + m_m * e^-v) |
| 97 | // = (d * cosh(v) + s * sinh(v)) / (s * cosh(v) + d * sinh(v)) |
| 98 | // where: |
| 99 | // m_m = 2^-k |
| 100 | // s = 1 + m_m |
| 101 | // d = 1 - m_m. |
| 102 | // |
| 103 | // Key properties across the domain: |
| 104 | // - For |x| < log(2) / 4, k = 0 gives v = |x| (exact, t2 = 0) and m_m = 1, so |
| 105 | // d = 0 and s = 2, reducing the ratio to: |
| 106 | // (2 * sinh(v)) / (2 * cosh(v)) = sinh(x) / cosh(x) |
| 107 | // with no cancellation down to |x| = 2^-27. |
| 108 | // |
| 109 | // - For 1 <= k <= 53, 2^-k is in [2^-53, 2^-1], so both: |
| 110 | // s = 1.0 + m_m |
| 111 | // d = 1.0 - m_m |
| 112 | // are exact in all rounding modes. For k >= 54: |
| 113 | // m_m <= 2^-54 < 0.5 * ulp(1.0), |
| 114 | // so clamping m_m to 0.0 avoids a 1-ULP rounding error in directed rounding |
| 115 | // modes. |
| 116 | LIBC_MATH_SMALL_TABLES_INLINE double tanh_eval(double x) { |
| 117 | using FPBits = fputil::FPBits<double>; |
| 118 | FPBits xbits(x); |
| 119 | double x_a = xbits.abs().get_val(); |
| 120 | |
| 121 | // Using ROUND_BIAS, ROUND_SHIFT, and ROUND_MASK (see exp_utils.h), we compute |
| 122 | // k_biased = fma(x_a, LOG2_E_2, ROUND_BIAS) and extract in parallel: |
| 123 | // kd = (k_biased & ROUND_MASK) - ROUND_SHIFT |
| 124 | // k = (k_bits >> 32) & 0x3'ffff. |
| 125 | constexpr double LOG2_E_2 = 2.0 * LOG2_E_DD.hi; |
| 126 | double k_biased = fputil::multiply_add(x_a, LOG2_E_2, ROUND_BIAS); |
| 127 | double kd = fputil::fp_and(k_biased, ROUND_MASK) - ROUND_SHIFT; |
| 128 | int k = static_cast<int>(cpp::bit_cast<uint64_t>(k_biased) >> 32) & 0x3'ffff; |
| 129 | |
| 130 | // Secondary scale factor: |
| 131 | // m_m = 2^-k for k < 54 |
| 132 | // m_m = 0.0 for k >= 54 (where 2^-k <= 2^-54). |
| 133 | double m_m = (k < 54) ? scale_exp2(1.0, -k) : 0.0; |
| 134 | |
| 135 | // Reduced argument: |
| 136 | // v = (x_a + kd * MLOG_2_HALF_HI) + kd * MLOG_2_HALF_LO. |
| 137 | // Since MLOG_2_HALF_HI has 35 significant bits and |kd| <= 55, |
| 138 | // t1 = fma(kd, MLOG_2_HALF_HI, x_a) |
| 139 | // is exact in all rounding modes with or without FMA. |
| 140 | double t1 = fputil::multiply_add(kd, MLOG_2_HALF_HI, x_a); // exact |
| 141 | double t2 = kd * MLOG_2_HALF_LO; |
| 142 | double v = t1 + t2; |
| 143 | double v2 = v * v; |
| 144 | double v4 = v2 * v2; |
| 145 | |
| 146 | // Evaluate degree-3 even and odd minimax polynomials P_even(v^2) and |
| 147 | // P_odd(v^2) in parallel using Estrin's scheme: |
| 148 | // cosh(v) ~ 1 + v^2 * P_even(v^2) |
| 149 | // sinh(v) ~ t1 + (t2 + v^3 * P_odd(v^2)) |
| 150 | |
| 151 | // First Estrin layer: c_0 + v^2 * c_1 and c_2 + v^2 * c_3 |
| 152 | double e01 = |
| 153 | fputil::multiply_add(v2, TANH_EVEN_N0_COEFFS[1], TANH_EVEN_N0_COEFFS[0]); |
| 154 | double o01 = |
| 155 | fputil::multiply_add(v2, TANH_ODD_N0_COEFFS[1], TANH_ODD_N0_COEFFS[0]); |
| 156 | double e23 = |
| 157 | fputil::multiply_add(v2, TANH_EVEN_N0_COEFFS[3], TANH_EVEN_N0_COEFFS[2]); |
| 158 | double o23 = |
| 159 | fputil::multiply_add(v2, TANH_ODD_N0_COEFFS[3], TANH_ODD_N0_COEFFS[2]); |
| 160 | |
| 161 | // Final polynomial values P_even(v^2) and P_odd(v^2): |
| 162 | double p_even = fputil::multiply_add(v4, e23, e01); |
| 163 | double p_odd = fputil::multiply_add(v4, o23, o01); |
| 164 | |
| 165 | // Low parts of cosh(v) - 1 and sinh(v) - t1: |
| 166 | // c_lo ~ v^2 * P_even(v^2) |
| 167 | // s_lo ~ t2 + v^3 * P_odd(v^2) |
| 168 | double c_lo = v2 * p_even; |
| 169 | double v3 = v * v2; |
| 170 | double s_lo = fputil::multiply_add(v3, p_odd, t2); |
| 171 | |
| 172 | // Since m_m is either 0 or 2^-k with 0 <= k <= 53, both s and d are exact in |
| 173 | // all rounding modes: |
| 174 | double s = 1.0 + m_m; // exact |
| 175 | double d = 1.0 - m_m; // exact |
| 176 | |
| 177 | // Apply sign(x) to the numerator weights s_num and d_num before accumulating |
| 178 | // the polynomial so the numerator rounds in the active rounding direction for |
| 179 | // negative x as well: |
| 180 | double sign_x = fputil::fp_xor(x, x_a); |
| 181 | double s_num = fputil::fp_xor(s, sign_x); |
| 182 | double d_num = fputil::fp_xor(d, sign_x); |
| 183 | |
| 184 | // Numerator: |
| 185 | // num ~ sign(x) * (d * (1 + c_lo) + s * (t1 + s_lo)) |
| 186 | // = d_num + (s_num * t1 + (d_num * c_lo + s_num * s_lo)) |
| 187 | double s_s_lo = s_num * s_lo; |
| 188 | double num_lo = fputil::multiply_add(d_num, c_lo, s_s_lo); |
| 189 | double num_mid = fputil::multiply_add(s_num, t1, num_lo); |
| 190 | double num = d_num + num_mid; |
| 191 | |
| 192 | // Denominator: |
| 193 | // den ~ s * (1 + c_lo) + d * (t1 + s_lo) |
| 194 | // = s + (d * t1 + (s * c_lo + d * s_lo)) |
| 195 | double d_s_lo = d * s_lo; |
| 196 | double den_lo = fputil::multiply_add(s, c_lo, d_s_lo); |
| 197 | double den_mid = fputil::multiply_add(d, t1, den_lo); |
| 198 | double den = s + den_mid; |
| 199 | |
| 200 | return num / den; |
| 201 | } |
| 202 | |
| 203 | #else // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS |
| 204 | |
| 205 | // Leading 11-bit split of c_0 ~ -1/3 for exact cubic reconstruction: |
| 206 | // TANH_C0_HI_11 = round(-1/3, 11, RZ) = -0x1.554p-2 |
| 207 | // -1/3 - TANH_C0_HI_11 = -0x1.5555555555555p-14. |
| 208 | LIBC_INLINE_VAR constexpr double TANH_C0_HI_11 = -0x1.554p-2; |
| 209 | |
| 210 | // Polynomial approximation for ((tanh(x) - x) / x^3 - (-0x1.554p-2)) on |
| 211 | // [-2^-3, 2^-3] as a degree-5 even polynomial in x^2: |
| 212 | // tanh(x) - x ~ x^3 * (TANH_C0_HI_11 + P_rem(x^2)) |
| 213 | // |
| 214 | // Minimax polynomial coefficients generated by Sollya with: |
| 215 | // > display = hexadecimal!; |
| 216 | // > prec = 500; |
| 217 | // > c0 = -0x1.554p-2 + (-0x1.5555555555555p-14); |
| 218 | // > P = fpminimax((tanh(sqrt(x))/sqrt(x) - 1)/x - c0, [|1, 2, 3, 4, 5|], |
| 219 | // [|D...|], [2^(-100), 2^(-6)], absolute); |
| 220 | // > for i from 1 to 5 do print(coeff(P, i), ","); |
| 221 | // > dirtyinfnorm((tanh(x) - x)/x^3 - (c0 + P(x^2)), [2^(-50), 2^(-3)]); |
| 222 | // 0x1.9e2e...p-57 < 2^-56. |
| 223 | // > dirtyinfnorm(tanh(x) - (x + x^3 * (c0 + P(x^2))), [-2^(-3), 2^(-3)]); |
| 224 | // 0x1.9dd6...p-66 < 2^-65. |
| 225 | LIBC_INLINE_VAR constexpr double TANH_SMALL_P_COEFFS[6] = { |
| 226 | -0x1.5555555555555p-14, // c_0 - (-0x1.554p-2) ~ -1/3 + 0x1.554p-2 |
| 227 | 0x1.1111111110c74p-3, // c_1 ~ 2/15 |
| 228 | -0x1.ba1ba1b6047f3p-5, // c_2 ~ -17/315 |
| 229 | 0x1.664f3f15b86a5p-6, // c_3 ~ 62/2835 |
| 230 | -0x1.2264b8b90e812p-7, // c_4 ~ -1382/155925 |
| 231 | 0x1.ce2449b1b72dcp-9, // c_5 ~ 21844/6081075 |
| 232 | }; |
| 233 | |
| 234 | // Evaluate tanh(x) in double-double precision for |
| 235 | // 2^-3 <= |x| < 0x1.30fc1931f09cap+4, falling back to tanh_accurate_128(x) when |
| 236 | // Ziv's rounding test fails. |
| 237 | // |
| 238 | // Range reduction: |
| 239 | // Let 2 * |x| = k_p * (log(2) / 2^7) + u |
| 240 | // in which: |
| 241 | // k_p = round(2 * |x| * 2^7 * log2(e)), k_m = -k_p |
| 242 | // hi_m = k_m >> 7, idx_m = k_m & 0x7f |
| 243 | // |u| <= 2^-8 * log(2) |
| 244 | // then: |
| 245 | // e^(2 * |x|) = 2^(k_p / 2^7) * e^u = e^u / m |
| 246 | // where: |
| 247 | // m = 2^hi_m * 2^(idx_m / 2^7) = 2^hi_m * EXP2_MID[idx_m]. |
| 248 | // |
| 249 | // Rewriting tanh(|x|) = (e^(2 * |x|) - 1) / (e^(2 * |x|) + 1) in terms of e^u |
| 250 | // and m avoids multiplying m by e^u: |
| 251 | // tanh(|x|) = (e^u - m) / (e^u + m) = 1 - (2 * m) / (e^u + m). |
| 252 | LIBC_MATH_SMALL_TABLES_INLINE double tanh_eval(double x) { |
| 253 | using FPBits = fputil::FPBits<double>; |
| 254 | FPBits xbits(x); |
| 255 | double x_a = xbits.abs().get_val(); |
| 256 | double x_2 = 2.0 * x_a; |
| 257 | |
| 258 | constexpr double LOG2_E_SCALE = EXP_SCALE * LOG2_E_DD.hi; |
| 259 | // Using ROUND_BIAS, ROUND_SHIFT, and ROUND_MASK (see exp_utils.h), we compute |
| 260 | // k_biased = fma(x_2, LOG2_E_SCALE, ROUND_BIAS) and extract in parallel: |
| 261 | // kd = (k_biased & ROUND_MASK) - ROUND_SHIFT |
| 262 | // k_p = (k_bits >> 32) & 0x3'ffff. |
| 263 | double k_biased = fputil::multiply_add(x: x_2, y: LOG2_E_SCALE, z: ROUND_BIAS); |
| 264 | double kd = fputil::fp_and(a: k_biased, b: ROUND_MASK) - ROUND_SHIFT; |
| 265 | int k_p = |
| 266 | static_cast<int>(cpp::bit_cast<uint64_t>(from: k_biased) >> 32) & 0x3'ffff; |
| 267 | int k_m = -k_p; |
| 268 | |
| 269 | int hi_m = k_m >> EXP_N_BITS; |
| 270 | unsigned idx_m = static_cast<unsigned>(k_m & EXP_MASK); |
| 271 | |
| 272 | // Scale m = EXP2_MID[idx_m] by 2^hi_m: |
| 273 | // For 2^-3 <= |x| < 0x1.30fc1931f09cap+4, hi_m is in [-56, 0], so scaling by |
| 274 | // 2^hi_m is always in the normal range: |
| 275 | // - Since m.hi is in [1, 2), we use scale_exp2(m.hi, hi_m) directly. |
| 276 | // - Since EXP2_MID[0].lo == 0.0, we scale m.lo by multiplying by: |
| 277 | // s_m = scale_exp2(1.0, hi_m). |
| 278 | double s_m = scale_exp2(r: 1.0, hi: hi_m); |
| 279 | DoubleDouble m = EXP2_MID[idx_m]; |
| 280 | m.lo *= s_m; |
| 281 | m.hi = scale_exp2(r: m.hi, hi: hi_m); |
| 282 | |
| 283 | // Reduced argument: |
| 284 | // u = (x_2 + kd * MLOG_2_EXP2_HI) + kd * MLOG_2_EXP2_MID. |
| 285 | // Since MLOG_2_EXP2_HI has 35 significant bits and |kd| < 2^13, |
| 286 | // t1 = fma(kd, MLOG_2_EXP2_HI, x_2) |
| 287 | // is exact in all rounding modes with or without FMA. |
| 288 | double t1 = fputil::multiply_add(x: kd, y: MLOG_2_EXP2_HI, z: x_2); // exact |
| 289 | double t2 = kd * MLOG_2_EXP2_MID; |
| 290 | double u = t1 + t2; |
| 291 | |
| 292 | // Evaluate degree-3 minimax polynomial P(u) ~ (e^u - 1 - u) / u^2: |
| 293 | // e^u ~ 1 + t1 + exp_lo |
| 294 | // where: |
| 295 | // exp_lo = t2 + u^2 * P(u). |
| 296 | double u2 = u * u; |
| 297 | double p = fputil::estrin_polyeval(x: u, a: EXP_P_COEFFS); |
| 298 | double exp_lo = fputil::multiply_add(x: u2, y: p, z: t2); |
| 299 | |
| 300 | // Denominator D = e^u + m ~ (1.0 + m.hi + t1 + exp_lo) + m.lo: |
| 301 | // s.hi + s.lo = 1.0 + m.hi (exact FastTwoSum since m.hi <= 1.0) |
| 302 | // d1.hi + d1.lo = s.hi + t1 (FastTwoSum, |t1| < 2^-8 < s.hi) |
| 303 | // d.hi + d.lo = d1.hi + exp_lo (FastTwoSum, |exp_lo| < 2^-17 < d1.hi). |
| 304 | // Accumulating exp_lo into d.hi keeps |d_lo| <= 2^-50 so the linear |
| 305 | // Karp-Markstein remainder is accurate to 105 bits: |
| 306 | DoubleDouble s = fputil::exact_add(a: 1.0, b: m.hi); |
| 307 | DoubleDouble d1 = fputil::exact_add(a: s.hi, b: t1); |
| 308 | DoubleDouble d = fputil::exact_add(a: d1.hi, b: exp_lo); |
| 309 | |
| 310 | // Combine FastTwoSum residuals and table low part: |
| 311 | // d_lo = s.lo + d1.lo + d.lo + m.lo |
| 312 | double sd1_lo = s.lo + d1.lo; |
| 313 | double sdd_lo = sd1_lo + d.lo; |
| 314 | double d_lo = sdd_lo + m.lo; |
| 315 | |
| 316 | // Karp-Markstein division for q_hi + q_lo ~ (2 * m) / (d.hi + d_lo): |
| 317 | // y_hi = round(1.0 / d.hi) |
| 318 | // q_hi = round(2 * m.hi * y_hi) |
| 319 | // rem_hi = 2 * m.hi - q_hi * d.hi |
| 320 | // rem_lo = 2 * m.lo - q_hi * d_lo |
| 321 | // q_lo = (rem_hi + rem_lo) * y_hi. |
| 322 | double two_m_hi = 2.0 * m.hi; |
| 323 | double two_m_lo = 2.0 * m.lo; |
| 324 | double y_hi = 1.0 / d.hi; |
| 325 | double q_hi = two_m_hi * y_hi; |
| 326 | #ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE |
| 327 | double rem_hi = fputil::sub_multiply(two_m_hi, q_hi, d.hi); |
| 328 | double rem_lo = fputil::sub_multiply(two_m_lo, q_hi, d_lo); |
| 329 | #else |
| 330 | // Split q_hi into 26 + 27 bits and d.hi into 27 + 26 bits via bit-masks so |
| 331 | // that q_hi_hi * d_hi_hi (53 bits) and q_hi_hi * d_hi_lo (52 bits) are exact |
| 332 | // without FMA: |
| 333 | LIBC_BIT_CAST_CONSTEXPR_VAR double HI_MASK_26 = |
| 334 | cpp::bit_cast<double>(from: 0xffff'ffff'f800'0000ULL); |
| 335 | LIBC_BIT_CAST_CONSTEXPR_VAR double HI_MASK_27 = |
| 336 | cpp::bit_cast<double>(from: 0xffff'ffff'fc00'0000ULL); |
| 337 | double q_hi_hi = fputil::fp_and(a: q_hi, b: HI_MASK_26); |
| 338 | double q_hi_lo = q_hi - q_hi_hi; |
| 339 | double d_hi_hi = fputil::fp_and(a: d.hi, b: HI_MASK_27); |
| 340 | double d_hi_lo = d.hi - d_hi_hi; |
| 341 | |
| 342 | // High remainder rem_hi = 2 * m.hi - q_hi * d.hi: |
| 343 | double prod_hh = q_hi_hi * d_hi_hi; // exact (53 bits) |
| 344 | double prod_hl = q_hi_hi * d_hi_lo; // exact (52 bits) |
| 345 | double prod_lh = q_hi_lo * d_hi_hi; |
| 346 | double prod_ll = q_hi_lo * d_hi_lo; |
| 347 | |
| 348 | double rem_1 = two_m_hi - prod_hh; // exact |
| 349 | double rem_2 = rem_1 - prod_hl; |
| 350 | double rem_3 = rem_2 - prod_lh; |
| 351 | double rem_hi = rem_3 - prod_ll; |
| 352 | |
| 353 | // Low remainder rem_lo = 2 * m.lo - q_hi * d_lo: |
| 354 | double q_d_lo = q_hi * d_lo; |
| 355 | double rem_lo = two_m_lo - q_d_lo; |
| 356 | #endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE |
| 357 | double rem = rem_hi + rem_lo; |
| 358 | double q_lo = rem * y_hi; |
| 359 | |
| 360 | // Subtract q_hi + q_lo from 1.0: |
| 361 | // r.hi + r.lo = 1.0 - q_hi (FastTwoSum) |
| 362 | // r_lo = r.lo - q_lo. |
| 363 | // Since 1.0 is an exact power of two and q_hi is in (0, 0.88), |
| 364 | // exact_sub(1.0, q_hi) is accurate to within 2^-105 in all rounding modes. |
| 365 | DoubleDouble r = fputil::exact_sub(a: 1.0, b: q_hi); |
| 366 | double r_lo = r.lo - q_lo; |
| 367 | |
| 368 | // Apply sign(x) before Ziv's rounding test so that (r_hi + lo_u) and |
| 369 | // (r_hi + lo_l) round in the active rounding direction for negative x: |
| 370 | double sign_x = fputil::fp_xor(a: x, b: x_a); |
| 371 | double r_hi = fputil::fp_xor(a: r.hi, b: sign_x); |
| 372 | r_lo = fputil::fp_xor(a: r_lo, b: sign_x); |
| 373 | |
| 374 | // Error bound for Ziv's rounding test: |
| 375 | // Since any error in e^u or q is damped by q_hi ~ 2 * e^(-2 * |x|), we scale |
| 376 | // the relative error bound by q_hi: |
| 377 | // err = q_hi * TANH_ERR_REL + TANH_ERR_ABS. |
| 378 | constexpr double TANH_ERR_REL = 0x1.8p-63; |
| 379 | constexpr double TANH_ERR_ABS = 0x1.0p-104; |
| 380 | double err = fputil::multiply_add(x: q_hi, y: TANH_ERR_REL, z: TANH_ERR_ABS); |
| 381 | double lo_u = r_lo + err; |
| 382 | double lo_l = r_lo - err; |
| 383 | |
| 384 | // Ziv's rounding test in the current rounding mode: |
| 385 | double upper = r_hi + lo_u; |
| 386 | double lower = r_hi + lo_l; |
| 387 | |
| 388 | if (LIBC_LIKELY(upper == lower)) |
| 389 | return upper; |
| 390 | |
| 391 | return tanh_accurate_128(x); |
| 392 | } |
| 393 | |
| 394 | #endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS |
| 395 | |
| 396 | // Evaluate tanh(x) for |x| < SMALL_BOUND (2^-27 + ulp(2^-27) when |
| 397 | // LIBC_MATH_HAS_SKIP_ACCURATE_PASS is defined, or 2^-3 otherwise): |
| 398 | LIBC_INLINE double tanh_small_eval(double x, uint64_t x_abs) { |
| 399 | using FPBits = fputil::FPBits<double>; |
| 400 | |
| 401 | // Tiny and subnormal range: |x| <= 2^-27. |
| 402 | // For 0 < |x| <= 2^-27: |
| 403 | // 0 < (|x| - |tanh(x)|) / |x| ~ x^2 / 3 < 2^-55 < 2^-1 * ulp(x), |
| 404 | // so tanh(x) rounds to: |
| 405 | // - nextafter(x, 0) when rounding mode is FE_TOWARDZERO, or (x > 0 and |
| 406 | // FE_DOWNWARD), or (x < 0 and FE_UPWARD), |
| 407 | // - x otherwise, with FE_INEXACT raised for all x != 0 and FE_UNDERFLOW + |
| 408 | // errno = ERANGE raised when |tanh(x)| < 2^-1022. |
| 409 | #ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS |
| 410 | if (LIBC_UNLIKELY(x_abs <= 0x3e40'0000'0000'0000ULL)) |
| 411 | #endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS |
| 412 | { |
| 413 | if (LIBC_UNLIKELY(x_abs == 0)) |
| 414 | return x; |
| 415 | #ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE |
| 416 | double res = fputil::sub_multiply(x, x, 0x1.0p-55); |
| 417 | #else |
| 418 | // Avoid spurious FE_UNDERFLOW on x * -2^-55 when 2^-1022 < |x| < 2^-967, |
| 419 | // and preserve directed rounding for subnormal |x| <= 2^-1022: |
| 420 | double x_scaled = fputil::multiply_sub(x, y: 0x1.0p55, z: x); |
| 421 | double res = x_scaled * 0x1.0p-55; |
| 422 | #endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE |
| 423 | if (LIBC_UNLIKELY(x_abs <= 0x0010'0000'0000'0000ULL)) { |
| 424 | if (FPBits(res).abs().uintval() < 0x0010'0000'0000'0000ULL) { |
| 425 | fputil::set_errno_if_required(ERANGE); |
| 426 | fputil::raise_underflow_except_if_required<double>(); |
| 427 | } |
| 428 | } |
| 429 | return res; |
| 430 | } |
| 431 | |
| 432 | #ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS |
| 433 | using fputil::DoubleDouble; |
| 434 | |
| 435 | // 2^-27 < |x| < 2^-3: |
| 436 | // Split x = x_hi + x_lo into 14-bit x_hi and 39-bit x_lo so that: |
| 437 | // x2_hi = x_hi * x_hi (28 bits) |
| 438 | // x3_hi = x2_hi * x_hi (42 bits) |
| 439 | // y_hi = x3_hi * TANH_C0_HI_11 (42 + 11 = 53 bits) |
| 440 | // are all exact in double precision across all rounding modes (with or |
| 441 | // without FMA). |
| 442 | LIBC_BIT_CAST_CONSTEXPR_VAR double HI_MASK_14 = |
| 443 | cpp::bit_cast<double>(from: 0xffff'ff80'0000'0000ULL); |
| 444 | double x_hi = fputil::fp_and(a: x, b: HI_MASK_14); |
| 445 | double x_lo = x - x_hi; |
| 446 | double x2 = x * x; |
| 447 | double x2_hi = x_hi * x_hi; // exact (28 bits) |
| 448 | double x3_hi = x2_hi * x_hi; // exact (42 bits) |
| 449 | double y_hi = x3_hi * TANH_C0_HI_11; // exact (53 bits) |
| 450 | |
| 451 | // Cubic remainder: |
| 452 | // x3_lo ~ x^3 - x_hi^3 = x_lo * (x^2 + x * x_hi + x_hi^2) |
| 453 | double x2_mid = fputil::multiply_add(x, y: x_hi, z: x2); |
| 454 | double x3_sum = x2_mid + x2_hi; |
| 455 | double x3_lo = x_lo * x3_sum; |
| 456 | |
| 457 | // Polynomial remainder: |
| 458 | // x^3 * (TANH_C0_HI_11 + P_rem(x^2)) = y_hi + y_lo |
| 459 | // where: |
| 460 | // y_lo ~ x_hi^3 * P_rem(x^2) + x3_lo * (TANH_C0_HI_11 + P_rem(x^2)) |
| 461 | double p_rem = fputil::estrin_polyeval(x: x2, a: TANH_SMALL_P_COEFFS); |
| 462 | double p_full = TANH_C0_HI_11 + p_rem; |
| 463 | double y_lo_t = x3_lo * p_full; |
| 464 | double y_lo = fputil::multiply_add(x: x3_hi, y: p_rem, z: y_lo_t); |
| 465 | |
| 466 | // Leading sum r.hi + r.lo = x + y_hi (exact FastTwoSum in all rounding modes |
| 467 | // since |y_hi| < 2^-7 * |x|): |
| 468 | DoubleDouble r = fputil::exact_add(a: x, b: y_hi); |
| 469 | |
| 470 | // Error bound for Ziv's rounding test: |
| 471 | // The error of y_hi + y_lo is bounded by |y_hi| * 0x1.8p-54, and the rounding |
| 472 | // error of r.lo + y_lo is bounded by |x| * 2^-103. |
| 473 | double x_err = x * 0x1.0p-103; |
| 474 | double err = fputil::sub_multiply(x: x_err, y: y_hi, z: 0x1.8p-54); |
| 475 | |
| 476 | // Low tail bounds lo_{u,l} ~ r.lo + (y_lo +/- err): |
| 477 | double y_lo_u = y_lo + err; |
| 478 | double y_lo_l = y_lo - err; |
| 479 | double lo_u = r.lo + y_lo_u; |
| 480 | double lo_l = r.lo + y_lo_l; |
| 481 | |
| 482 | // Ziv's rounding test in the current rounding mode: |
| 483 | double upper = r.hi + lo_u; |
| 484 | double lower = r.hi + lo_l; |
| 485 | |
| 486 | if (LIBC_LIKELY(upper == lower)) |
| 487 | return upper; |
| 488 | |
| 489 | return tanh_small_accurate_128(x); |
| 490 | #endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS |
| 491 | } |
| 492 | |
| 493 | // Handle saturation, inf, and nan for tanh(x) when |
| 494 | // |x| >= 0x1.30fc1931f09cap+4: |
| 495 | LIBC_NOINLINE LIBC_INLINE double tanh_exceptional(double x) { |
| 496 | using FPBits = fputil::FPBits<double>; |
| 497 | FPBits xbits(x); |
| 498 | |
| 499 | if (xbits.is_nan()) |
| 500 | return x + 1.0; |
| 501 | |
| 502 | if (xbits.is_inf()) |
| 503 | return xbits.is_neg() ? -1.0 : 1.0; |
| 504 | |
| 505 | return xbits.is_neg() ? fputil::round_result_slightly_up(value_rn: -1.0) |
| 506 | : fputil::round_result_slightly_down(value_rn: 1.0); |
| 507 | } |
| 508 | |
| 509 | LIBC_ALWAYS_INLINE double tanh(double x) { |
| 510 | using FPBits = fputil::FPBits<double>; |
| 511 | FPBits xbits(x); |
| 512 | |
| 513 | uint64_t x_abs = xbits.abs().uintval(); |
| 514 | |
| 515 | // Bound constants (with biased exponent and mantissa bits): |
| 516 | #ifdef LIBC_MATH_HAS_SKIP_ACCURATE_PASS |
| 517 | // - Small bound: 2^-27 + ulp(2^-27) (tanh_eval handles |x| > 2^-27 directly |
| 518 | // with k = 0 and zero cancellation when |x| < log(2) / 4) |
| 519 | constexpr uint64_t SMALL_BOUND = 0x3e40'0000'0000'0001ULL; |
| 520 | #else |
| 521 | // - Small bound: 2^-3 = 0x1.0p-3 |
| 522 | constexpr uint64_t SMALL_BOUND = 0x3fc0'0000'0000'0000ULL; |
| 523 | #endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS |
| 524 | // - Saturation bound: |
| 525 | // > round(atanh(1 - 2^-54), D, RU) = 0x1.30fc1931f09cap+4 |
| 526 | // > round(atanh(1 - 2^-54), D, RD) = 0x1.30fc1931f09c9p+4 |
| 527 | // > round(tanh(0x1.30fc1931f09c9p+4), D, RN) = 0x1.fffffffffffffp-1 |
| 528 | // > round(tanh(0x1.30fc1931f09cap+4), D, RN) = 0x1.0p+0 |
| 529 | constexpr uint64_t OVERFLOW_BOUND = 0x4033'0fc1'931f'09caULL; |
| 530 | |
| 531 | if (LIBC_UNLIKELY(x_abs < SMALL_BOUND || x_abs >= OVERFLOW_BOUND)) { |
| 532 | if (x_abs < SMALL_BOUND) |
| 533 | return tanh_small_eval(x, x_abs); |
| 534 | |
| 535 | return tanh_exceptional(x); |
| 536 | } |
| 537 | |
| 538 | return tanh_eval(x); |
| 539 | } |
| 540 | |
| 541 | } // namespace double_eval |
| 542 | } // namespace math |
| 543 | } // namespace LIBC_NAMESPACE_DECL |
| 544 | |
| 545 | #endif // LLVM_LIBC_SRC___SUPPORT_MATH_TANH_DOUBLE_EVAL_H |
| 546 | |