| 1 | //===----------------------------------------------------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | /// |
| 9 | /// \file |
| 10 | /// Shared utilities for double-precision sinh and cosh functions. |
| 11 | /// |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
| 14 | #ifndef LLVM_LIBC_SRC___SUPPORT_MATH_SINHCOSH_UTILS_H |
| 15 | #define LLVM_LIBC_SRC___SUPPORT_MATH_SINHCOSH_UTILS_H |
| 16 | |
| 17 | #include "src/__support/CPP/bit.h" |
| 18 | #include "src/__support/FPUtil/FPBits.h" |
| 19 | #include "src/__support/FPUtil/double_double.h" |
| 20 | #include "src/__support/FPUtil/fp_bitwise.h" |
| 21 | #include "src/__support/FPUtil/multiply_add.h" |
| 22 | #include "src/__support/common.h" |
| 23 | #include "src/__support/macros/attributes.h" |
| 24 | #include "src/__support/macros/config.h" |
| 25 | #include "src/__support/macros/optimization.h" |
| 26 | #include "src/__support/macros/properties/cpu_features.h" |
| 27 | #include "src/__support/math/exp_utils.h" |
| 28 | |
| 29 | #ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS |
| 30 | #include "src/__support/math/sinhcosh_accurate_128.h" |
| 31 | #endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS |
| 32 | |
| 33 | namespace LIBC_NAMESPACE_DECL { |
| 34 | namespace math { |
| 35 | namespace double_eval { |
| 36 | |
| 37 | #ifdef LIBC_MATH_HAS_SKIP_ACCURATE_PASS |
| 38 | |
| 39 | // Splitting -log(2) ~ hi + lo with 35-bit-precision hi part so that |
| 40 | // kd * MLOG_2_HI is exact for |kd| < 2^18: |
| 41 | // > hi = round(-log(2), 35, RN); |
| 42 | // > lo = round(-log(2) - hi, D, RN); |
| 43 | LIBC_INLINE_VAR constexpr double MLOG_2_HI = -0x1.62e42fefcp-1; |
| 44 | LIBC_INLINE_VAR constexpr double MLOG_2_LO = 0x1.c610ca86c3899p-37; |
| 45 | |
| 46 | // Degree-4 minimax polynomials in u^2 for (cosh(u) - 1) / u^2 and |
| 47 | // (sinh(u) - u) / u^3 on [-log(2) / 2, log(2) / 2]: |
| 48 | // cosh(u) - 1 ~ u^2 * P_even(u^2) |
| 49 | // sinh(u) - u ~ u^3 * P_odd(u^2). |
| 50 | // |
| 51 | // Generated by Sollya with: |
| 52 | // > display = hexadecimal!; |
| 53 | // > prec = 500; |
| 54 | // > B = (log(2) / 2)^2; |
| 55 | // > P_even = fpminimax((cosh(sqrt(x)) - 1) / x, 4, [|D...|], [2^(-100), B]); |
| 56 | // > for i from 0 to 4 do print(coeff(P_even, i), ","); |
| 57 | // > dirtyinfnorm((cosh(sqrt(x)) - 1) - x * P_even, [0, B]); |
| 58 | // 0x1.c244...p-57 < 2^-56. |
| 59 | LIBC_INLINE_VAR constexpr double EXP_EVEN_N0_COEFFS[5] = { |
| 60 | 0x1.0000000000001p-1, // ~ 1/2! |
| 61 | 0x1.5555555553ce4p-5, // ~ 1/4! |
| 62 | 0x1.6c16c178a71cep-10, // ~ 1/6! |
| 63 | 0x1.a019b8c6ec68dp-16, // ~ 1/8! |
| 64 | 0x1.2891c21cf0c03p-22, // ~ 1/10! |
| 65 | }; |
| 66 | |
| 67 | // > P_odd = fpminimax((sinh(sqrt(x)) / sqrt(x) - 1) / x, 4, [|D...|], |
| 68 | // [2^(-100), B]); |
| 69 | // > for i from 0 to 4 do print(coeff(P_odd, i), ","); |
| 70 | // > dirtyinfnorm((sinh(sqrt(x)) - sqrt(x)) - x * sqrt(x) * P_odd, [0, B]); |
| 71 | // 0x1.4882...p-62 < 2^-61. |
| 72 | LIBC_INLINE_VAR constexpr double EXP_ODD_N0_COEFFS[5] = { |
| 73 | 0x1.5555555555556p-3, // ~ 1/3! |
| 74 | 0x1.111111111070cp-7, // ~ 1/5! |
| 75 | 0x1.a01a01a901652p-13, // ~ 1/7! |
| 76 | 0x1.71de0a3fd7069p-19, // ~ 1/9! |
| 77 | 0x1.af3effc3463e9p-26, // ~ 1/11! |
| 78 | }; |
| 79 | |
| 80 | // Tableless (N = 0) evaluation of: |
| 81 | // sinh(x) = (e^x - e^-x) / 2 (when is_sinh is true) |
| 82 | // cosh(x) = (e^x + e^-x) / 2 (when is_sinh is false, with x >= 0) |
| 83 | // for 2^-26 < |x| < 0x1.633ce8fb9f87ep+9 (sinh) or |
| 84 | // 2^-27 < x < 0x1.633ce8fb9f87ep+9 (cosh). |
| 85 | // |
| 86 | // Range reduction: |
| 87 | // Let |x| = k * log(2) + u, where: |
| 88 | // k = round(|x| * log2(e)) |
| 89 | // |u| <= log(2) / 2 |
| 90 | // then: |
| 91 | // e^|x| = 2^k * e^u |
| 92 | // e^-|x| = 2^-k * e^-u = 2^k * 2^(-2 * k) * e^-u. |
| 93 | // |
| 94 | // Factoring out 2^(k - 1) and separating e^(+/- u) = cosh(u) +/- sinh(u): |
| 95 | // (e^|x| -/+ e^-|x|) / 2 = 2^(k - 1) * (e^u + m_m * e^-u) |
| 96 | // = 2^(k - 1) * (s * cosh(u) + d * sinh(u)) |
| 97 | // where: |
| 98 | // m_m = (-/+ 1) * 2^(-2 * k) |
| 99 | // s = 1 + m_m |
| 100 | // d = 1 - m_m. |
| 101 | // |
| 102 | // Key properties across the domain: |
| 103 | // - For |x| < 0x1.633ce8fb9f87ep+9, k is in [0, 1025] so k - 1 is in |
| 104 | // [-1, 1024]. When k = 1025, u < 0 keeps: |
| 105 | // r = e^u + m_m * e^-u < 1.0, |
| 106 | // so 2^1024 * r stays in the normal range and scale_exp2(r, k - 1) handles |
| 107 | // the entire domain without a separate top-octave branch. |
| 108 | // |
| 109 | // - For |x| < log(2) / 2, k = 0 gives u = |x| (exact, t2 = 0) and m_m = -/+ 1: |
| 110 | // * For sinh (m_m = -1): s = 0 and d = 2, reducing the evaluation to: |
| 111 | // 2^-1 * 2 * (u + u^3 * P_odd(u^2)) = x + x^3 * P_odd(x^2) |
| 112 | // with no cancellation down to |x| = 2^-26. |
| 113 | // * For cosh (m_m = +1): s = 2 and d = 0, reducing to: |
| 114 | // 1 + u^2 * P_even(u^2). |
| 115 | // |
| 116 | // - For 1 <= k <= 26, 2^(-2 * k) is in [2^-52, 2^-2], so both: |
| 117 | // s = 1.0 + m_m |
| 118 | // d = 1.0 - m_m |
| 119 | // are exact in all rounding modes. For k >= 27: |
| 120 | // |m_m| <= 2^-54 < 0.5 * ulp(1.0), |
| 121 | // so clamping m_m to 0.0 avoids a 1-ULP rounding error in directed rounding |
| 122 | // modes. |
| 123 | LIBC_MATH_SMALL_TABLES_INLINE double exp_pm_eval(double x, bool is_sinh) { |
| 124 | using FPBits = fputil::FPBits<double>; |
| 125 | FPBits xbits(x); |
| 126 | double x_a = xbits.abs().get_val(); |
| 127 | |
| 128 | // Using ROUND_BIAS, ROUND_SHIFT, and ROUND_MASK (see exp_utils.h), we compute |
| 129 | // k_biased = fma(x_a, LOG2_E_DD.hi, ROUND_BIAS) and extract in parallel: |
| 130 | // kd = (k_biased & ROUND_MASK) - ROUND_SHIFT |
| 131 | // k = (k_bits >> 32) & 0x3'ffff. |
| 132 | double k_biased = fputil::multiply_add(x_a, LOG2_E_DD.hi, ROUND_BIAS); |
| 133 | double kd = fputil::fp_and(k_biased, ROUND_MASK) - ROUND_SHIFT; |
| 134 | int k = static_cast<int>(cpp::bit_cast<uint64_t>(k_biased) >> 32) & 0x3'ffff; |
| 135 | |
| 136 | // Secondary scale factor: |
| 137 | // m_m = (-/+ 1) * 2^(-2 * k) for k < 27 |
| 138 | // m_m = 0.0 for k >= 27 (where 2^(-2 * k) <= 2^-54). |
| 139 | // |
| 140 | // Subtracting (1 << 11) = 0x800 (for sinh) sets bit 11 of the 12-bit |
| 141 | // two's-complement exponent field, flipping the sign bit of m_m when shifted |
| 142 | // left by 52 in scale_exp2: |
| 143 | int sign_shift = static_cast<int>(is_sinh) << 11; |
| 144 | int exp_diff = -2 * k - sign_shift; |
| 145 | double m_m = (k < 27) ? scale_exp2(1.0, exp_diff) : 0.0; |
| 146 | |
| 147 | // Reduced argument: |
| 148 | // u = (x_a + kd * MLOG_2_HI) + kd * MLOG_2_LO. |
| 149 | // Since MLOG_2_HI has 35 significant bits and |kd| <= 1025, |
| 150 | // t1 = fma(kd, MLOG_2_HI, x_a) |
| 151 | // is exact in all rounding modes with or without FMA. |
| 152 | double t1 = fputil::multiply_add(kd, MLOG_2_HI, x_a); // exact |
| 153 | double t2 = kd * MLOG_2_LO; |
| 154 | double u = t1 + t2; |
| 155 | double u2 = u * u; |
| 156 | double u4 = u2 * u2; |
| 157 | |
| 158 | // Evaluate degree-4 even and odd minimax polynomials P_even(u^2) and |
| 159 | // P_odd(u^2) in parallel using Estrin's scheme: |
| 160 | // cosh(u) ~ 1 + u^2 * P_even(u^2) |
| 161 | // sinh(u) ~ t1 + (t2 + u^3 * P_odd(u^2)) |
| 162 | |
| 163 | // First Estrin layer: c_0 + u^2 * c_1 and c_2 + u^2 * c_3 |
| 164 | double e01 = |
| 165 | fputil::multiply_add(u2, EXP_EVEN_N0_COEFFS[1], EXP_EVEN_N0_COEFFS[0]); |
| 166 | double o01 = |
| 167 | fputil::multiply_add(u2, EXP_ODD_N0_COEFFS[1], EXP_ODD_N0_COEFFS[0]); |
| 168 | double e23 = |
| 169 | fputil::multiply_add(u2, EXP_EVEN_N0_COEFFS[3], EXP_EVEN_N0_COEFFS[2]); |
| 170 | double o23 = |
| 171 | fputil::multiply_add(u2, EXP_ODD_N0_COEFFS[3], EXP_ODD_N0_COEFFS[2]); |
| 172 | |
| 173 | // Second Estrin layer: (c_2 + u^2 * c_3) + u^4 * c_4 |
| 174 | double e24 = fputil::multiply_add(u4, EXP_EVEN_N0_COEFFS[4], e23); |
| 175 | double o24 = fputil::multiply_add(u4, EXP_ODD_N0_COEFFS[4], o23); |
| 176 | |
| 177 | // Final polynomial values P_even(u^2) and P_odd(u^2): |
| 178 | double p_even = fputil::multiply_add(u4, e24, e01); |
| 179 | double p_odd = fputil::multiply_add(u4, o24, o01); |
| 180 | |
| 181 | // Since m_m is either 0 or +/- 2^(-2 * k) with 0 <= k <= 26, both s and d are |
| 182 | // exact in all rounding modes: |
| 183 | double s = 1.0 + m_m; // exact |
| 184 | double d = 1.0 - m_m; // exact |
| 185 | |
| 186 | // Apply sign(x) to the exact weights s and d before accumulating the |
| 187 | // polynomial so every subsequent rounding operation rounds in the active |
| 188 | // rounding direction for negative x as well (for cosh, the caller passes |
| 189 | // x >= 0 so sign_x == +0.0): |
| 190 | double sign_x = fputil::fp_xor(x, x_a); |
| 191 | s = fputil::fp_xor(s, sign_x); |
| 192 | d = fputil::fp_xor(d, sign_x); |
| 193 | |
| 194 | // Accumulate from smallest to largest magnitude before adding the leading |
| 195 | // constant s. Adding s last avoids a double rounding at the leading ULP |
| 196 | // position of s when |d * t1| (< 2^-1) and |s * u^2 * P_even(u^2)| (< 2^-4) |
| 197 | // are added: |
| 198 | double d_t2 = d * t2; |
| 199 | double s_u2 = s * u2; |
| 200 | double d_u = d * u; |
| 201 | double d_u3 = d_u * u2; |
| 202 | |
| 203 | // even_lo ~ s * u^2 * P_even(u^2) + d * t2 |
| 204 | double even_lo = fputil::multiply_add(s_u2, p_even, d_t2); |
| 205 | // poly_lo ~ d * u^3 * P_odd(u^2) + even_lo |
| 206 | double poly_lo = fputil::multiply_add(d_u3, p_odd, even_lo); |
| 207 | // poly ~ d * t1 + poly_lo |
| 208 | double poly = fputil::multiply_add(d, t1, poly_lo); |
| 209 | |
| 210 | // r ~ s * cosh(u) + d * sinh(u) |
| 211 | double r = s + poly; |
| 212 | return scale_exp2(r, k - 1); |
| 213 | } |
| 214 | |
| 215 | #else // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS |
| 216 | |
| 217 | // Fixed-point splitting constants (ulp = 2^-22, 2^-29) for exact multiplication |
| 218 | // without Veltkamp splitting: |
| 219 | LIBC_INLINE_VAR constexpr double SPLIT_22 = 0x1.8p30; // 1.5 * 2^(-22 + 52) |
| 220 | LIBC_INLINE_VAR constexpr double SPLIT_29 = 0x1.8p23; // 1.5 * 2^(-29 + 52) |
| 221 | |
| 222 | // Evaluate: |
| 223 | // sinh(x) = (e^x - e^-x) / 2 (when is_sinh is true) |
| 224 | // cosh(x) = (e^x + e^-x) / 2 (when is_sinh is false, with x >= 0) |
| 225 | // in double-double precision for 2^-2 <= |x| < 0x1.633ce8fb9f87ep+9 (sinh) or |
| 226 | // 2^-27 < x < 0x1.633ce8fb9f87ep+9 (cosh), falling back to |
| 227 | // exp_pm_accurate_128(x, is_sinh) when Ziv's rounding test fails. |
| 228 | // |
| 229 | // Range reduction: |
| 230 | // Let |x| = k_p * (log(2) / 2^7) + u |
| 231 | // in which: |
| 232 | // k_p = round(|x| * 2^7 * log2(e)), k_m = -k_p |
| 233 | // hi_p = k_p >> 7, idx_p = k_p & 0x7f |
| 234 | // hi_m = k_m >> 7, idx_m = k_m & 0x7f |
| 235 | // |u| <= 2^-8 * log(2) |
| 236 | // then: |
| 237 | // e^|x| = 2^hi_p * 2^(idx_p / 2^7) * e^u |
| 238 | // e^-|x| = 2^hi_m * 2^(idx_m / 2^7) * e^-u |
| 239 | // = 2^hi_p * (2^(hi_m - hi_p) * 2^(idx_m / 2^7)) * e^-u. |
| 240 | // |
| 241 | // Factoring out 2^(hi_p - 1): |
| 242 | // (e^|x| -/+ e^-|x|) / 2 = 2^(hi_p - 1) * (m_p * e^u + m_m * e^-u) |
| 243 | // where: |
| 244 | // m_p = 2^(idx_p / 2^7) |
| 245 | // m_m = (-/+ 1) * 2^(hi_m - hi_p) * 2^(idx_m / 2^7). |
| 246 | // |
| 247 | // For |x| < 0x1.633ce8fb9f87ep+9, hi_p is in [0, 1025] so hi_p - 1 is in |
| 248 | // [-1, 1024]. When hi_p == 1025, idx_p == 0 and u < 0, which keeps: |
| 249 | // r = m_p * e^u + m_m * e^-u < 1.0 |
| 250 | // (biased exponent 1022). Adding hi_p - 1 = 1024 in scale_exp2(r, hi_p - 1) |
| 251 | // produces biased exponent 2046, handling the top octave |
| 252 | // [1024 * log(2), 1025 * log(2)) without a separate branch. |
| 253 | LIBC_MATH_SMALL_TABLES_INLINE double exp_pm_eval(double x, bool is_sinh) { |
| 254 | using FPBits = fputil::FPBits<double>; |
| 255 | FPBits xbits(x); |
| 256 | double x_a = xbits.abs().get_val(); |
| 257 | |
| 258 | constexpr double LOG2_E_SCALE = EXP_SCALE * LOG2_E_DD.hi; |
| 259 | // Using ROUND_BIAS, ROUND_SHIFT, and ROUND_MASK (see exp_utils.h), we compute |
| 260 | // k_biased = fma(x_a, LOG2_E_SCALE, ROUND_BIAS) and extract in parallel: |
| 261 | // kd = (k_biased & ROUND_MASK) - ROUND_SHIFT |
| 262 | // k_p = (k_bits >> 32) & 0x3'ffff. |
| 263 | double k_biased = fputil::multiply_add(x: x_a, y: LOG2_E_SCALE, z: ROUND_BIAS); |
| 264 | double kd = fputil::fp_and(a: k_biased, b: ROUND_MASK) - ROUND_SHIFT; |
| 265 | int k_p = |
| 266 | static_cast<int>(cpp::bit_cast<uint64_t>(from: k_biased) >> 32) & 0x3'ffff; |
| 267 | int k_m = -k_p; |
| 268 | |
| 269 | int hi_p = k_p >> EXP_N_BITS; |
| 270 | unsigned idx_p = static_cast<unsigned>(k_p & EXP_MASK); |
| 271 | |
| 272 | int hi_m = k_m >> EXP_N_BITS; |
| 273 | unsigned idx_m = static_cast<unsigned>(k_m & EXP_MASK); |
| 274 | |
| 275 | // Secondary exponent offset: |
| 276 | // exp_diff = hi_m - hi_p |
| 277 | // clamped at -100 (where e^-|x| <= 2^-99 * e^|x| << SINH_ERR_D) to avoid |
| 278 | // underflow in m_m: |
| 279 | int exp_diff = hi_m - hi_p; |
| 280 | int clamped_diff = (exp_diff > -100) ? exp_diff : -100; |
| 281 | |
| 282 | // Subtracting (1 << 11) = 0x800 (for sinh) sets bit 11 of the 12-bit |
| 283 | // two's-complement exponent field, flipping the sign bit of m_m when shifted |
| 284 | // left by 52 in scale_exp2: |
| 285 | int sign_shift = static_cast<int>(is_sinh) << 11; |
| 286 | int scale_m = clamped_diff - sign_shift; |
| 287 | |
| 288 | DoubleDouble m_p = EXP2_MID[idx_p]; |
| 289 | // Scale m_m = EXP2_MID[idx_m] by (-/+ 1) * 2^clamped_diff: |
| 290 | // - Since m_m.hi is in [1, 2), we use scale_exp2(m_m.hi, scale_m) directly. |
| 291 | // - Since EXP2_MID[0].lo == 0.0, we scale m_m.lo by multiplying by: |
| 292 | // s_m = scale_exp2(1.0, scale_m). |
| 293 | double s_m = scale_exp2(r: 1.0, hi: scale_m); |
| 294 | DoubleDouble m_m = EXP2_MID[idx_m]; |
| 295 | m_m.lo *= s_m; |
| 296 | m_m.hi = scale_exp2(r: m_m.hi, hi: scale_m); |
| 297 | |
| 298 | // Reduced argument: |
| 299 | // u = (x_a + kd * MLOG_2_EXP2_HI) + kd * MLOG_2_EXP2_MID. |
| 300 | // Since MLOG_2_EXP2_HI has 35 significant bits and |kd| < 2^18, |
| 301 | // t1 = fma(kd, MLOG_2_EXP2_HI, x_a) |
| 302 | // is exact in all rounding modes with or without FMA. |
| 303 | double t1 = fputil::multiply_add(x: kd, y: MLOG_2_EXP2_HI, z: x_a); // exact |
| 304 | double u = fputil::multiply_add(x: kd, y: MLOG_2_EXP2_MID, z: t1); |
| 305 | double u2 = u * u; |
| 306 | |
| 307 | // Separate e^(+/-u) into even and odd parts: |
| 308 | // e^u ~ (1 + u^2 * P_even(u^2)) + (u + u^3 * P_odd(u^2)) |
| 309 | // e^-u ~ (1 + u^2 * P_even(u^2)) - (u + u^3 * P_odd(u^2)) |
| 310 | // where: |
| 311 | // P_even(u^2) = p_0 + p_2 * u^2 |
| 312 | // P_odd(u^2) = p_1 + p_3 * u^2 |
| 313 | // Then: |
| 314 | // m_p * e^u + m_m * e^-u |
| 315 | // ~ (m_p + m_m) * (1 + u^2 * P_even(u^2)) |
| 316 | // + (m_p - m_m) * (u + u^3 * P_odd(u^2)). |
| 317 | double p_even = fputil::multiply_add(x: u2, y: EXP_P_COEFFS[2], z: EXP_P_COEFFS[0]); |
| 318 | double p_odd = fputil::multiply_add(x: u2, y: EXP_P_COEFFS[3], z: EXP_P_COEFFS[1]); |
| 319 | |
| 320 | #ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE |
| 321 | // Decompose m_p * e^u + m_m * e^-u ~ r_hi + r_lo: |
| 322 | // s.hi + s.lo = m_p.hi + m_m.hi (exact FastTwoSum) |
| 323 | // d.hi + d.lo = m_p.hi - m_m.hi (exact FastTwoSum) |
| 324 | // r_hi + r_err = s.hi + d.hi * t1 |
| 325 | // r_lo ~ r_err + (s.lo + (m_p.lo - m_m.lo + d.lo) * t1) |
| 326 | // + ((m_p.lo + m_m.lo) + s.hi * u^2 * P_even(u^2)) |
| 327 | // + d.hi * (kd * MLOG_2_EXP2_MID + u^3 * P_odd(u^2)). |
| 328 | // |
| 329 | // For |x| >= 2^-2, k_p >= 46 so hi_p >= 0 and hi_m <= -1, hence |
| 330 | // m_p.hi >= 1.0 > |m_m.hi|. By Sterbenz's lemma, both (s.hi - m_p.hi) and |
| 331 | // (m_p.hi - d.hi) are exact in all rounding modes. |
| 332 | DoubleDouble s = fputil::exact_add(m_p.hi, m_m.hi); |
| 333 | DoubleDouble d = fputil::exact_sub(m_p.hi, m_m.hi); |
| 334 | |
| 335 | // Leading term r_hi = round(s.hi + d.hi * t1) and its rounding error r_err: |
| 336 | // Since |d.hi * t1| <= 2^-7 * log(2) < 2^-7 and s.hi > 2^-2, (s.hi - r_hi) is |
| 337 | // exact by Sterbenz's lemma. |
| 338 | double r_hi = fputil::multiply_add(d.hi, t1, s.hi); |
| 339 | double r_t = s.hi - r_hi; // exact |
| 340 | double r_err = fputil::multiply_add(d.hi, t1, r_t); |
| 341 | |
| 342 | // Odd polynomial and low reduced-argument correction: |
| 343 | // odd_lo ~ (u - t1) + u^3 * P_odd(u^2) = kd * MLOG_2_EXP2_MID + u^3 * p_odd |
| 344 | double t2 = kd * MLOG_2_EXP2_MID; |
| 345 | double u3 = u * u2; |
| 346 | double odd_lo = fputil::multiply_add(u3, p_odd, t2); |
| 347 | |
| 348 | // Combine table low parts and polynomial terms in parallel with the |
| 349 | // FastTwoSum residuals (s.lo, d.lo) and r_hi rounding error (r_err): |
| 350 | // mp_lo ~ m_p.lo * (1 + t1) |
| 351 | // mm_lo ~ m_m.lo * (1 - t1) |
| 352 | // even_lo ~ mp_lo + (s.hi * u^2) * P_even(u^2) |
| 353 | // poly_lo ~ even_lo + d.hi * odd_lo |
| 354 | double mp_lo = fputil::multiply_add(m_p.lo, t1, m_p.lo); |
| 355 | double mm_lo = fputil::sub_multiply(m_m.lo, m_m.lo, t1); |
| 356 | double s_u2 = s.hi * u2; |
| 357 | double even_lo = fputil::multiply_add(s_u2, p_even, mp_lo); |
| 358 | double poly_lo = fputil::multiply_add(d.hi, odd_lo, even_lo); |
| 359 | |
| 360 | // Total linear error from FastTwoSum residuals and r_hi rounding: |
| 361 | // sd_lo ~ s.lo + d.lo * t1 |
| 362 | // err_tot ~ r_err + sd_lo |
| 363 | double sd_lo = fputil::multiply_add(d.lo, t1, s.lo); |
| 364 | double err_tot = r_err + sd_lo; |
| 365 | |
| 366 | // Apply sign(x) to r_hi, mm_lo, poly_lo, and err_tot before the final Ziv |
| 367 | // test additions so that (r_hi + lo_u) and (r_hi + lo_l) round in the active |
| 368 | // rounding direction for negative inputs (for cosh, the caller passes x >= 0 |
| 369 | // so sign_x == +0.0): |
| 370 | double sign_x = fputil::fp_xor(x, x_a); |
| 371 | r_hi = fputil::fp_xor(r_hi, sign_x); |
| 372 | mm_lo = fputil::fp_xor(mm_lo, sign_x); |
| 373 | poly_lo = fputil::fp_xor(poly_lo, sign_x); |
| 374 | err_tot = fputil::fp_xor(err_tot, sign_x); |
| 375 | |
| 376 | // Fold +/-SINH_ERR_D into mm_lo off the critical path so tail_{u,l} and |
| 377 | // err_tot finish in parallel: |
| 378 | constexpr double SINH_ERR_D = 0x1.4p-64; |
| 379 | double mm_lo_u = mm_lo + SINH_ERR_D; |
| 380 | double mm_lo_l = mm_lo - SINH_ERR_D; |
| 381 | double tail_u = poly_lo + mm_lo_u; |
| 382 | double tail_l = poly_lo + mm_lo_l; |
| 383 | |
| 384 | double lo_u = err_tot + tail_u; |
| 385 | double lo_l = err_tot + tail_l; |
| 386 | #else // LIBC_TARGET_CPU_HAS_FMA_DOUBLE |
| 387 | // Fixed-point grid splitting for exact leading reconstruction in all rounding |
| 388 | // modes without FMA: |
| 389 | // lsb(mp_hi), lsb(mm_hi) >= 2^-22 => lsb(s_hi), lsb(d_hi) >= 2^-22 |
| 390 | // lsb(u_hi) >= 2^-29 |
| 391 | // Since s_hi = mp_hi + mm_hi > 2^-2, both s_hi, d_hi, and |
| 392 | // r_hi = s_hi + d_hi * u_hi (with lsb >= 2^-51) |
| 393 | // are exact in double. |
| 394 | |
| 395 | // Split u = u_hi + u_lo with lsb(u_hi) >= 2^-29 and |u_lo| <= 2^-30: |
| 396 | double u_hi_t = t1 + SPLIT_29; |
| 397 | double u_hi = u_hi_t - SPLIT_29; // exact, lsb = 2^-29 |
| 398 | double u_lo_t = t1 - u_hi; // exact |
| 399 | double u_lo = fputil::multiply_add(x: kd, y: MLOG_2_EXP2_MID, z: u_lo_t); |
| 400 | |
| 401 | // Split m_p = mp_hi + mp_lo and m_m = mm_hi + mm_lo with lsb >= 2^-22: |
| 402 | double mp_hi_t = m_p.hi + SPLIT_22; |
| 403 | double mp_hi = mp_hi_t - SPLIT_22; // exact, lsb = 2^-22 |
| 404 | double mp_lo_t = m_p.hi - mp_hi; // exact |
| 405 | double mp_lo = mp_lo_t + m_p.lo; |
| 406 | |
| 407 | double mm_hi_t = m_m.hi + SPLIT_22; |
| 408 | double mm_hi = mm_hi_t - SPLIT_22; // exact, lsb = 2^-22 |
| 409 | double mm_lo_t = m_m.hi - mm_hi; // exact |
| 410 | double mm_lo = mm_lo_t + m_m.lo; |
| 411 | |
| 412 | // Exact leading sum, difference, and linear term r_hi = s_hi + d_hi * u_hi: |
| 413 | double s_hi = mp_hi + mm_hi; // exact, lsb = 2^-22 |
| 414 | double d_hi = mp_hi - mm_hi; // exact, lsb = 2^-22 |
| 415 | double r_hi = fputil::multiply_add(x: d_hi, y: u_hi, z: s_hi); // exact, lsb = 2^-51 |
| 416 | |
| 417 | // Low parts of the split table values and their linear contribution: |
| 418 | // s_lo = mp_lo + mm_lo, d_lo = mp_lo - mm_lo |
| 419 | // m_lo ~ s_lo + d_lo * u_hi |
| 420 | double s_lo = mp_lo + mm_lo; |
| 421 | double d_lo = mp_lo - mm_lo; |
| 422 | double m_lo = fputil::multiply_add(x: d_lo, y: u_hi, z: s_lo); |
| 423 | |
| 424 | // Full-precision leading weights for the quadratic and cubic terms: |
| 425 | double s_m_hi = m_p.hi + m_m.hi; |
| 426 | double d_m_hi = m_p.hi - m_m.hi; |
| 427 | |
| 428 | // Apply sign(x) before the final polynomial accumulation so that (r_hi + |
| 429 | // lo_u) and (r_hi + lo_l) round in the active rounding direction for negative |
| 430 | // x: |
| 431 | double sign_x = fputil::fp_xor(a: x, b: x_a); |
| 432 | r_hi = fputil::fp_xor(a: r_hi, b: sign_x); |
| 433 | s_m_hi = fputil::fp_xor(a: s_m_hi, b: sign_x); |
| 434 | d_m_hi = fputil::fp_xor(a: d_m_hi, b: sign_x); |
| 435 | m_lo = fputil::fp_xor(a: m_lo, b: sign_x); |
| 436 | |
| 437 | // Combine even and odd polynomial terms with m_lo: |
| 438 | // even_lo ~ m_lo + s_m_hi * u^2 * P_even(u^2) |
| 439 | // odd_lo ~ u_lo + u^3 * P_odd(u^2) |
| 440 | // r_lo ~ even_lo + d_m_hi * odd_lo |
| 441 | double u2_peven = u2 * p_even; |
| 442 | double even_lo = fputil::multiply_add(x: s_m_hi, y: u2_peven, z: m_lo); |
| 443 | double u3 = u * u2; |
| 444 | double odd_lo = fputil::multiply_add(x: u3, y: p_odd, z: u_lo); |
| 445 | double r_lo = fputil::multiply_add(x: d_m_hi, y: odd_lo, z: even_lo); |
| 446 | |
| 447 | // Total approximation and rounding error is bounded by SINH_ERR_D = |
| 448 | // 0x1.4p-64: |
| 449 | constexpr double SINH_ERR_D = 0x1.4p-64; |
| 450 | double lo_u = r_lo + SINH_ERR_D; |
| 451 | double lo_l = r_lo - SINH_ERR_D; |
| 452 | #endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE |
| 453 | |
| 454 | // Ziv's rounding test in the current rounding mode: |
| 455 | double upper = r_hi + lo_u; |
| 456 | double lower = r_hi + lo_l; |
| 457 | |
| 458 | if (LIBC_LIKELY(upper == lower)) |
| 459 | return scale_exp2(r: upper, hi: hi_p - 1); |
| 460 | |
| 461 | return exp_pm_accurate_128(x, is_sinh); |
| 462 | } |
| 463 | |
| 464 | #endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS |
| 465 | |
| 466 | } // namespace double_eval |
| 467 | } // namespace math |
| 468 | } // namespace LIBC_NAMESPACE_DECL |
| 469 | |
| 470 | #endif // LLVM_LIBC_SRC___SUPPORT_MATH_SINHCOSH_UTILS_H |
| 471 | |