| 1 | //===----------------------------------------------------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | /// |
| 9 | /// \file |
| 10 | /// 128-bit accurate paths for double-precision sinh and cosh functions. |
| 11 | /// |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
| 14 | #ifndef LLVM_LIBC_SRC___SUPPORT_MATH_SINHCOSH_ACCURATE_128_H |
| 15 | #define LLVM_LIBC_SRC___SUPPORT_MATH_SINHCOSH_ACCURATE_128_H |
| 16 | |
| 17 | #include "src/__support/FPUtil/FPBits.h" |
| 18 | #include "src/__support/FPUtil/dyadic_float.h" |
| 19 | #include "src/__support/FPUtil/polyeval.h" |
| 20 | #include "src/__support/common.h" |
| 21 | #include "src/__support/frac128.h" |
| 22 | #include "src/__support/frac64.h" |
| 23 | #include "src/__support/macros/attributes.h" |
| 24 | #include "src/__support/macros/config.h" |
| 25 | #include "src/__support/macros/optimization.h" |
| 26 | #include "src/__support/math/exp2_accurate_128.h" |
| 27 | #include "src/__support/uint128.h" |
| 28 | |
| 29 | namespace LIBC_NAMESPACE_DECL { |
| 30 | namespace math { |
| 31 | namespace double_eval { |
| 32 | |
| 33 | // Polynomial approximation for (sinh(x) - x) / x^3 on [-2^-2, 2^-2] as a |
| 34 | // degree-9 polynomial in x^2 in 128-bit fixed-point format: |
| 35 | // P(x^2) ~ (sinh(x) - x) / x^3 |
| 36 | // sinh(x) = x + x^3 * P(x^2). |
| 37 | // |
| 38 | // Minimax polynomial coefficients generated by Sollya with: |
| 39 | // > display = hexadecimal!; |
| 40 | // > prec = 500; |
| 41 | // > P = fpminimax((sinh(sqrt(x))/sqrt(x) - 1)/x, 9, [|128...|], |
| 42 | // [2^(-150), 2^(-4)], fixed); |
| 43 | // > for i from 0 to 9 do { |
| 44 | // c = coeff(P, i); |
| 45 | // c_int = round(c * 2^128, 256, RN); |
| 46 | // hi = floor(c_int * 2^(-64)); |
| 47 | // lo = c_int - hi * 2^64; |
| 48 | // print(" Frac128({", lo, "ULL,", hi, "ULL}), // c_", i); |
| 49 | // }; |
| 50 | // > dirtyinfnorm(sinh(x) - (x + x^3 * subst(P, x, x^2)), [-2^-2, 2^-2]); |
| 51 | // 0x1.1cb4...p-142 < 2^-141. |
| 52 | LIBC_INLINE_VAR constexpr Frac128 SINH_SMALL_POLY_128[10] = { |
| 53 | // lsb = 2^-128 |
| 54 | Frac128({0xaaaaaaaaaaaaaaabULL, 0x2aaaaaaaaaaaaaaaULL}), // c_0 |
| 55 | Frac128({0x2222222222222115ULL, 0x0222222222222222ULL}), // c_1 |
| 56 | Frac128({0x00d00d00d00de2b5ULL, 0x000d00d00d00d00dULL}), // c_2 |
| 57 | Frac128({0x671f558390d8d543ULL, 0x00002e3bc74aad8eULL}), // c_3 |
| 58 | Frac128({0x138e3f9d219de9cbULL, 0x0000006b99159fd5ULL}), // c_4 |
| 59 | Frac128({0x43684be86ba485efULL, 0x00000000b092309dULL}), // c_5 |
| 60 | Frac128({0x399dc020c9d4a0d9ULL, 0x0000000000d73f9fULL}), // c_6 |
| 61 | Frac128({0x3b819e88de5f1579ULL, 0x000000000000ca96ULL}), // c_7 |
| 62 | Frac128({0xa4d89e719113ac86ULL, 0x0000000000000097ULL}), // c_8 |
| 63 | Frac128({0x5c7ba5e9742ae90bULL, 0x0000000000000000ULL}), // c_9 |
| 64 | }; |
| 65 | |
| 66 | // 128-bit accurate path for sinh(x) when 2^-26 < |x| < 2^-2: |
| 67 | LIBC_NOINLINE LIBC_INLINE double sinh_small_accurate_128(double x) { |
| 68 | using FPBits = fputil::FPBits<double>; |
| 69 | FPBits xbits(x); |
| 70 | int exp_x = xbits.get_exponent(); // in [-26, -3] |
| 71 | |
| 72 | // Normalize |x|'s 53-bit mantissa to m_x in [1, 2) with lsb = 2^-63: |
| 73 | // |x| = 2^exp_x * m_x. |
| 74 | uint64_t mant_x = (xbits.get_mantissa() | (1ULL << 52)) << 11; // lsb = 2^-63 |
| 75 | |
| 76 | // Normalized mantissa of |x| and m_x^2: |
| 77 | // m_x_frac = m_x / 2 (lsb = 2^-127) |
| 78 | // m_x2 = mant_x^2 * 2^-126 (lsb = 2^-126) |
| 79 | Frac128 m_x_frac({0, mant_x}); // lsb = 2^-127 |
| 80 | UInt<128> m_x2 = UInt<64>(mant_x).ful_mul(other: UInt<64>(mant_x)); // lsb = 2^-126 |
| 81 | |
| 82 | // Since Frac128(m_x2.val) represents m_x^2 / 4 (lsb = 2^-128) and m_x_frac |
| 83 | // represents m_x / 2 (lsb = 2^-127), their Frac128 product (>> 128) |
| 84 | // represents m_x^3 / 8 with lsb = 2^-125: |
| 85 | Frac128 m_x3 = Frac128(m_x2.val) * m_x_frac; // lsb = 2^-125 |
| 86 | |
| 87 | // Scale m_x^2 by 2^(2 * exp_x) to obtain x^2 = 2^(2 * exp_x) * m_x^2 with |
| 88 | // lsb = 2^-128. Since exp_x is in [-26, -3], shift = -2 * exp_x - 2 is in |
| 89 | // [4, 50]: |
| 90 | unsigned shift = static_cast<unsigned>(-2 * exp_x - 2); // in [4, 50] |
| 91 | Frac128 x2_frac = Frac128(m_x2.val) >> shift; // lsb = 2^-128 |
| 92 | |
| 93 | // Evaluate P(x^2) ~ (sinh(x) - x) / x^3 in Horner form: |
| 94 | Frac128 p = |
| 95 | fputil::loop_polyeval(x: x2_frac, a: SINH_SMALL_POLY_128); // lsb = 2^-128 |
| 96 | |
| 97 | // Reconstruction of |sinh(x)| = |x| + |x|^3 * P(x^2): |
| 98 | // Since y_poly = m_x3 * p represents m_x^3 * P(x^2) with lsb = 2^-125, |
| 99 | // shifting right by shift = -2 * exp_x - 2 aligns: |
| 100 | // 2^(3 * exp_x) * m_x^3 * P(x^2) |
| 101 | // to the same scale as m_x_frac (lsb = 2^(exp_x - 127)): |
| 102 | Frac128 y_poly = m_x3 * p; // lsb = 2^-125 |
| 103 | Frac128 y_m = y_poly >> shift; // lsb = 2^(exp_x - 127) |
| 104 | |
| 105 | // Add |x| and |x|^3 * P(x^2), normalizing on carry out: |
| 106 | UInt<128> x_m = m_x_frac; // lsb = 2^(exp_x - 127) |
| 107 | int exp_res = exp_x - 127; |
| 108 | if (LIBC_UNLIKELY(x_m.add_overflow(y_m) != 0)) { |
| 109 | x_m = (x_m >> 1) | (UInt<128>(1) << 127); |
| 110 | ++exp_res; |
| 111 | } |
| 112 | |
| 113 | DFloat128 r; |
| 114 | r.sign = xbits.sign(); |
| 115 | r.exponent = exp_res; |
| 116 | r.mantissa = x_m; |
| 117 | return static_cast<double>(r); |
| 118 | } |
| 119 | |
| 120 | // Polynomial approximations for sinh(u * log(2)) and cosh(u * log(2)) - 1 on |
| 121 | // u in [0, 2^-5] in terms of v = 32 * u in [0, 1] and w = v^2 in [0, 1] in |
| 122 | // 128-bit fixed-point format (lsb = 2^-128): |
| 123 | // sinh(u * log(2)) = sinh(2^-5 * v * log(2)) ~ v * P_odd(w) |
| 124 | // cosh(u * log(2)) - 1 = cosh(2^-5 * v * log(2)) - 1 ~ w * P_even(w). |
| 125 | // |
| 126 | // For k >= 4, the coefficients c_k of P_odd(w) and P_even(w) satisfy |
| 127 | // c_k < 2^-64, so their high 64-bit words val[1] are identically zero and are |
| 128 | // stored as 64-bit low words in EXP2_SINH_W_LO64 and EXP2_COSH_W_LO64. |
| 129 | // |
| 130 | // Minimax polynomial coefficients generated by Sollya with: |
| 131 | // > prec = 500; |
| 132 | // > P_odd = fpminimax(sinh(2^-5 * sqrt(x) * log(2)) / sqrt(x), 6, [|128...|], |
| 133 | // [2^(-150), 1], fixed); |
| 134 | // > for i from 0 to 6 do { |
| 135 | // c = coeff(P_odd, i); |
| 136 | // c_int = round(c * 2^128, 256, RN); |
| 137 | // hi = floor(c_int * 2^(-64)); |
| 138 | // lo = c_int - hi * 2^64; |
| 139 | // print("Frac128({", lo, "ULL,", hi, "ULL}), // odd_", i); |
| 140 | // }; |
| 141 | // > dirtyinfnorm(sinh(2^-5 * sqrt(x) * log(2)) - sqrt(x) * P_odd, [0, 1]); |
| 142 | // 0x1.6276...p-132 < 2^-131. |
| 143 | LIBC_INLINE_VAR constexpr Frac128 EXP2_SINH_W_128[4] = { |
| 144 | // lsb = 2^-128 |
| 145 | Frac128({0x5e4f1d9cc01f97b5ULL, 0x058b90bfbe8e7bcdULL}), // odd_0 |
| 146 | Frac128({0xbf8b33a762bb32caULL, 0x00001c6b08d704a0ULL}), // odd_1 |
| 147 | Frac128({0x14ce6220e2fed2eaULL, 0x000000002bb0ffcfULL}), // odd_2 |
| 148 | Frac128({0xbfc588b0c686b280ULL, 0x0000000000001ffcULL}), // odd_3 |
| 149 | }; |
| 150 | |
| 151 | LIBC_INLINE_VAR constexpr Frac64 EXP2_SINH_W_LO64[3] = { |
| 152 | // lsb = 2^-128 (val[1] == 0) |
| 153 | Frac64(0x0da929e9caf3e033ULL), // odd_4 |
| 154 | Frac64(0x000003d1958e6b49ULL), // odd_5 |
| 155 | Frac64(0x0000000000c0b0b4ULL), // odd_6 |
| 156 | }; |
| 157 | |
| 158 | // > P_even = fpminimax((cosh(2^-5 * sqrt(x) * log(2)) - 1) / x, 6, [|128...|], |
| 159 | // [2^(-150), 1], fixed); |
| 160 | // > for i from 0 to 6 do { |
| 161 | // c = coeff(P_even, i); |
| 162 | // c_int = round(c * 2^128, 256, RN); |
| 163 | // hi = floor(c_int * 2^(-64)); |
| 164 | // lo = c_int - hi * 2^64; |
| 165 | // print("Frac128({", lo, "ULL,", hi, "ULL}), // even_", i); |
| 166 | // }; |
| 167 | // > dirtyinfnorm((cosh(2^-5 * sqrt(x) * log(2)) - 1) - x * P_even, [0, 1]); |
| 168 | // 0x1.a7d4...p-130 < 2^-129. |
| 169 | LIBC_INLINE_VAR constexpr Frac128 EXP2_COSH_W_128[4] = { |
| 170 | // lsb = 2^-128 |
| 171 | Frac128({0x54378b583764b9b0ULL, 0x000f5fdeffc162c7ULL}), // even_0 |
| 172 | Frac128({0x9cee539977c16a7dULL, 0x000000276556df74ULL}), // even_1 |
| 173 | Frac128({0x5f0d8f0ede96150bULL, 0x0000000000286122ULL}), // even_2 |
| 174 | Frac128({0x2c0223a5c823fdddULL, 0x0000000000000016ULL}), // even_3 |
| 175 | }; |
| 176 | |
| 177 | LIBC_INLINE_VAR constexpr Frac64 EXP2_COSH_W_LO64[3] = { |
| 178 | // lsb = 2^-128 (val[1] == 0) |
| 179 | Frac64(0x0007933d4562e2e0ULL), // even_4 |
| 180 | Frac64(0x00000001c3bd65faULL), // even_5 |
| 181 | Frac64(0x0000000000004bf6ULL), // even_6 |
| 182 | }; |
| 183 | |
| 184 | // Compute: |
| 185 | // sinh(x) = (e^x - e^-x) / 2 (when is_sinh is true) |
| 186 | // cosh(x) = (e^x + e^-x) / 2 (when is_sinh is false) |
| 187 | // in 128-bit precision using direct base-2 192-bit fixed-point range reduction |
| 188 | // for 2^-2 <= |x| < 0x1.633ce8fb9f87ep+9 (sinh) or |
| 189 | // 2^-27 < |x| < 0x1.633ce8fb9f87ep+9 (cosh). |
| 190 | LIBC_NOINLINE LIBC_INLINE double exp_pm_accurate_128(double x, bool is_sinh) { |
| 191 | using FPBits = fputil::FPBits<double>; |
| 192 | FPBits xbits(x); |
| 193 | int exp_x = xbits.get_exponent(); // in [-2, 9] for sinh, [-27, 9] for cosh |
| 194 | |
| 195 | // Normalize |x|'s 53-bit mantissa to [2^63, 2^64): |
| 196 | uint64_t mant_x = (xbits.get_mantissa() | (1ULL << 52)) << 11; |
| 197 | |
| 198 | // 1-limb x 3-limb product for |z| = |x| * log2(e): |
| 199 | // mant_x has lsb = 2^(exp_x - 63) |
| 200 | // LOG2_E_192.log2_b has lsb = 2^-191 |
| 201 | // p = mant_x * LOG2_E_192.log2_b has lsb = 2^(exp_x - 254) |
| 202 | UInt<256> p = LOG2_E_192.log2_b.ful_mul(other: UInt<64>(mant_x)); |
| 203 | |
| 204 | // Nearest-integer range reduction: |
| 205 | // k_p = round(|z| * 2^4), u = |z| - k_p * 2^-4 in [-2^-5, 2^-5] |
| 206 | // v = 32 * |u| in [0, 1) (lsb = 2^-128) |
| 207 | // w = v^2 in [0, 1) (lsb = 2^-128). |
| 208 | int k_p = 0; |
| 209 | bool u_is_neg = false; |
| 210 | Frac128 v; |
| 211 | if (LIBC_LIKELY(exp_x >= -6)) { |
| 212 | // Since p.val[3] has lsb = 2^(exp_x - 62), the 2^-5 bit of |z| is at bit |
| 213 | // shr = 57 - exp_x (in [48, 63]) of p.val[3]: |
| 214 | int shr = 57 - exp_x; |
| 215 | uint64_t k_twice = p.val[3] >> shr; // floor(|z| * 2^5), lsb = 2^-5 |
| 216 | k_p = static_cast<int>((k_twice + 1) >> 1); |
| 217 | |
| 218 | // Shift p left by shl = 64 - shr = exp_x + 7 (in [1, 16]) so that the top |
| 219 | // two limbs hold the fractional bits of 32 * (|z| - k_p * 2^-4) with |
| 220 | // lsb = 2^-128: |
| 221 | int shl = exp_x + 7; |
| 222 | uint64_t v_lo = (p.val[2] << shl) | (p.val[1] >> shr); |
| 223 | uint64_t v_hi = (p.val[3] << shl) | (p.val[2] >> shr); |
| 224 | Frac128 u_bits({v_lo, v_hi}); |
| 225 | |
| 226 | // When u < 0 (k_twice is odd), one's-complement negation (~u_bits) gives |
| 227 | // v = 32 * |u| in [0, 1) to within 2^-128: |
| 228 | u_is_neg = (k_twice & 1ULL) != 0; |
| 229 | v = u_is_neg ? ~u_bits : u_bits; // lsb = 2^-128 |
| 230 | } else { |
| 231 | // For cosh with exp_x in [-27, -7], |z| < 2^-5 so k_p = 0, u = |z| >= 0, |
| 232 | // and v = 32 * |z| is obtained by shifting the top 128 bits of p right by |
| 233 | // (-7 - exp_x): |
| 234 | size_t shift_v = static_cast<size_t>(-7 - exp_x); |
| 235 | Frac128 p_hi({p.val[2], p.val[3]}); |
| 236 | v = p_hi >> shift_v; // lsb = 2^-128 |
| 237 | } |
| 238 | Frac128 w = v * v; // lsb = 2^-128 |
| 239 | |
| 240 | // Decompose k_p = 16 * hi_p + idx_p and k_m = -k_p = 16 * hi_m + idx_m: |
| 241 | // (e^|x| -/+ e^-|x|) / 2 = 2^hi_p * (m_p * 2^u -/+ m_m * 2^-u) |
| 242 | // where: |
| 243 | // m_p = 2^(idx_p / 16) / 2 in [2^-1, 1) (lsb = 2^-127) |
| 244 | // m_m = 2^(hi_m - hi_p) * 2^(idx_m / 16) / 2 < 2^-1 (lsb = 2^-127). |
| 245 | int k_m = -k_p; |
| 246 | int hi_p = k_p >> 4; |
| 247 | unsigned idx_p = static_cast<unsigned>(k_p & 0x0f); |
| 248 | int hi_m = k_m >> 4; |
| 249 | unsigned idx_m = static_cast<unsigned>(k_m & 0x0f); |
| 250 | |
| 251 | Frac128 m_p = EXP2_MID_FRAC128[idx_p] >> 1; // lsb = 2^-127 |
| 252 | unsigned shift_m = static_cast<unsigned>(hi_p - hi_m + 1); |
| 253 | Frac128 m_m(0); |
| 254 | if (LIBC_LIKELY(shift_m < 128)) |
| 255 | m_m = EXP2_MID_FRAC128[idx_m] >> shift_m; // lsb = 2^-127 |
| 256 | |
| 257 | // Split 2^(+/- u) = cosh(u * log(2)) +/- sinh(u * log(2)): |
| 258 | // m_p * 2^u -/+ m_m * 2^-u |
| 259 | // = s * (1 + w * P_even(w)) + sign(u) * d * v * P_odd(w) |
| 260 | // where s = m_p -/+ m_m and d = m_p +/- m_m are both in (2^-2, 2) with |
| 261 | // lsb = 2^-127. |
| 262 | Frac128 s = is_sinh ? (m_p - m_m) : (m_p + m_m); // lsb = 2^-127 |
| 263 | Frac128 d = is_sinh ? (m_p + m_m) : (m_p - m_m); // lsb = 2^-127 |
| 264 | |
| 265 | Frac128 d_v = d * v; // lsb = 2^-127 |
| 266 | Frac128 s_w = s * w; // lsb = 2^-127 |
| 267 | |
| 268 | // High-degree Horner steps (k = 6, 5, 4 -> 3) using 64-bit multiplications |
| 269 | // since the high 64-bit words val[1] of c_4, c_5, c_6 are zero: |
| 270 | Frac64 w_hi = w.to_frac64(); // lsb = 2^-64 |
| 271 | Frac64 odd_lo = fputil::polyeval(x: w_hi, a: EXP2_SINH_W_LO64); |
| 272 | Frac64 even_lo = fputil::polyeval(x: w_hi, a: EXP2_COSH_W_LO64); |
| 273 | odd_lo = odd_lo * w_hi + Frac64(EXP2_SINH_W_128[3].val[0]); |
| 274 | even_lo = even_lo * w_hi + Frac64(EXP2_COSH_W_128[3].val[0]); |
| 275 | |
| 276 | // Low-degree Horner steps (k = 2, 1, 0) in 128-bit fixed-point: |
| 277 | Frac128 p_odd({odd_lo.val[0], EXP2_SINH_W_128[3].val[1]}); // lsb = 2^-128 |
| 278 | Frac128 p_even({even_lo.val[0], EXP2_COSH_W_128[3].val[1]}); // lsb = 2^-128 |
| 279 | LIBC_LOOP_NOUNROLL |
| 280 | for (int i = 2; i >= 0; --i) { |
| 281 | p_odd = p_odd * w + EXP2_SINH_W_128[i]; |
| 282 | p_even = p_even * w + EXP2_COSH_W_128[i]; |
| 283 | } |
| 284 | |
| 285 | // Reconstruct: |
| 286 | // r_m = s + s_w * P_even(w) + sign(u) * d_v * P_odd(w) |
| 287 | // in (2^-2, 2) with lsb = 2^-127: |
| 288 | Frac128 even_part = s + s_w * p_even; |
| 289 | Frac128 odd_part = d_v * p_odd; |
| 290 | Frac128 r_m = u_is_neg ? (even_part - odd_part) |
| 291 | : (even_part + odd_part); // lsb = 2^-127 |
| 292 | |
| 293 | // Normalize r_m and round 2^hi_p * r_m (with lsb = 2^(hi_p - 127)) to double |
| 294 | // in the current rounding mode: |
| 295 | Sign sign = is_sinh ? xbits.sign() : Sign::POS; |
| 296 | DFloat128 r(sign, hi_p - 127, r_m); |
| 297 | return static_cast<double>(r); |
| 298 | } |
| 299 | |
| 300 | } // namespace double_eval |
| 301 | } // namespace math |
| 302 | } // namespace LIBC_NAMESPACE_DECL |
| 303 | |
| 304 | #endif // LLVM_LIBC_SRC___SUPPORT_MATH_SINHCOSH_ACCURATE_128_H |
| 305 | |