| 1 | //===----------------------------------------------------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | /// |
| 9 | /// \file |
| 10 | /// 128-bit accurate paths for double-precision tanh function. |
| 11 | /// |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
| 14 | #ifndef LLVM_LIBC_SRC___SUPPORT_MATH_TANH_ACCURATE_128_H |
| 15 | #define LLVM_LIBC_SRC___SUPPORT_MATH_TANH_ACCURATE_128_H |
| 16 | |
| 17 | #include "src/__support/FPUtil/FPBits.h" |
| 18 | #include "src/__support/FPUtil/dyadic_float.h" |
| 19 | #include "src/__support/FPUtil/polyeval.h" |
| 20 | #include "src/__support/common.h" |
| 21 | #include "src/__support/frac128.h" |
| 22 | #include "src/__support/frac64.h" |
| 23 | #include "src/__support/macros/attributes.h" |
| 24 | #include "src/__support/macros/config.h" |
| 25 | #include "src/__support/macros/optimization.h" |
| 26 | #include "src/__support/math/exp2_accurate_128.h" |
| 27 | #include "src/__support/uint128.h" |
| 28 | |
| 29 | namespace LIBC_NAMESPACE_DECL { |
| 30 | namespace math { |
| 31 | namespace double_eval { |
| 32 | |
| 33 | // Polynomial approximation for (x - tanh(x)) / x^3 on [-2^-3, 2^-3] in terms |
| 34 | // of w = 64 * x^2 in [0, 1] as a degree-12 alternating polynomial in 128-bit |
| 35 | // fixed-point format (lsb = 2^-128): |
| 36 | // P(w) = c_0 - c_1 * w + c_2 * w^2 - ... + c_12 * w^12 |
| 37 | // tanh(x) ~ x - x^3 * P(64 * x^2). |
| 38 | // |
| 39 | // For k >= 9, the scaled coefficients c_k satisfy c_k < 2^-64, so their high |
| 40 | // 64-bit words val[1] are identically zero and are stored as 64-bit low words |
| 41 | // in TANH_SMALL_W_LO64. |
| 42 | // |
| 43 | // Minimax polynomial coefficients generated by Sollya with: |
| 44 | // > display = hexadecimal!; |
| 45 | // > prec = 500; |
| 46 | // > P = fpminimax((1 - tanh(2^-3 * sqrt(x)) / (2^-3 * sqrt(x))) / (2^-6 * x), |
| 47 | // 12, [|128...|], [2^(-150), 1], fixed); |
| 48 | // > for i from 0 to 12 do { |
| 49 | // c = abs(coeff(P, i)); |
| 50 | // c_int = round(c * 2^128, 256, RN); |
| 51 | // hi = floor(c_int * 2^(-64)); |
| 52 | // lo = c_int - hi * 2^64; |
| 53 | // print("Frac128({", lo, "ULL,", hi, "ULL}), // c_", i); |
| 54 | // }; |
| 55 | // > dirtyinfnorm(tanh(x) - (x - x^3 * subst(P, x, 64 * x^2)), [-2^-3, 2^-3]); |
| 56 | // 0x1.7978...p-131 < 2^-130. |
| 57 | LIBC_INLINE_VAR constexpr Frac128 TANH_SMALL_W_128[9] = { |
| 58 | // lsb = 2^-128 |
| 59 | Frac128({0x5555555555555501ULL, 0x5555555555555555ULL}), // c_0 |
| 60 | Frac128({0x8888888888881951ULL, 0x0088888888888888ULL}), // c_1 |
| 61 | Frac128({0xdd0dd0dd0db889b5ULL, 0x0000dd0dd0dd0dd0ULL}), // c_2 |
| 62 | Frac128({0x0f9f32d6b98d67dfULL, 0x000001664f4882c1ULL}), // c_3 |
| 63 | Frac128({0xd84791eb52525c45ULL, 0x0000000244dc6abcULL}), // c_4 |
| 64 | Frac128({0xc2afbbf61d062422ULL, 0x0000000003ada7a1ULL}), // c_5 |
| 65 | Frac128({0x914add75e7e579bbULL, 0x000000000005f68dULL}), // c_6 |
| 66 | Frac128({0xc12401ab85fc9a4aULL, 0x00000000000009aaULL}), // c_7 |
| 67 | Frac128({0xabebb9964eabe8bdULL, 0x000000000000000fULL}), // c_8 |
| 68 | }; |
| 69 | |
| 70 | LIBC_INLINE_VAR constexpr Frac64 TANH_SMALL_W_LO64[4] = { |
| 71 | // lsb = 2^-128 (val[1] == 0) |
| 72 | Frac64(0x1967e174ab55fb50ULL), // c_9 |
| 73 | Frac64(0x00292f9d250de4b0ULL), // c_10 |
| 74 | Frac64(0x000042b85e8bcc87ULL), // c_11 |
| 75 | Frac64(0x00000067e2f87d8eULL), // c_12 |
| 76 | }; |
| 77 | |
| 78 | // 128-bit accurate path for tanh(x) when 2^-27 < |x| < 2^-3: |
| 79 | LIBC_NOINLINE LIBC_INLINE double tanh_small_accurate_128(double x) { |
| 80 | using FPBits = fputil::FPBits<double>; |
| 81 | FPBits xbits(x); |
| 82 | int exp_x = xbits.get_exponent(); // in [-27, -4] |
| 83 | |
| 84 | // Normalize |x|'s 53-bit mantissa to m_x in [1, 2) with lsb = 2^-63: |
| 85 | // |x| = 2^exp_x * m_x. |
| 86 | uint64_t mant_x = (xbits.get_mantissa() | (1ULL << 52)) << 11; // lsb = 2^-63 |
| 87 | |
| 88 | // Normalized mantissa of |x| and m_x^2: |
| 89 | // m_x_frac = m_x / 2 (lsb = 2^-127) |
| 90 | // m_x2 = mant_x^2 * 2^-126 (lsb = 2^-126) |
| 91 | Frac128 m_x_frac({0, mant_x}); // lsb = 2^-127 |
| 92 | UInt<128> m_x2 = UInt<64>(mant_x).ful_mul(other: UInt<64>(mant_x)); // lsb = 2^-126 |
| 93 | |
| 94 | // Since Frac128(m_x2.val) represents m_x^2 / 4 (lsb = 2^-128) and m_x_frac |
| 95 | // represents m_x / 2 (lsb = 2^-127), their Frac128 product (>> 128) |
| 96 | // represents m_x^3 / 8 with lsb = 2^-125: |
| 97 | Frac128 m_x3 = Frac128(m_x2.val) * m_x_frac; // lsb = 2^-125 |
| 98 | |
| 99 | // Scale m_x^2 / 4 by 2^(2 * exp_x + 8) to obtain w = 64 * x^2 in (0, 1) with |
| 100 | // lsb = 2^-128. Since exp_x is in [-27, -4], shift_w = -2 * exp_x - 8 is in |
| 101 | // [0, 46]: |
| 102 | unsigned shift_w = static_cast<unsigned>(-2 * exp_x - 8); // in [0, 46] |
| 103 | Frac128 w = Frac128(m_x2.val) >> shift_w; // lsb = 2^-128 |
| 104 | |
| 105 | // High-degree alternating Horner steps (k = 12, 11, 10, 9 -> 8) using 64-bit |
| 106 | // multiplications since val[1] == 0 for c_9..c_12: |
| 107 | Frac64 w_hi = w.to_frac64(); // lsb = 2^-64 |
| 108 | Frac64 p_lo = fputil::altpolyeval(x: w_hi, a: TANH_SMALL_W_LO64); |
| 109 | p_lo = Frac64(TANH_SMALL_W_128[8].val[0]) - p_lo * w_hi; |
| 110 | |
| 111 | // Low-degree alternating Horner steps (k = 7..0) in 128-bit fixed-point: |
| 112 | Frac128 p({p_lo.val[0], TANH_SMALL_W_128[8].val[1]}); // lsb = 2^-128 |
| 113 | LIBC_LOOP_NOUNROLL |
| 114 | for (int i = 7; i >= 0; --i) |
| 115 | p = TANH_SMALL_W_128[i] - p * w; // lsb = 2^-128 |
| 116 | |
| 117 | // Reconstruction of |tanh(x)| = |x| - |x|^3 * P(64 * x^2): |
| 118 | // Since y_poly = m_x3 * p represents m_x^3 * P(w) with lsb = 2^-125, |
| 119 | // shifting right by shift = -2 * exp_x - 2 (in [6, 52]) aligns: |
| 120 | // 2^(3 * exp_x) * m_x^3 * P(w) |
| 121 | // to the same scale as m_x_frac (lsb = 2^(exp_x - 127)): |
| 122 | unsigned shift = static_cast<unsigned>(-2 * exp_x - 2); // in [6, 52] |
| 123 | Frac128 y_poly = m_x3 * p; // lsb = 2^-125 |
| 124 | Frac128 y_m = y_poly >> shift; // lsb = 2^(exp_x - 127) |
| 125 | |
| 126 | // Subtract |x|^3 * P(w) from |x| and round to double in the current rounding |
| 127 | // mode (DFloat128 normalizes x_m if the leading bit shifts to bit 126): |
| 128 | UInt<128> x_m = UInt<128>(m_x_frac) - UInt<128>(y_m); // lsb = 2^(exp_x - 127) |
| 129 | DFloat128 r(xbits.sign(), exp_x - 127, x_m); |
| 130 | return static_cast<double>(r); |
| 131 | } |
| 132 | |
| 133 | // Compute tanh(x) = (e^(2x) - 1) / (e^(2x) + 1) in 128-bit precision using |
| 134 | // direct base-2 192-bit fixed-point range reduction for |
| 135 | // 2^-3 <= |x| < 0x1.30fc1931f09cap+4: |
| 136 | // |
| 137 | // Writing z = 2 * |x| * log2(e) = hi_p + idx_p * 2^-4 + u with hi_p in [0, 55], |
| 138 | // idx_p in [0, 15], and u in [0, 2^-4), we have: |
| 139 | // e^(2 * |x|) = 2^hi_p * m_final |
| 140 | // where m_final = exp2_eval_frac128(idx_p, u) = 2^(idx_p * 2^-4 + u) in [1, 2). |
| 141 | // Dividing numerator and denominator by 2^hi_p gives: |
| 142 | // tanh(|x|) = (m_final - 2^-hi_p) / (m_final + 2^-hi_p) |
| 143 | // where 2^-hi_p is a single bit in Frac128. |
| 144 | LIBC_NOINLINE LIBC_INLINE double tanh_accurate_128(double x) { |
| 145 | using FPBits = fputil::FPBits<double>; |
| 146 | FPBits xbits(x); |
| 147 | int exp_x = xbits.get_exponent(); // in [-3, 4] |
| 148 | |
| 149 | // Normalize |x|'s 53-bit mantissa to [2^63, 2^64): |
| 150 | uint64_t mant_x = (xbits.get_mantissa() | (1ULL << 52)) << 11; |
| 151 | |
| 152 | // 1-limb x 3-limb product for z = 2 * |x| * log2(e): |
| 153 | // 2 * |x| = 2^(exp_x - 62) * mant_x |
| 154 | // LOG2_E_192.log2_b has lsb = 2^-191 |
| 155 | // p = mant_x * LOG2_E_192.log2_b has lsb = 2^(exp_x - 253). |
| 156 | UInt<256> p = LOG2_E_192.log2_b.ful_mul(other: UInt<64>(mant_x)); |
| 157 | |
| 158 | // Since p.val[3] has lsb = 2^(exp_x - 61), the 2^-4 bit of z is at bit |
| 159 | // shr = 57 - exp_x (in [53, 60]) of p.val[3]: |
| 160 | int shr = 57 - exp_x; // in [53, 60] |
| 161 | int shl = exp_x + 7; // in [4, 11], with shr + shl = 64 |
| 162 | int k_p = static_cast<int>(p.val[3] >> shr); // floor(z * 2^4) |
| 163 | uint64_t u_lo = (p.val[2] << shl) | (p.val[1] >> shr); |
| 164 | uint64_t u_hi = (p.val[3] << shl) | (p.val[2] >> shr); |
| 165 | // Scale (u_lo, u_hi) from lsb = 2^-132 to u in [0, 2^-4) with lsb = 2^-128: |
| 166 | Frac128 u = Frac128({u_lo, u_hi}) >> 4; // lsb = 2^-128 |
| 167 | |
| 168 | // Decompose k_p = 16 * hi_p + idx_p and evaluate |
| 169 | // m_final = 2^(idx_p * 2^-4 + u) in [1, 2) with lsb = 2^-127: |
| 170 | int hi_p = k_p >> 4; // in [0, 55] |
| 171 | unsigned idx_p = static_cast<unsigned>(k_p & 0x0f); |
| 172 | Frac128 m_final = exp2_eval_frac128(idx: idx_p, u); // lsb = 2^-127 |
| 173 | |
| 174 | // Scale m_final and 2^-hi_p to lsb = 2^-126 so that both |
| 175 | // num = m_final - 2^-hi_p in (0.28, 2.0) |
| 176 | // den = m_final + 2^-hi_p in (1.0, 3.0) |
| 177 | // fit in Frac128 without overflow: |
| 178 | Frac128 m_half = m_final >> 1; // lsb = 2^-126 |
| 179 | Frac128 one_neg_hi({0, 1ULL << (62 - hi_p)}); // 2^-hi_p, lsb = 2^-126 |
| 180 | Frac128 num = m_half - one_neg_hi; // lsb = 2^-126 |
| 181 | Frac128 den = m_half + one_neg_hi; // lsb = 2^-126 |
| 182 | |
| 183 | // Normalize den to d_norm in (1, 2) with lsb = 2^-127 (shift_d is 0 if |
| 184 | // m_final + 2^-hi_p >= 2, or 1 if m_final + 2^-hi_p < 2): |
| 185 | unsigned shift_d = static_cast<unsigned>((den.val[1] >> 63) == 0); |
| 186 | Frac128 d_norm = den << shift_d; // in (1, 2), lsb = 2^-127 |
| 187 | Frac128 n_norm = num << shift_d; // in (0.12, d_norm), lsb = 2^-127 |
| 188 | |
| 189 | // Compute 126-bit reciprocal r2 ~ 1 / d_norm (lsb = 2^-128) via two |
| 190 | // Newton-Raphson iterations starting from a 53-bit double reciprocal: |
| 191 | // r_{k+1} = r_k * (2 - d_norm * r_k). |
| 192 | // Since d_norm * r_k has lsb = 2^-127, ~dr_k represents (2 - d_norm * r_k) |
| 193 | // with lsb = 2^-127 to within 2^-127. |
| 194 | // Convert (1.0 / d_d) * 2^62 in [2^61, 2^62] via signed int64_t and shift |
| 195 | // left by 2 to avoid spurious FE_INVALID on x86-64 double-to-uint64_t |
| 196 | // conversions of values >= 2^63: |
| 197 | int64_t d_hi_53 = static_cast<int64_t>((d_norm.val[1] >> 11) + 1); |
| 198 | double d_d = static_cast<double>(d_hi_53) * 0x1.0p-52; |
| 199 | double r0_d = (1.0 / d_d) * 0x1.0p62; |
| 200 | uint64_t r0_hi = static_cast<uint64_t>(static_cast<int64_t>(r0_d)) << 2; |
| 201 | Frac128 r0({0, r0_hi}); // lsb = 2^-128 (~52-bit accurate) |
| 202 | |
| 203 | Frac128 dr0 = d_norm * r0; // lsb = 2^-127 |
| 204 | Frac128 r1 = (r0 * (~dr0)) << 1; // lsb = 2^-128 (~104-bit accurate) |
| 205 | |
| 206 | Frac128 dr1 = d_norm * r1; // lsb = 2^-127 |
| 207 | Frac128 r2 = (r1 * (~dr1)) << 1; // lsb = 2^-128 (~126-bit accurate) |
| 208 | |
| 209 | // Quotient q = n_norm * r2 ~ num / den = tanh(|x|) in (0.12, 1) with |
| 210 | // lsb = 2^-127: |
| 211 | Frac128 q = n_norm * r2; // lsb = 2^-127 |
| 212 | DFloat128 r(xbits.sign(), -127, UInt<128>(q)); |
| 213 | return static_cast<double>(r); |
| 214 | } |
| 215 | |
| 216 | } // namespace double_eval |
| 217 | } // namespace math |
| 218 | } // namespace LIBC_NAMESPACE_DECL |
| 219 | |
| 220 | #endif // LLVM_LIBC_SRC___SUPPORT_MATH_TANH_ACCURATE_128_H |
| 221 | |