1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// 128-bit accurate paths for double-precision tanh function.
11///
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_LIBC_SRC___SUPPORT_MATH_TANH_ACCURATE_128_H
15#define LLVM_LIBC_SRC___SUPPORT_MATH_TANH_ACCURATE_128_H
16
17#include "src/__support/FPUtil/FPBits.h"
18#include "src/__support/FPUtil/dyadic_float.h"
19#include "src/__support/FPUtil/polyeval.h"
20#include "src/__support/common.h"
21#include "src/__support/frac128.h"
22#include "src/__support/frac64.h"
23#include "src/__support/macros/attributes.h"
24#include "src/__support/macros/config.h"
25#include "src/__support/macros/optimization.h"
26#include "src/__support/math/exp2_accurate_128.h"
27#include "src/__support/uint128.h"
28
29namespace LIBC_NAMESPACE_DECL {
30namespace math {
31namespace double_eval {
32
33// Polynomial approximation for (x - tanh(x)) / x^3 on [-2^-3, 2^-3] in terms
34// of w = 64 * x^2 in [0, 1] as a degree-12 alternating polynomial in 128-bit
35// fixed-point format (lsb = 2^-128):
36// P(w) = c_0 - c_1 * w + c_2 * w^2 - ... + c_12 * w^12
37// tanh(x) ~ x - x^3 * P(64 * x^2).
38//
39// For k >= 9, the scaled coefficients c_k satisfy c_k < 2^-64, so their high
40// 64-bit words val[1] are identically zero and are stored as 64-bit low words
41// in TANH_SMALL_W_LO64.
42//
43// Minimax polynomial coefficients generated by Sollya with:
44// > display = hexadecimal!;
45// > prec = 500;
46// > P = fpminimax((1 - tanh(2^-3 * sqrt(x)) / (2^-3 * sqrt(x))) / (2^-6 * x),
47// 12, [|128...|], [2^(-150), 1], fixed);
48// > for i from 0 to 12 do {
49// c = abs(coeff(P, i));
50// c_int = round(c * 2^128, 256, RN);
51// hi = floor(c_int * 2^(-64));
52// lo = c_int - hi * 2^64;
53// print("Frac128({", lo, "ULL,", hi, "ULL}), // c_", i);
54// };
55// > dirtyinfnorm(tanh(x) - (x - x^3 * subst(P, x, 64 * x^2)), [-2^-3, 2^-3]);
56// 0x1.7978...p-131 < 2^-130.
57LIBC_INLINE_VAR constexpr Frac128 TANH_SMALL_W_128[9] = {
58 // lsb = 2^-128
59 Frac128({0x5555555555555501ULL, 0x5555555555555555ULL}), // c_0
60 Frac128({0x8888888888881951ULL, 0x0088888888888888ULL}), // c_1
61 Frac128({0xdd0dd0dd0db889b5ULL, 0x0000dd0dd0dd0dd0ULL}), // c_2
62 Frac128({0x0f9f32d6b98d67dfULL, 0x000001664f4882c1ULL}), // c_3
63 Frac128({0xd84791eb52525c45ULL, 0x0000000244dc6abcULL}), // c_4
64 Frac128({0xc2afbbf61d062422ULL, 0x0000000003ada7a1ULL}), // c_5
65 Frac128({0x914add75e7e579bbULL, 0x000000000005f68dULL}), // c_6
66 Frac128({0xc12401ab85fc9a4aULL, 0x00000000000009aaULL}), // c_7
67 Frac128({0xabebb9964eabe8bdULL, 0x000000000000000fULL}), // c_8
68};
69
70LIBC_INLINE_VAR constexpr Frac64 TANH_SMALL_W_LO64[4] = {
71 // lsb = 2^-128 (val[1] == 0)
72 Frac64(0x1967e174ab55fb50ULL), // c_9
73 Frac64(0x00292f9d250de4b0ULL), // c_10
74 Frac64(0x000042b85e8bcc87ULL), // c_11
75 Frac64(0x00000067e2f87d8eULL), // c_12
76};
77
78// 128-bit accurate path for tanh(x) when 2^-27 < |x| < 2^-3:
79LIBC_NOINLINE LIBC_INLINE double tanh_small_accurate_128(double x) {
80 using FPBits = fputil::FPBits<double>;
81 FPBits xbits(x);
82 int exp_x = xbits.get_exponent(); // in [-27, -4]
83
84 // Normalize |x|'s 53-bit mantissa to m_x in [1, 2) with lsb = 2^-63:
85 // |x| = 2^exp_x * m_x.
86 uint64_t mant_x = (xbits.get_mantissa() | (1ULL << 52)) << 11; // lsb = 2^-63
87
88 // Normalized mantissa of |x| and m_x^2:
89 // m_x_frac = m_x / 2 (lsb = 2^-127)
90 // m_x2 = mant_x^2 * 2^-126 (lsb = 2^-126)
91 Frac128 m_x_frac({0, mant_x}); // lsb = 2^-127
92 UInt<128> m_x2 = UInt<64>(mant_x).ful_mul(other: UInt<64>(mant_x)); // lsb = 2^-126
93
94 // Since Frac128(m_x2.val) represents m_x^2 / 4 (lsb = 2^-128) and m_x_frac
95 // represents m_x / 2 (lsb = 2^-127), their Frac128 product (>> 128)
96 // represents m_x^3 / 8 with lsb = 2^-125:
97 Frac128 m_x3 = Frac128(m_x2.val) * m_x_frac; // lsb = 2^-125
98
99 // Scale m_x^2 / 4 by 2^(2 * exp_x + 8) to obtain w = 64 * x^2 in (0, 1) with
100 // lsb = 2^-128. Since exp_x is in [-27, -4], shift_w = -2 * exp_x - 8 is in
101 // [0, 46]:
102 unsigned shift_w = static_cast<unsigned>(-2 * exp_x - 8); // in [0, 46]
103 Frac128 w = Frac128(m_x2.val) >> shift_w; // lsb = 2^-128
104
105 // High-degree alternating Horner steps (k = 12, 11, 10, 9 -> 8) using 64-bit
106 // multiplications since val[1] == 0 for c_9..c_12:
107 Frac64 w_hi = w.to_frac64(); // lsb = 2^-64
108 Frac64 p_lo = fputil::altpolyeval(x: w_hi, a: TANH_SMALL_W_LO64);
109 p_lo = Frac64(TANH_SMALL_W_128[8].val[0]) - p_lo * w_hi;
110
111 // Low-degree alternating Horner steps (k = 7..0) in 128-bit fixed-point:
112 Frac128 p({p_lo.val[0], TANH_SMALL_W_128[8].val[1]}); // lsb = 2^-128
113 LIBC_LOOP_NOUNROLL
114 for (int i = 7; i >= 0; --i)
115 p = TANH_SMALL_W_128[i] - p * w; // lsb = 2^-128
116
117 // Reconstruction of |tanh(x)| = |x| - |x|^3 * P(64 * x^2):
118 // Since y_poly = m_x3 * p represents m_x^3 * P(w) with lsb = 2^-125,
119 // shifting right by shift = -2 * exp_x - 2 (in [6, 52]) aligns:
120 // 2^(3 * exp_x) * m_x^3 * P(w)
121 // to the same scale as m_x_frac (lsb = 2^(exp_x - 127)):
122 unsigned shift = static_cast<unsigned>(-2 * exp_x - 2); // in [6, 52]
123 Frac128 y_poly = m_x3 * p; // lsb = 2^-125
124 Frac128 y_m = y_poly >> shift; // lsb = 2^(exp_x - 127)
125
126 // Subtract |x|^3 * P(w) from |x| and round to double in the current rounding
127 // mode (DFloat128 normalizes x_m if the leading bit shifts to bit 126):
128 UInt<128> x_m = UInt<128>(m_x_frac) - UInt<128>(y_m); // lsb = 2^(exp_x - 127)
129 DFloat128 r(xbits.sign(), exp_x - 127, x_m);
130 return static_cast<double>(r);
131}
132
133// Compute tanh(x) = (e^(2x) - 1) / (e^(2x) + 1) in 128-bit precision using
134// direct base-2 192-bit fixed-point range reduction for
135// 2^-3 <= |x| < 0x1.30fc1931f09cap+4:
136//
137// Writing z = 2 * |x| * log2(e) = hi_p + idx_p * 2^-4 + u with hi_p in [0, 55],
138// idx_p in [0, 15], and u in [0, 2^-4), we have:
139// e^(2 * |x|) = 2^hi_p * m_final
140// where m_final = exp2_eval_frac128(idx_p, u) = 2^(idx_p * 2^-4 + u) in [1, 2).
141// Dividing numerator and denominator by 2^hi_p gives:
142// tanh(|x|) = (m_final - 2^-hi_p) / (m_final + 2^-hi_p)
143// where 2^-hi_p is a single bit in Frac128.
144LIBC_NOINLINE LIBC_INLINE double tanh_accurate_128(double x) {
145 using FPBits = fputil::FPBits<double>;
146 FPBits xbits(x);
147 int exp_x = xbits.get_exponent(); // in [-3, 4]
148
149 // Normalize |x|'s 53-bit mantissa to [2^63, 2^64):
150 uint64_t mant_x = (xbits.get_mantissa() | (1ULL << 52)) << 11;
151
152 // 1-limb x 3-limb product for z = 2 * |x| * log2(e):
153 // 2 * |x| = 2^(exp_x - 62) * mant_x
154 // LOG2_E_192.log2_b has lsb = 2^-191
155 // p = mant_x * LOG2_E_192.log2_b has lsb = 2^(exp_x - 253).
156 UInt<256> p = LOG2_E_192.log2_b.ful_mul(other: UInt<64>(mant_x));
157
158 // Since p.val[3] has lsb = 2^(exp_x - 61), the 2^-4 bit of z is at bit
159 // shr = 57 - exp_x (in [53, 60]) of p.val[3]:
160 int shr = 57 - exp_x; // in [53, 60]
161 int shl = exp_x + 7; // in [4, 11], with shr + shl = 64
162 int k_p = static_cast<int>(p.val[3] >> shr); // floor(z * 2^4)
163 uint64_t u_lo = (p.val[2] << shl) | (p.val[1] >> shr);
164 uint64_t u_hi = (p.val[3] << shl) | (p.val[2] >> shr);
165 // Scale (u_lo, u_hi) from lsb = 2^-132 to u in [0, 2^-4) with lsb = 2^-128:
166 Frac128 u = Frac128({u_lo, u_hi}) >> 4; // lsb = 2^-128
167
168 // Decompose k_p = 16 * hi_p + idx_p and evaluate
169 // m_final = 2^(idx_p * 2^-4 + u) in [1, 2) with lsb = 2^-127:
170 int hi_p = k_p >> 4; // in [0, 55]
171 unsigned idx_p = static_cast<unsigned>(k_p & 0x0f);
172 Frac128 m_final = exp2_eval_frac128(idx: idx_p, u); // lsb = 2^-127
173
174 // Scale m_final and 2^-hi_p to lsb = 2^-126 so that both
175 // num = m_final - 2^-hi_p in (0.28, 2.0)
176 // den = m_final + 2^-hi_p in (1.0, 3.0)
177 // fit in Frac128 without overflow:
178 Frac128 m_half = m_final >> 1; // lsb = 2^-126
179 Frac128 one_neg_hi({0, 1ULL << (62 - hi_p)}); // 2^-hi_p, lsb = 2^-126
180 Frac128 num = m_half - one_neg_hi; // lsb = 2^-126
181 Frac128 den = m_half + one_neg_hi; // lsb = 2^-126
182
183 // Normalize den to d_norm in (1, 2) with lsb = 2^-127 (shift_d is 0 if
184 // m_final + 2^-hi_p >= 2, or 1 if m_final + 2^-hi_p < 2):
185 unsigned shift_d = static_cast<unsigned>((den.val[1] >> 63) == 0);
186 Frac128 d_norm = den << shift_d; // in (1, 2), lsb = 2^-127
187 Frac128 n_norm = num << shift_d; // in (0.12, d_norm), lsb = 2^-127
188
189 // Compute 126-bit reciprocal r2 ~ 1 / d_norm (lsb = 2^-128) via two
190 // Newton-Raphson iterations starting from a 53-bit double reciprocal:
191 // r_{k+1} = r_k * (2 - d_norm * r_k).
192 // Since d_norm * r_k has lsb = 2^-127, ~dr_k represents (2 - d_norm * r_k)
193 // with lsb = 2^-127 to within 2^-127.
194 // Convert (1.0 / d_d) * 2^62 in [2^61, 2^62] via signed int64_t and shift
195 // left by 2 to avoid spurious FE_INVALID on x86-64 double-to-uint64_t
196 // conversions of values >= 2^63:
197 int64_t d_hi_53 = static_cast<int64_t>((d_norm.val[1] >> 11) + 1);
198 double d_d = static_cast<double>(d_hi_53) * 0x1.0p-52;
199 double r0_d = (1.0 / d_d) * 0x1.0p62;
200 uint64_t r0_hi = static_cast<uint64_t>(static_cast<int64_t>(r0_d)) << 2;
201 Frac128 r0({0, r0_hi}); // lsb = 2^-128 (~52-bit accurate)
202
203 Frac128 dr0 = d_norm * r0; // lsb = 2^-127
204 Frac128 r1 = (r0 * (~dr0)) << 1; // lsb = 2^-128 (~104-bit accurate)
205
206 Frac128 dr1 = d_norm * r1; // lsb = 2^-127
207 Frac128 r2 = (r1 * (~dr1)) << 1; // lsb = 2^-128 (~126-bit accurate)
208
209 // Quotient q = n_norm * r2 ~ num / den = tanh(|x|) in (0.12, 1) with
210 // lsb = 2^-127:
211 Frac128 q = n_norm * r2; // lsb = 2^-127
212 DFloat128 r(xbits.sign(), -127, UInt<128>(q));
213 return static_cast<double>(r);
214}
215
216} // namespace double_eval
217} // namespace math
218} // namespace LIBC_NAMESPACE_DECL
219
220#endif // LLVM_LIBC_SRC___SUPPORT_MATH_TANH_ACCURATE_128_H
221