1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// Double-precision implementation of tanh.
11///
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_LIBC_SRC___SUPPORT_MATH_TANH_DOUBLE_EVAL_H
15#define LLVM_LIBC_SRC___SUPPORT_MATH_TANH_DOUBLE_EVAL_H
16
17#include "hdr/errno_macros.h"
18#include "hdr/fenv_macros.h"
19#include "src/__support/CPP/bit.h"
20#include "src/__support/FPUtil/FEnvImpl.h"
21#include "src/__support/FPUtil/FPBits.h"
22#include "src/__support/FPUtil/double_double.h"
23#include "src/__support/FPUtil/except_value_utils.h"
24#include "src/__support/FPUtil/fp_bitwise.h"
25#include "src/__support/FPUtil/multiply_add.h"
26#include "src/__support/FPUtil/polyeval.h"
27#include "src/__support/common.h"
28#include "src/__support/macros/attributes.h"
29#include "src/__support/macros/config.h"
30#include "src/__support/macros/optimization.h"
31#include "src/__support/macros/properties/cpu_features.h"
32#include "src/__support/math/exp_utils.h"
33
34#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
35#include "src/__support/math/tanh_accurate_128.h"
36#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
37
38namespace LIBC_NAMESPACE_DECL {
39namespace math {
40namespace double_eval {
41
42#ifdef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
43
44// Splitting -log(2) / 2 ~ hi + lo with 35-bit-precision hi part so that
45// kd * MLOG_2_HALF_HI is exact for |kd| < 2^18:
46// > hi = round(-log(2) / 2, 35, RN);
47// > lo = round(-log(2) / 2 - hi, D, RN);
48LIBC_INLINE_VAR constexpr double MLOG_2_HALF_HI = -0x1.62e42fefcp-2;
49LIBC_INLINE_VAR constexpr double MLOG_2_HALF_LO = 0x1.c610ca86c3899p-38;
50
51// Degree-3 minimax polynomials in v^2 for (cosh(v) - 1) / v^2 and
52// (sinh(v) - v) / v^3 on [-log(2) / 4, log(2) / 4]:
53// cosh(v) - 1 ~ v^2 * P_even(v^2)
54// sinh(v) - v ~ v^3 * P_odd(v^2).
55//
56// Generated by Sollya with:
57// > display = hexadecimal!;
58// > prec = 500;
59// > B = (log(2) / 4)^2;
60// > P_even = fpminimax((cosh(sqrt(x)) - 1) / x, 3, [|D...|], [2^(-100), B]);
61// > for i from 0 to 3 do print(coeff(P_even, i), ",");
62// > dirtyinfnorm((cosh(sqrt(x)) - 1) - x * P_even, [0, B]);
63// 0x1.e49e...p-55 < 2^-54.
64LIBC_INLINE_VAR constexpr double TANH_EVEN_N0_COEFFS[4] = {
65 0x1.fffffffffffep-2, // ~ 1/2!
66 0x1.5555555597352p-5, // ~ 1/4!
67 0x1.6c16bc13d272cp-10, // ~ 1/6!
68 0x1.a06123b7cd784p-16, // ~ 1/8!
69};
70
71// > P_odd = fpminimax((sinh(sqrt(x)) / sqrt(x) - 1) / x, 3, [|D...|],
72// [2^(-100), B]);
73// > for i from 0 to 3 do print(coeff(P_odd, i), ",");
74// > dirtyinfnorm((sinh(sqrt(x)) - sqrt(x)) - x * sqrt(x) * P_odd, [0, B]);
75// 0x1.ee40...p-61 < 2^-60.
76LIBC_INLINE_VAR constexpr double TANH_ODD_N0_COEFFS[4] = {
77 0x1.555555555555p-3, // ~ 1/3!
78 0x1.111111112884cp-7, // ~ 1/5!
79 0x1.a019fdc55d34fp-13, // ~ 1/7!
80 0x1.7211ce0221666p-19, // ~ 1/9!
81};
82
83// Tableless (N = 0) evaluation of tanh(x) = sinh(x) / cosh(x) for
84// 2^-27 < |x| < 0x1.30fc1931f09cap+4.
85//
86// Range reduction:
87// Let |x| = k * (log(2) / 2) + v, where:
88// k = round(|x| * 2 * log2(e))
89// |v| <= log(2) / 4
90// then:
91// e^|x| = 2^(k / 2) * e^v
92// e^-|x| = 2^(-k / 2) * e^-v = 2^(k / 2) * 2^-k * e^-v.
93//
94// Factoring out 2^(k / 2) in numerator and denominator and separating
95// e^(+/- v) = cosh(v) +/- sinh(v):
96// tanh(|x|) = (e^v - m_m * e^-v) / (e^v + m_m * e^-v)
97// = (d * cosh(v) + s * sinh(v)) / (s * cosh(v) + d * sinh(v))
98// where:
99// m_m = 2^-k
100// s = 1 + m_m
101// d = 1 - m_m.
102//
103// Key properties across the domain:
104// - For |x| < log(2) / 4, k = 0 gives v = |x| (exact, t2 = 0) and m_m = 1, so
105// d = 0 and s = 2, reducing the ratio to:
106// (2 * sinh(v)) / (2 * cosh(v)) = sinh(x) / cosh(x)
107// with no cancellation down to |x| = 2^-27.
108//
109// - For 1 <= k <= 53, 2^-k is in [2^-53, 2^-1], so both:
110// s = 1.0 + m_m
111// d = 1.0 - m_m
112// are exact in all rounding modes. For k >= 54:
113// m_m <= 2^-54 < 0.5 * ulp(1.0),
114// so clamping m_m to 0.0 avoids a 1-ULP rounding error in directed rounding
115// modes.
116LIBC_MATH_SMALL_TABLES_INLINE double tanh_eval(double x) {
117 using FPBits = fputil::FPBits<double>;
118 FPBits xbits(x);
119 double x_a = xbits.abs().get_val();
120
121 // Using ROUND_BIAS, ROUND_SHIFT, and ROUND_MASK (see exp_utils.h), we compute
122 // k_biased = fma(x_a, LOG2_E_2, ROUND_BIAS) and extract in parallel:
123 // kd = (k_biased & ROUND_MASK) - ROUND_SHIFT
124 // k = (k_bits >> 32) & 0x3'ffff.
125 constexpr double LOG2_E_2 = 2.0 * LOG2_E_DD.hi;
126 double k_biased = fputil::multiply_add(x_a, LOG2_E_2, ROUND_BIAS);
127 double kd = fputil::fp_and(k_biased, ROUND_MASK) - ROUND_SHIFT;
128 int k = static_cast<int>(cpp::bit_cast<uint64_t>(k_biased) >> 32) & 0x3'ffff;
129
130 // Secondary scale factor:
131 // m_m = 2^-k for k < 54
132 // m_m = 0.0 for k >= 54 (where 2^-k <= 2^-54).
133 double m_m = (k < 54) ? scale_exp2(1.0, -k) : 0.0;
134
135 // Reduced argument:
136 // v = (x_a + kd * MLOG_2_HALF_HI) + kd * MLOG_2_HALF_LO.
137 // Since MLOG_2_HALF_HI has 35 significant bits and |kd| <= 55,
138 // t1 = fma(kd, MLOG_2_HALF_HI, x_a)
139 // is exact in all rounding modes with or without FMA.
140 double t1 = fputil::multiply_add(kd, MLOG_2_HALF_HI, x_a); // exact
141 double t2 = kd * MLOG_2_HALF_LO;
142 double v = t1 + t2;
143 double v2 = v * v;
144 double v4 = v2 * v2;
145
146 // Evaluate degree-3 even and odd minimax polynomials P_even(v^2) and
147 // P_odd(v^2) in parallel using Estrin's scheme:
148 // cosh(v) ~ 1 + v^2 * P_even(v^2)
149 // sinh(v) ~ t1 + (t2 + v^3 * P_odd(v^2))
150
151 // First Estrin layer: c_0 + v^2 * c_1 and c_2 + v^2 * c_3
152 double e01 =
153 fputil::multiply_add(v2, TANH_EVEN_N0_COEFFS[1], TANH_EVEN_N0_COEFFS[0]);
154 double o01 =
155 fputil::multiply_add(v2, TANH_ODD_N0_COEFFS[1], TANH_ODD_N0_COEFFS[0]);
156 double e23 =
157 fputil::multiply_add(v2, TANH_EVEN_N0_COEFFS[3], TANH_EVEN_N0_COEFFS[2]);
158 double o23 =
159 fputil::multiply_add(v2, TANH_ODD_N0_COEFFS[3], TANH_ODD_N0_COEFFS[2]);
160
161 // Final polynomial values P_even(v^2) and P_odd(v^2):
162 double p_even = fputil::multiply_add(v4, e23, e01);
163 double p_odd = fputil::multiply_add(v4, o23, o01);
164
165 // Low parts of cosh(v) - 1 and sinh(v) - t1:
166 // c_lo ~ v^2 * P_even(v^2)
167 // s_lo ~ t2 + v^3 * P_odd(v^2)
168 double c_lo = v2 * p_even;
169 double v3 = v * v2;
170 double s_lo = fputil::multiply_add(v3, p_odd, t2);
171
172 // Since m_m is either 0 or 2^-k with 0 <= k <= 53, both s and d are exact in
173 // all rounding modes:
174 double s = 1.0 + m_m; // exact
175 double d = 1.0 - m_m; // exact
176
177 // Apply sign(x) to the numerator weights s_num and d_num before accumulating
178 // the polynomial so the numerator rounds in the active rounding direction for
179 // negative x as well:
180 double sign_x = fputil::fp_xor(x, x_a);
181 double s_num = fputil::fp_xor(s, sign_x);
182 double d_num = fputil::fp_xor(d, sign_x);
183
184 // Numerator:
185 // num ~ sign(x) * (d * (1 + c_lo) + s * (t1 + s_lo))
186 // = d_num + (s_num * t1 + (d_num * c_lo + s_num * s_lo))
187 double s_s_lo = s_num * s_lo;
188 double num_lo = fputil::multiply_add(d_num, c_lo, s_s_lo);
189 double num_mid = fputil::multiply_add(s_num, t1, num_lo);
190 double num = d_num + num_mid;
191
192 // Denominator:
193 // den ~ s * (1 + c_lo) + d * (t1 + s_lo)
194 // = s + (d * t1 + (s * c_lo + d * s_lo))
195 double d_s_lo = d * s_lo;
196 double den_lo = fputil::multiply_add(s, c_lo, d_s_lo);
197 double den_mid = fputil::multiply_add(d, t1, den_lo);
198 double den = s + den_mid;
199
200 return num / den;
201}
202
203#else // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
204
205// Leading 11-bit split of c_0 ~ -1/3 for exact cubic reconstruction:
206// TANH_C0_HI_11 = round(-1/3, 11, RZ) = -0x1.554p-2
207// -1/3 - TANH_C0_HI_11 = -0x1.5555555555555p-14.
208LIBC_INLINE_VAR constexpr double TANH_C0_HI_11 = -0x1.554p-2;
209
210// Polynomial approximation for ((tanh(x) - x) / x^3 - (-0x1.554p-2)) on
211// [-2^-3, 2^-3] as a degree-5 even polynomial in x^2:
212// tanh(x) - x ~ x^3 * (TANH_C0_HI_11 + P_rem(x^2))
213//
214// Minimax polynomial coefficients generated by Sollya with:
215// > display = hexadecimal!;
216// > prec = 500;
217// > c0 = -0x1.554p-2 + (-0x1.5555555555555p-14);
218// > P = fpminimax((tanh(sqrt(x))/sqrt(x) - 1)/x - c0, [|1, 2, 3, 4, 5|],
219// [|D...|], [2^(-100), 2^(-6)], absolute);
220// > for i from 1 to 5 do print(coeff(P, i), ",");
221// > dirtyinfnorm((tanh(x) - x)/x^3 - (c0 + P(x^2)), [2^(-50), 2^(-3)]);
222// 0x1.9e2e...p-57 < 2^-56.
223// > dirtyinfnorm(tanh(x) - (x + x^3 * (c0 + P(x^2))), [-2^(-3), 2^(-3)]);
224// 0x1.9dd6...p-66 < 2^-65.
225LIBC_INLINE_VAR constexpr double TANH_SMALL_P_COEFFS[6] = {
226 -0x1.5555555555555p-14, // c_0 - (-0x1.554p-2) ~ -1/3 + 0x1.554p-2
227 0x1.1111111110c74p-3, // c_1 ~ 2/15
228 -0x1.ba1ba1b6047f3p-5, // c_2 ~ -17/315
229 0x1.664f3f15b86a5p-6, // c_3 ~ 62/2835
230 -0x1.2264b8b90e812p-7, // c_4 ~ -1382/155925
231 0x1.ce2449b1b72dcp-9, // c_5 ~ 21844/6081075
232};
233
234// Evaluate tanh(x) in double-double precision for
235// 2^-3 <= |x| < 0x1.30fc1931f09cap+4, falling back to tanh_accurate_128(x) when
236// Ziv's rounding test fails.
237//
238// Range reduction:
239// Let 2 * |x| = k_p * (log(2) / 2^7) + u
240// in which:
241// k_p = round(2 * |x| * 2^7 * log2(e)), k_m = -k_p
242// hi_m = k_m >> 7, idx_m = k_m & 0x7f
243// |u| <= 2^-8 * log(2)
244// then:
245// e^(2 * |x|) = 2^(k_p / 2^7) * e^u = e^u / m
246// where:
247// m = 2^hi_m * 2^(idx_m / 2^7) = 2^hi_m * EXP2_MID[idx_m].
248//
249// Rewriting tanh(|x|) = (e^(2 * |x|) - 1) / (e^(2 * |x|) + 1) in terms of e^u
250// and m avoids multiplying m by e^u:
251// tanh(|x|) = (e^u - m) / (e^u + m) = 1 - (2 * m) / (e^u + m).
252LIBC_MATH_SMALL_TABLES_INLINE double tanh_eval(double x) {
253 using FPBits = fputil::FPBits<double>;
254 FPBits xbits(x);
255 double x_a = xbits.abs().get_val();
256 double x_2 = 2.0 * x_a;
257
258 constexpr double LOG2_E_SCALE = EXP_SCALE * LOG2_E_DD.hi;
259 // Using ROUND_BIAS, ROUND_SHIFT, and ROUND_MASK (see exp_utils.h), we compute
260 // k_biased = fma(x_2, LOG2_E_SCALE, ROUND_BIAS) and extract in parallel:
261 // kd = (k_biased & ROUND_MASK) - ROUND_SHIFT
262 // k_p = (k_bits >> 32) & 0x3'ffff.
263 double k_biased = fputil::multiply_add(x: x_2, y: LOG2_E_SCALE, z: ROUND_BIAS);
264 double kd = fputil::fp_and(a: k_biased, b: ROUND_MASK) - ROUND_SHIFT;
265 int k_p =
266 static_cast<int>(cpp::bit_cast<uint64_t>(from: k_biased) >> 32) & 0x3'ffff;
267 int k_m = -k_p;
268
269 int hi_m = k_m >> EXP_N_BITS;
270 unsigned idx_m = static_cast<unsigned>(k_m & EXP_MASK);
271
272 // Scale m = EXP2_MID[idx_m] by 2^hi_m:
273 // For 2^-3 <= |x| < 0x1.30fc1931f09cap+4, hi_m is in [-56, 0], so scaling by
274 // 2^hi_m is always in the normal range:
275 // - Since m.hi is in [1, 2), we use scale_exp2(m.hi, hi_m) directly.
276 // - Since EXP2_MID[0].lo == 0.0, we scale m.lo by multiplying by:
277 // s_m = scale_exp2(1.0, hi_m).
278 double s_m = scale_exp2(r: 1.0, hi: hi_m);
279 DoubleDouble m = EXP2_MID[idx_m];
280 m.lo *= s_m;
281 m.hi = scale_exp2(r: m.hi, hi: hi_m);
282
283 // Reduced argument:
284 // u = (x_2 + kd * MLOG_2_EXP2_HI) + kd * MLOG_2_EXP2_MID.
285 // Since MLOG_2_EXP2_HI has 35 significant bits and |kd| < 2^13,
286 // t1 = fma(kd, MLOG_2_EXP2_HI, x_2)
287 // is exact in all rounding modes with or without FMA.
288 double t1 = fputil::multiply_add(x: kd, y: MLOG_2_EXP2_HI, z: x_2); // exact
289 double t2 = kd * MLOG_2_EXP2_MID;
290 double u = t1 + t2;
291
292 // Evaluate degree-3 minimax polynomial P(u) ~ (e^u - 1 - u) / u^2:
293 // e^u ~ 1 + t1 + exp_lo
294 // where:
295 // exp_lo = t2 + u^2 * P(u).
296 double u2 = u * u;
297 double p = fputil::estrin_polyeval(x: u, a: EXP_P_COEFFS);
298 double exp_lo = fputil::multiply_add(x: u2, y: p, z: t2);
299
300 // Denominator D = e^u + m ~ (1.0 + m.hi + t1 + exp_lo) + m.lo:
301 // s.hi + s.lo = 1.0 + m.hi (exact FastTwoSum since m.hi <= 1.0)
302 // d1.hi + d1.lo = s.hi + t1 (FastTwoSum, |t1| < 2^-8 < s.hi)
303 // d.hi + d.lo = d1.hi + exp_lo (FastTwoSum, |exp_lo| < 2^-17 < d1.hi).
304 // Accumulating exp_lo into d.hi keeps |d_lo| <= 2^-50 so the linear
305 // Karp-Markstein remainder is accurate to 105 bits:
306 DoubleDouble s = fputil::exact_add(a: 1.0, b: m.hi);
307 DoubleDouble d1 = fputil::exact_add(a: s.hi, b: t1);
308 DoubleDouble d = fputil::exact_add(a: d1.hi, b: exp_lo);
309
310 // Combine FastTwoSum residuals and table low part:
311 // d_lo = s.lo + d1.lo + d.lo + m.lo
312 double sd1_lo = s.lo + d1.lo;
313 double sdd_lo = sd1_lo + d.lo;
314 double d_lo = sdd_lo + m.lo;
315
316 // Karp-Markstein division for q_hi + q_lo ~ (2 * m) / (d.hi + d_lo):
317 // y_hi = round(1.0 / d.hi)
318 // q_hi = round(2 * m.hi * y_hi)
319 // rem_hi = 2 * m.hi - q_hi * d.hi
320 // rem_lo = 2 * m.lo - q_hi * d_lo
321 // q_lo = (rem_hi + rem_lo) * y_hi.
322 double two_m_hi = 2.0 * m.hi;
323 double two_m_lo = 2.0 * m.lo;
324 double y_hi = 1.0 / d.hi;
325 double q_hi = two_m_hi * y_hi;
326#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE
327 double rem_hi = fputil::sub_multiply(two_m_hi, q_hi, d.hi);
328 double rem_lo = fputil::sub_multiply(two_m_lo, q_hi, d_lo);
329#else
330 // Split q_hi into 26 + 27 bits and d.hi into 27 + 26 bits via bit-masks so
331 // that q_hi_hi * d_hi_hi (53 bits) and q_hi_hi * d_hi_lo (52 bits) are exact
332 // without FMA:
333 LIBC_BIT_CAST_CONSTEXPR_VAR double HI_MASK_26 =
334 cpp::bit_cast<double>(from: 0xffff'ffff'f800'0000ULL);
335 LIBC_BIT_CAST_CONSTEXPR_VAR double HI_MASK_27 =
336 cpp::bit_cast<double>(from: 0xffff'ffff'fc00'0000ULL);
337 double q_hi_hi = fputil::fp_and(a: q_hi, b: HI_MASK_26);
338 double q_hi_lo = q_hi - q_hi_hi;
339 double d_hi_hi = fputil::fp_and(a: d.hi, b: HI_MASK_27);
340 double d_hi_lo = d.hi - d_hi_hi;
341
342 // High remainder rem_hi = 2 * m.hi - q_hi * d.hi:
343 double prod_hh = q_hi_hi * d_hi_hi; // exact (53 bits)
344 double prod_hl = q_hi_hi * d_hi_lo; // exact (52 bits)
345 double prod_lh = q_hi_lo * d_hi_hi;
346 double prod_ll = q_hi_lo * d_hi_lo;
347
348 double rem_1 = two_m_hi - prod_hh; // exact
349 double rem_2 = rem_1 - prod_hl;
350 double rem_3 = rem_2 - prod_lh;
351 double rem_hi = rem_3 - prod_ll;
352
353 // Low remainder rem_lo = 2 * m.lo - q_hi * d_lo:
354 double q_d_lo = q_hi * d_lo;
355 double rem_lo = two_m_lo - q_d_lo;
356#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE
357 double rem = rem_hi + rem_lo;
358 double q_lo = rem * y_hi;
359
360 // Subtract q_hi + q_lo from 1.0:
361 // r.hi + r.lo = 1.0 - q_hi (FastTwoSum)
362 // r_lo = r.lo - q_lo.
363 // Since 1.0 is an exact power of two and q_hi is in (0, 0.88),
364 // exact_sub(1.0, q_hi) is accurate to within 2^-105 in all rounding modes.
365 DoubleDouble r = fputil::exact_sub(a: 1.0, b: q_hi);
366 double r_lo = r.lo - q_lo;
367
368 // Apply sign(x) before Ziv's rounding test so that (r_hi + lo_u) and
369 // (r_hi + lo_l) round in the active rounding direction for negative x:
370 double sign_x = fputil::fp_xor(a: x, b: x_a);
371 double r_hi = fputil::fp_xor(a: r.hi, b: sign_x);
372 r_lo = fputil::fp_xor(a: r_lo, b: sign_x);
373
374 // Error bound for Ziv's rounding test:
375 // Since any error in e^u or q is damped by q_hi ~ 2 * e^(-2 * |x|), we scale
376 // the relative error bound by q_hi:
377 // err = q_hi * TANH_ERR_REL + TANH_ERR_ABS.
378 constexpr double TANH_ERR_REL = 0x1.8p-63;
379 constexpr double TANH_ERR_ABS = 0x1.0p-104;
380 double err = fputil::multiply_add(x: q_hi, y: TANH_ERR_REL, z: TANH_ERR_ABS);
381 double lo_u = r_lo + err;
382 double lo_l = r_lo - err;
383
384 // Ziv's rounding test in the current rounding mode:
385 double upper = r_hi + lo_u;
386 double lower = r_hi + lo_l;
387
388 if (LIBC_LIKELY(upper == lower))
389 return upper;
390
391 return tanh_accurate_128(x);
392}
393
394#endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS
395
396// Evaluate tanh(x) for |x| < SMALL_BOUND (2^-27 + ulp(2^-27) when
397// LIBC_MATH_HAS_SKIP_ACCURATE_PASS is defined, or 2^-3 otherwise):
398LIBC_INLINE double tanh_small_eval(double x, uint64_t x_abs) {
399 using FPBits = fputil::FPBits<double>;
400
401 // Tiny and subnormal range: |x| <= 2^-27.
402 // For 0 < |x| <= 2^-27:
403 // 0 < (|x| - |tanh(x)|) / |x| ~ x^2 / 3 < 2^-55 < 2^-1 * ulp(x),
404 // so tanh(x) rounds to:
405 // - nextafter(x, 0) when rounding mode is FE_TOWARDZERO, or (x > 0 and
406 // FE_DOWNWARD), or (x < 0 and FE_UPWARD),
407 // - x otherwise, with FE_INEXACT raised for all x != 0 and FE_UNDERFLOW +
408 // errno = ERANGE raised when |tanh(x)| < 2^-1022.
409#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
410 if (LIBC_UNLIKELY(x_abs <= 0x3e40'0000'0000'0000ULL))
411#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
412 {
413 if (LIBC_UNLIKELY(x_abs == 0))
414 return x;
415#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE
416 double res = fputil::sub_multiply(x, x, 0x1.0p-55);
417#else
418 // Avoid spurious FE_UNDERFLOW on x * -2^-55 when 2^-1022 < |x| < 2^-967,
419 // and preserve directed rounding for subnormal |x| <= 2^-1022:
420 double x_scaled = fputil::multiply_sub(x, y: 0x1.0p55, z: x);
421 double res = x_scaled * 0x1.0p-55;
422#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE
423 if (LIBC_UNLIKELY(x_abs <= 0x0010'0000'0000'0000ULL)) {
424 if (FPBits(res).abs().uintval() < 0x0010'0000'0000'0000ULL) {
425 fputil::set_errno_if_required(ERANGE);
426 fputil::raise_underflow_except_if_required<double>();
427 }
428 }
429 return res;
430 }
431
432#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
433 using fputil::DoubleDouble;
434
435 // 2^-27 < |x| < 2^-3:
436 // Split x = x_hi + x_lo into 14-bit x_hi and 39-bit x_lo so that:
437 // x2_hi = x_hi * x_hi (28 bits)
438 // x3_hi = x2_hi * x_hi (42 bits)
439 // y_hi = x3_hi * TANH_C0_HI_11 (42 + 11 = 53 bits)
440 // are all exact in double precision across all rounding modes (with or
441 // without FMA).
442 LIBC_BIT_CAST_CONSTEXPR_VAR double HI_MASK_14 =
443 cpp::bit_cast<double>(from: 0xffff'ff80'0000'0000ULL);
444 double x_hi = fputil::fp_and(a: x, b: HI_MASK_14);
445 double x_lo = x - x_hi;
446 double x2 = x * x;
447 double x2_hi = x_hi * x_hi; // exact (28 bits)
448 double x3_hi = x2_hi * x_hi; // exact (42 bits)
449 double y_hi = x3_hi * TANH_C0_HI_11; // exact (53 bits)
450
451 // Cubic remainder:
452 // x3_lo ~ x^3 - x_hi^3 = x_lo * (x^2 + x * x_hi + x_hi^2)
453 double x2_mid = fputil::multiply_add(x, y: x_hi, z: x2);
454 double x3_sum = x2_mid + x2_hi;
455 double x3_lo = x_lo * x3_sum;
456
457 // Polynomial remainder:
458 // x^3 * (TANH_C0_HI_11 + P_rem(x^2)) = y_hi + y_lo
459 // where:
460 // y_lo ~ x_hi^3 * P_rem(x^2) + x3_lo * (TANH_C0_HI_11 + P_rem(x^2))
461 double p_rem = fputil::estrin_polyeval(x: x2, a: TANH_SMALL_P_COEFFS);
462 double p_full = TANH_C0_HI_11 + p_rem;
463 double y_lo_t = x3_lo * p_full;
464 double y_lo = fputil::multiply_add(x: x3_hi, y: p_rem, z: y_lo_t);
465
466 // Leading sum r.hi + r.lo = x + y_hi (exact FastTwoSum in all rounding modes
467 // since |y_hi| < 2^-7 * |x|):
468 DoubleDouble r = fputil::exact_add(a: x, b: y_hi);
469
470 // Error bound for Ziv's rounding test:
471 // The error of y_hi + y_lo is bounded by |y_hi| * 0x1.8p-54, and the rounding
472 // error of r.lo + y_lo is bounded by |x| * 2^-103.
473 double x_err = x * 0x1.0p-103;
474 double err = fputil::sub_multiply(x: x_err, y: y_hi, z: 0x1.8p-54);
475
476 // Low tail bounds lo_{u,l} ~ r.lo + (y_lo +/- err):
477 double y_lo_u = y_lo + err;
478 double y_lo_l = y_lo - err;
479 double lo_u = r.lo + y_lo_u;
480 double lo_l = r.lo + y_lo_l;
481
482 // Ziv's rounding test in the current rounding mode:
483 double upper = r.hi + lo_u;
484 double lower = r.hi + lo_l;
485
486 if (LIBC_LIKELY(upper == lower))
487 return upper;
488
489 return tanh_small_accurate_128(x);
490#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
491}
492
493// Handle saturation, inf, and nan for tanh(x) when
494// |x| >= 0x1.30fc1931f09cap+4:
495LIBC_NOINLINE LIBC_INLINE double tanh_exceptional(double x) {
496 using FPBits = fputil::FPBits<double>;
497 FPBits xbits(x);
498
499 if (xbits.is_nan())
500 return x + 1.0;
501
502 if (xbits.is_inf())
503 return xbits.is_neg() ? -1.0 : 1.0;
504
505 return xbits.is_neg() ? fputil::round_result_slightly_up(value_rn: -1.0)
506 : fputil::round_result_slightly_down(value_rn: 1.0);
507}
508
509LIBC_ALWAYS_INLINE double tanh(double x) {
510 using FPBits = fputil::FPBits<double>;
511 FPBits xbits(x);
512
513 uint64_t x_abs = xbits.abs().uintval();
514
515 // Bound constants (with biased exponent and mantissa bits):
516#ifdef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
517 // - Small bound: 2^-27 + ulp(2^-27) (tanh_eval handles |x| > 2^-27 directly
518 // with k = 0 and zero cancellation when |x| < log(2) / 4)
519 constexpr uint64_t SMALL_BOUND = 0x3e40'0000'0000'0001ULL;
520#else
521 // - Small bound: 2^-3 = 0x1.0p-3
522 constexpr uint64_t SMALL_BOUND = 0x3fc0'0000'0000'0000ULL;
523#endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS
524 // - Saturation bound:
525 // > round(atanh(1 - 2^-54), D, RU) = 0x1.30fc1931f09cap+4
526 // > round(atanh(1 - 2^-54), D, RD) = 0x1.30fc1931f09c9p+4
527 // > round(tanh(0x1.30fc1931f09c9p+4), D, RN) = 0x1.fffffffffffffp-1
528 // > round(tanh(0x1.30fc1931f09cap+4), D, RN) = 0x1.0p+0
529 constexpr uint64_t OVERFLOW_BOUND = 0x4033'0fc1'931f'09caULL;
530
531 if (LIBC_UNLIKELY(x_abs < SMALL_BOUND || x_abs >= OVERFLOW_BOUND)) {
532 if (x_abs < SMALL_BOUND)
533 return tanh_small_eval(x, x_abs);
534
535 return tanh_exceptional(x);
536 }
537
538 return tanh_eval(x);
539}
540
541} // namespace double_eval
542} // namespace math
543} // namespace LIBC_NAMESPACE_DECL
544
545#endif // LLVM_LIBC_SRC___SUPPORT_MATH_TANH_DOUBLE_EVAL_H
546