1//===-- Implementation header for sincos ------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#ifndef LLVM_LIBC_SRC___SUPPORT_MATH_SINCOS_H
10#define LLVM_LIBC_SRC___SUPPORT_MATH_SINCOS_H
11
12#include "hdr/errno_macros.h"
13#include "range_reduction_double_common.h"
14#include "sincos_eval.h"
15#include "src/__support/FPUtil/FEnvImpl.h"
16#include "src/__support/FPUtil/FPBits.h"
17#include "src/__support/FPUtil/double_double.h"
18#include "src/__support/FPUtil/dyadic_float.h"
19#include "src/__support/FPUtil/except_value_utils.h"
20#include "src/__support/FPUtil/multiply_add.h"
21#include "src/__support/FPUtil/rounding_mode.h"
22#include "src/__support/common.h"
23#include "src/__support/macros/config.h"
24#include "src/__support/macros/optimization.h" // LIBC_UNLIKELY
25#include "src/__support/macros/properties/cpu_features.h" // LIBC_TARGET_CPU_HAS_FMA
26
27#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE
28#include "range_reduction_double_fma.h"
29#else
30#include "range_reduction_double_nofma.h"
31#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE
32
33namespace LIBC_NAMESPACE_DECL {
34
35namespace math {
36
37LIBC_INLINE void sincos(double x, double *sin_x, double *cos_x) {
38 using DoubleDouble = fputil::DoubleDouble;
39 using namespace math::range_reduction_double_internal;
40 using FPBits = typename fputil::FPBits<double>;
41 FPBits xbits(x);
42
43 uint16_t x_e = xbits.get_biased_exponent();
44
45 DoubleDouble y;
46 unsigned k = 0;
47 LargeRangeReduction range_reduction_large{};
48
49 // |x| < 2^16
50 if (LIBC_LIKELY(x_e < FPBits::EXP_BIAS + FAST_PASS_EXPONENT)) {
51 // |x| < 2^-7
52 if (LIBC_UNLIKELY(x_e < FPBits::EXP_BIAS - 7)) {
53 // |x| < 2^-27
54 if (LIBC_UNLIKELY(x_e < FPBits::EXP_BIAS - 27)) {
55 // Signed zeros.
56 if (LIBC_UNLIKELY(x == 0.0)) {
57 *sin_x = x;
58 *cos_x = 1.0;
59 return;
60 }
61
62 // For |x| < 2^-27, max(|sin(x) - x|, |cos(x) - 1|) < ulp(x)/2.
63#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE
64 *sin_x = fputil::multiply_add(x, -0x1.0p-54, x);
65 *cos_x = fputil::multiply_add(x, -x, 1.0);
66#else
67 *cos_x = fputil::round_result_slightly_down(value_rn: 1.0);
68
69 if (LIBC_UNLIKELY(x_e < 4)) {
70#ifndef LIBC_MATH_HAS_ASSUME_ROUND_NEAREST_ONLY
71 int rounding_mode = fputil::quick_get_round();
72 if (rounding_mode == FE_TOWARDZERO ||
73 (xbits.sign() == Sign::POS && rounding_mode == FE_DOWNWARD) ||
74 (xbits.sign() == Sign::NEG && rounding_mode == FE_UPWARD))
75 *sin_x = FPBits(xbits.uintval() - 1).get_val();
76#endif // !LIBC_MATH_HAS_ASSUME_ROUND_NEAREST_ONLY
77 }
78 *sin_x = fputil::multiply_add(x, y: -0x1.0p-54, z: x);
79#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE
80 return;
81 }
82 // No range reduction needed.
83 k = 0;
84 y.lo = 0.0;
85 y.hi = x;
86 } else {
87 // Small range reduction.
88 k = range_reduction_small(x, u&: y);
89 }
90 } else {
91 // Inf or NaN
92 if (LIBC_UNLIKELY(x_e > 2 * FPBits::EXP_BIAS)) {
93 if (xbits.is_signaling_nan()) {
94 fputil::raise_except_if_required(FE_INVALID);
95 *sin_x = *cos_x = FPBits::quiet_nan().get_val();
96 return;
97 }
98
99 // sin(+-Inf) = NaN
100 if (xbits.get_mantissa() == 0) {
101 fputil::set_errno_if_required(EDOM);
102 fputil::raise_except_if_required(FE_INVALID);
103 }
104 *sin_x = *cos_x = x + FPBits::quiet_nan().get_val();
105 return;
106 }
107
108 // Large range reduction.
109 k = range_reduction_large.fast(x, u&: y);
110 }
111
112 DoubleDouble sin_y, cos_y;
113
114 [[maybe_unused]] double err =
115 math::sincos_eval_internal::sincos_eval(u: y, sin_u&: sin_y, cos_u&: cos_y);
116
117 // Look up sin(k * pi/128) and cos(k * pi/128)
118#ifdef LIBC_MATH_HAS_SMALL_TABLES
119 // Memory saving versions. Use 65-entry table.
120 auto get_idx_dd = [](unsigned kk) -> DoubleDouble {
121 unsigned idx = (kk & 64) ? 64 - (kk & 63) : (kk & 63);
122 DoubleDouble ans = SIN_K_PI_OVER_128[idx];
123 if (kk & 128) {
124 ans.hi = -ans.hi;
125 ans.lo = -ans.lo;
126 }
127 return ans;
128 };
129 DoubleDouble sin_k = get_idx_dd(k);
130 DoubleDouble cos_k = get_idx_dd(k + 64);
131#else
132 // Fast look up version, but needs 256-entry table.
133 // cos(k * pi/128) = sin(k * pi/128 + pi/2) = sin((k + 64) * pi/128).
134 DoubleDouble sin_k = SIN_K_PI_OVER_128[k & 255];
135 DoubleDouble cos_k = SIN_K_PI_OVER_128[(k + 64) & 255];
136#endif // LIBC_MATH_HAS_SMALL_TABLES
137
138 DoubleDouble msin_k{.lo: -sin_k.lo, .hi: -sin_k.hi};
139
140 // After range reduction, k = round(x * 128 / pi) and y = x - k * (pi / 128).
141 // So k is an integer and -pi / 256 <= y <= pi / 256.
142 // Then sin(x) = sin((k * pi/128 + y)
143 // = sin(y) * cos(k*pi/128) + cos(y) * sin(k*pi/128)
144 DoubleDouble sin_k_cos_y = fputil::quick_mult(a: cos_y, b: sin_k);
145 DoubleDouble cos_k_sin_y = fputil::quick_mult(a: sin_y, b: cos_k);
146 // cos(x) = cos((k * pi/128 + y)
147 // = cos(y) * cos(k*pi/128) - sin(y) * sin(k*pi/128)
148 DoubleDouble cos_k_cos_y = fputil::quick_mult(a: cos_y, b: cos_k);
149 DoubleDouble msin_k_sin_y = fputil::quick_mult(a: sin_y, b: msin_k);
150
151 DoubleDouble sin_dd =
152 fputil::exact_add<false>(a: sin_k_cos_y.hi, b: cos_k_sin_y.hi);
153 DoubleDouble cos_dd =
154 fputil::exact_add<false>(a: cos_k_cos_y.hi, b: msin_k_sin_y.hi);
155 sin_dd.lo += sin_k_cos_y.lo + cos_k_sin_y.lo;
156 cos_dd.lo += msin_k_sin_y.lo + cos_k_cos_y.lo;
157
158#ifdef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
159 *sin_x = sin_dd.hi + sin_dd.lo;
160 *cos_x = cos_dd.hi + cos_dd.lo;
161 return;
162#else
163 // Accurate test and pass for correctly rounded implementation.
164
165 double sin_lp = sin_dd.lo + err;
166 double sin_lm = sin_dd.lo - err;
167 double cos_lp = cos_dd.lo + err;
168 double cos_lm = cos_dd.lo - err;
169
170 double sin_upper = sin_dd.hi + sin_lp;
171 double sin_lower = sin_dd.hi + sin_lm;
172 double cos_upper = cos_dd.hi + cos_lp;
173 double cos_lower = cos_dd.hi + cos_lm;
174
175 // Ziv's rounding test.
176 if (LIBC_LIKELY(sin_upper == sin_lower && cos_upper == cos_lower)) {
177 *sin_x = sin_upper;
178 *cos_x = cos_upper;
179 return;
180 }
181
182 DFloat128 u_f128, sin_u, cos_u;
183 if (LIBC_LIKELY(x_e < FPBits::EXP_BIAS + FAST_PASS_EXPONENT))
184 u_f128 = range_reduction_small_f128(x);
185 else
186 u_f128 = range_reduction_large.accurate();
187
188 math::sincos_eval_internal::sincos_eval(u: u_f128, sin_u, cos_u);
189
190 auto get_sin_k = [](unsigned kk) -> DFloat128 {
191 unsigned idx = (kk & 64) ? 64 - (kk & 63) : (kk & 63);
192 DFloat128 ans = SIN_K_PI_OVER_128_F128[idx];
193 if (kk & 128)
194 ans.sign = Sign::NEG;
195 return ans;
196 };
197
198 // cos(k * pi/128) = sin(k * pi/128 + pi/2) = sin((k + 64) * pi/128).
199 DFloat128 sin_k_f128 = get_sin_k(k);
200 DFloat128 cos_k_f128 = get_sin_k(k + 64);
201 DFloat128 msin_k_f128 = get_sin_k(k + 128);
202
203 // TODO: Add assertion if Ziv's accuracy tests fail in debug mode.
204 // https://github.com/llvm/llvm-project/issues/96452.
205
206 if (sin_upper == sin_lower)
207 *sin_x = sin_upper;
208 else
209 // sin(x) = sin((k * pi/128 + u)
210 // = sin(u) * cos(k*pi/128) + cos(u) * sin(k*pi/128)
211 *sin_x = static_cast<double>(
212 fputil::quick_add(a: fputil::quick_mul(a: sin_k_f128, b: cos_u),
213 b: fputil::quick_mul(a: cos_k_f128, b: sin_u)));
214
215 if (cos_upper == cos_lower)
216 *cos_x = cos_upper;
217 else
218 // cos(x) = cos((k * pi/128 + u)
219 // = cos(u) * cos(k*pi/128) - sin(u) * sin(k*pi/128)
220 *cos_x = static_cast<double>(
221 fputil::quick_add(a: fputil::quick_mul(a: cos_k_f128, b: cos_u),
222 b: fputil::quick_mul(a: msin_k_f128, b: sin_u)));
223
224#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
225}
226
227} // namespace math
228
229} // namespace LIBC_NAMESPACE_DECL
230
231#endif // LLVM_LIBC_SRC___SUPPORT_MATH_SINCOS_H
232