1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// Double-precision evaluation implementation for powf(x, y).
11///
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_LIBC_SRC___SUPPORT_MATH_POWF_DOUBLE_EVAL_H
15#define LLVM_LIBC_SRC___SUPPORT_MATH_POWF_DOUBLE_EVAL_H
16
17#include "src/__support/CPP/bit.h"
18#include "src/__support/CPP/optional.h"
19#include "src/__support/FPUtil/FEnvImpl.h"
20#include "src/__support/FPUtil/FPBits.h"
21#include "src/__support/FPUtil/double_double.h"
22#include "src/__support/FPUtil/except_value_utils.h"
23#include "src/__support/FPUtil/multiply_add.h"
24#include "src/__support/FPUtil/nearest_integer.h"
25#include "src/__support/FPUtil/polyeval.h"
26#include "src/__support/common.h"
27#include "src/__support/macros/attributes.h"
28#include "src/__support/macros/config.h"
29#include "src/__support/macros/optimization.h"
30#include "src/__support/macros/properties/cpu_features.h"
31#include "src/__support/math/common_constants.h"
32#include "src/__support/math/exp_constants.h"
33#include "src/__support/math/powf_utils.h"
34
35namespace LIBC_NAMESPACE_DECL {
36namespace math {
37namespace double_eval {
38
39namespace powf_internal = LIBC_NAMESPACE::math::powf_internal;
40
41#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
42// Degree-11 minimax polynomial for log2(1 + x)/x on [-2^-8, 2^-7] in
43// double-double, generated by Sollya with:
44// > display = hexadecimal;
45// > prec = 256;
46// > P = fpminimax(log2(1 + x)/x, 11, [|DD...|], [-2^(-8), 2^(-7)]);
47// > for i from 0 to 11 do {
48// c = coeff(P, i);
49// hi = round(c, D, RN);
50// lo = round(c - hi, D, RN);
51// print("{", lo, ", ", hi, "},");
52// };
53// > dirtyinfnorm((log2(1 + x) - x * P) / x, [-2^(-8), 2^(-7)]);
54// 0x1.c598...p-104 < 2^-103.
55LIBC_INLINE_VAR constexpr fputil::DoubleDouble POWF_LOG2_COEFFS_DD[12] = {
56 {.lo: 0x1.777d0ffda0d34p-56, .hi: 0x1.71547652b82fep0},
57 {.lo: -0x1.777d0ffd88ca8p-57, .hi: -0x1.71547652b82fep-1},
58 {.lo: 0x1.d27f052d9d41dp-56, .hi: 0x1.ec709dc3a03fdp-2},
59 {.lo: -0x1.777f259c6c0ap-58, .hi: -0x1.71547652b82fep-2},
60 {.lo: 0x1.e5c055599afdbp-56, .hi: 0x1.2776c50ef9bfep-2},
61 {.lo: 0x1.e923056eef156p-58, .hi: -0x1.ec709dc3a03fdp-3},
62 {.lo: 0x1.1f24e26b5ad0dp-57, .hi: 0x1.a61762a7add76p-3},
63 {.lo: 0x1.16d9b5013b55ep-58, .hi: -0x1.71547652bf304p-3},
64 {.lo: -0x1.1551148c3d1d8p-57, .hi: 0x1.484b13f00425ep-3},
65 {.lo: -0x1.a1cf327c6703p-60, .hi: -0x1.2776ca937f335p-3},
66 {.lo: -0x1.d24f6254c2e6ap-57, .hi: 0x1.0c9211efe6d7ep-3},
67 {.lo: 0x1.a81cff2946d7cp-59, .hi: -0x1.e1f45baaf7f1dp-4},
68};
69
70// Degree-8 minimax polynomial for (2^x - 1)/x on [-2^-7, 2^-7] in
71// double-double, generated by Sollya with:
72// > display = hexadecimal;
73// > prec = 256;
74// > P = fpminimax((2^x - 1)/x, 8, [|DD...|], [-2^(-7), 2^(-7)]);
75// > for i from 0 to 8 do {
76// c = coeff(P, i);
77// hi = round(c, D, RN);
78// lo = round(c - hi, D, RN);
79// print("{", lo, ", ", hi, "},");
80// };
81// > dirtyinfnorm((2^x - (1 + x * P)) / (2^x), [-2^(-7), 2^(-7)]);
82// 0x1.e62d...p-106 < 2^-105.
83LIBC_INLINE_VAR constexpr fputil::DoubleDouble POWF_EXP2_COEFFS_DD[9] = {
84 {.lo: 0x1.abc9e3b39803ep-56, .hi: 0x1.62e42fefa39efp-1},
85 {.lo: -0x1.5e43a54066432p-57, .hi: 0x1.ebfbdff82c58fp-3},
86 {.lo: -0x1.d331626e9fe04p-59, .hi: 0x1.c6b08d704a0cp-5},
87 {.lo: 0x1.4f492022eeb74p-62, .hi: 0x1.3b2ab6fba4e77p-7},
88 {.lo: 0x1.035733b00abdp-66, .hi: 0x1.5d87fe78a6731p-10},
89 {.lo: 0x1.9e4f81ec951dep-68, .hi: 0x1.430912f86c123p-13},
90 {.lo: 0x1.7be627c299948p-74, .hi: 0x1.ffcbfc588b997p-17},
91 {.lo: -0x1.c8a006d5b39acp-74, .hi: 0x1.62c03345a64f6p-20},
92 {.lo: -0x1.fd14495e4b96p-81, .hi: 0x1.b5253eef8c0a9p-24},
93};
94
95// Accurate evaluation in DoubleDouble precision when Ziv's test fails.
96LIBC_INLINE double powf_accurate(float x, float y, double e_x, uint64_t sign) {
97 using FloatBits = fputil::FPBits<float>;
98 using DoubleBits = fputil::FPBits<double>;
99 using fputil::DoubleDouble;
100
101 uint32_t x_u = FloatBits(x).uintval();
102 uint32_t x_mant = x_u & FloatBits::FRACTION_MASK;
103 int idx_x = static_cast<int>(x_mant >> (FloatBits::FRACTION_LEN - 7));
104 // Append hidden bit.
105 float m_x = cpp::bit_cast<float>(from: x_mant | 0x3f80'0000);
106 double dx = fputil::multiply_sub(
107 x: static_cast<double>(m_x),
108 y: static_cast<double>(common_constants_internal::R[idx_x]), z: 1.0);
109
110 // Evaluate P(dx) ~ log2(1 + dx) / dx using Horner's scheme.
111 DoubleDouble p = POWF_LOG2_COEFFS_DD[11];
112 for (int i = 10; i >= 0; --i)
113 p = fputil::add(a: POWF_LOG2_COEFFS_DD[i], b: fputil::quick_mult(a: dx, b: p));
114
115 // log2(1 + dx) = dx * P(dx) in DoubleDouble.
116 DoubleDouble log2_1p = fputil::quick_mult(a: dx, b: p);
117
118 // -log2(r) is represented in TripleDouble as {lo, mid, hi}.
119 // We decompose log2(x) = e_x - log2(r) + log2(1 + dx) into:
120 // log2(x) = (e_x + hi) + (log2_1p + mid + lo)
121 // = (e_x + hi) + log2_tail.
122 // The lower part log2_tail accumulates log2_1p, mid, and lo in DoubleDouble:
123 DoubleDouble log2_tail =
124 fputil::add<false>(a: log2_1p, b: powf_internal::LOG2_R_TD[idx_x].mid);
125 log2_tail = fputil::add<false>(a: log2_tail, b: powf_internal::LOG2_R_TD[idx_x].lo);
126
127 // Range reduction for 2^(y * log2(x)):
128 // Scale by 2^6 = 64 to use the 64-entry EXP2_MID1 table:
129 // 64 * y * log2(x) = y6 * (e_x + hi) + y6 * log2_tail.
130 //
131 // Since e_x + hi has at most 8 + 20 = 28 bits of precision and y has 24 bits,
132 // the product y6 * (e_x + hi) has at most 28 + 24 = 52 bits, making y6_hi
133 // exact in double precision.
134 constexpr double SCALE = 0x1.0p6;
135 double y6 = static_cast<double>(y) * SCALE;
136 DoubleDouble prod = fputil::quick_mult(a: y6, b: log2_tail);
137
138 double e_x_hi = e_x + powf_internal::LOG2_R_TD[idx_x].hi;
139 DoubleDouble y6_hi = fputil::exact_mult(a: y6, b: e_x_hi);
140
141 // hm = round(64 * y * log2(x)) is the nearest integer index.
142 double hm_d = y6_hi.hi + prod.hi;
143 double hm = fputil::nearest_integer(x: hm_d);
144
145 // lo6 = 64 * y * log2(x) - hm = (y6_hi - hm) + prod:
146 DoubleDouble lo6_hi = fputil::exact_add<false>(a: y6_hi.hi, b: -hm);
147 lo6_hi.lo += y6_hi.lo;
148
149 DoubleDouble lo6 = fputil::add<false>(a: prod, b: lo6_hi);
150
151 // lo = lo6 / 64, with |lo| <= 2^-7.
152 DoubleDouble lo = fputil::quick_mult(a: 0x1.0p-6, b: lo6);
153
154 int64_t hm_i = static_cast<int64_t>(hm);
155 int idx_y = static_cast<int>(hm_i & 0x3f);
156 int hi = static_cast<int>(hm_i >> 6);
157 int64_t exp_hi_i = static_cast<int64_t>(static_cast<uint64_t>(hi)
158 << DoubleBits::FRACTION_LEN);
159
160 DoubleDouble exp2_p = fputil::loop_polyeval(x: lo, a: POWF_EXP2_COEFFS_DD);
161
162 DoubleDouble p_lo = fputil::quick_mult(a: lo, b: exp2_p);
163 DoubleDouble exp2_lo = fputil::exact_add(a: 1.0, b: p_lo.hi);
164 exp2_lo.lo += p_lo.lo;
165
166 DoubleDouble exp2_hi_mid_dd;
167 exp2_hi_mid_dd.hi = cpp::bit_cast<double>(
168 from: exp_hi_i + cpp::bit_cast<int64_t>(from: EXP2_MID1[idx_y].hi) + sign);
169 exp2_hi_mid_dd.lo =
170 (idx_y != 0)
171 ? cpp::bit_cast<double>(
172 from: exp_hi_i + cpp::bit_cast<int64_t>(from: EXP2_MID1[idx_y].mid) + sign)
173 : 0.0;
174
175 DoubleDouble rr = fputil::quick_mult(a: exp2_hi_mid_dd, b: exp2_lo);
176
177 DoubleDouble r = fputil::exact_add(a: rr.hi, b: rr.lo);
178
179 // Round to odd to avoid double rounding when converting to float.
180 uint64_t r_bits = cpp::bit_cast<uint64_t>(from: r.hi);
181 if (LIBC_UNLIKELY(((r_bits & 0x0fff'ffffULL) == 0) && (r.lo != 0.0))) {
182 if (DoubleBits(r.hi).sign() == DoubleBits(r.lo).sign()) {
183 ++r_bits;
184 } else if ((r_bits & DoubleBits::FRACTION_MASK) > 0) {
185 --r_bits;
186 }
187 }
188
189 return cpp::bit_cast<double>(from: r_bits);
190}
191#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
192
193#if !defined(LIBC_MATH_HAS_SKIP_ACCURATE_PASS) && \
194 !defined(LIBC_MATH_HAS_SMALL_TABLES)
195// Check if x^y is an exact rounding boundary.
196// When x^y = exact_m * 2^exact_exp with exact_m <= 2^25:
197// - exact_m has at most 26 bits, so it fits in double precision without
198// rounding.
199// - For single precision, exact_exp is in [-200, 127], well within the normal
200// range [-1022, 1023] of double precision.
201// We compute exact_d = exact_m * 2^exact_exp in double precision and cast to
202// float for the final rounded result.
203LIBC_INLINE cpp::optional<float>
204check_exact_boundary(float x, float y, uint64_t sign,
205 Sign out_sign = Sign::POS) {
206 uint32_t exact_m = 0;
207 int exact_exp = 0;
208 if (LIBC_UNLIKELY(powf_internal::is_exact_rounding_boundary(x, y, exact_m,
209 exact_exp))) {
210 // Number of bits in exact_m: 2^(l - 1) <= exact_m < 2^l.
211 int l = 32 - cpp::countl_zero(value: exact_m);
212 // Unbiased exponent of exact_m * 2^exact_exp.
213 int unbiased_exp = exact_exp + l - 1;
214
215 if (LIBC_UNLIKELY(unbiased_exp > 127))
216 return fputil::set_overflow<float>(out_sign);
217
218 // Below the minimum subnormal float (2^-149).
219 if (LIBC_UNLIKELY(unbiased_exp < -149)) {
220 fputil::set_errno_if_required(ERANGE);
221 fputil::raise_underflow_except_if_required<float>();
222 }
223
224 // Scale factor = sign * 2^exact_exp.
225 using DoubleBits = fputil::FPBits<double>;
226 double scale = cpp::bit_cast<double>(
227 from: (static_cast<uint64_t>(exact_exp + DoubleBits::EXP_BIAS)
228 << DoubleBits::FRACTION_LEN) |
229 sign);
230 double exact_d = static_cast<double>(exact_m) * scale;
231 return static_cast<float>(exact_d);
232 }
233 return cpp::nullopt;
234}
235#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS && !LIBC_MATH_HAS_SMALL_TABLES
236
237// Degree-6 minimax polynomial approximation for log2(1 + dx)/dx on
238// [-2^-6, 2^-5]. Generated by Sollya with:
239// > display = hexadecimal;
240// > p6 = fpminimax(log2(1 + x)/x, 6, [|D...|], [-2^-6, 2^-5]);
241// > for i from 0 to 6 do print(round(coeff(p6, i), D, RN), ",");
242// > dirtyinfnorm((log2(1 + x) - x * p6) / log2(1 + x), [-2^-6, 2^-5]);
243// 0x1.050a...p-47
244// > dirtyinfnorm(log2(1 + x) - x * p6, [-2^-6, 2^-5]);
245// 0x1.712b...p-52
246LIBC_INLINE_VAR constexpr double POWF_LOG2_COEFFS[7] = {
247 0x1.71547652b831fp0, -0x1.71547652b2ff5p-1, 0x1.ec709dbd02463p-2,
248 -0x1.7154787f73adep-2, 0x1.2777b7f57b29fp-2, -0x1.ec55ea50eaba7p-3,
249 0x1.92c78e7d5a931p-3,
250};
251
252// Degree-7 polynomial approximation P(f) ~ (2^f - 1)/f on [-0.5, 0.5]
253// Generated by Sollya with:
254// > display = hexadecimal;
255// > p = fpminimax((2^x - 1)/x, 7, [|D...|], [-0.5, 0.5]);
256// > for i from 0 to 7 do print(round(coeff(p, i), D, RN), ",");
257// > dirtyinfnorm(2^x - (1 + x * p), [-0.5, 0.5]);
258// 0x1.0496...p-39
259// > dirtyinfnorm((2^x - (1 + x * p)) / 2^x, [-0.5, 0.5]);
260// 0x1.0496...p-39
261LIBC_INLINE_VAR constexpr double POWF_EXP2_COEFFS[8] = {
262 0x1.62e42fef9cdf7p-1, 0x1.ebfbdff85f49p-3, 0x1.c6b08da69c963p-5,
263 0x1.3b2ab6a0e1172p-7, 0x1.5d87762dfc134p-10, 0x1.43099b4e829a9p-13,
264 0x1.00c080f7699bep-16, 0x1.62c0108a065d8p-20,
265};
266
267// Overview of powf(x, y) = x^y computation in double precision:
268//
269// Let x = 2^(e_x) * m_x > 0. Then:
270// x^y = 2^( y * log2(x) )
271// = 2^( y * ( e_x + log2(m_x) ) )
272// = 2^( k + f )
273// = 2^k * 2^f,
274// where:
275// k = round(y * log2(x)),
276// f = y * log2(x) - k.
277//
278// In particular, k is an integer, and |f| <= 0.5.
279// For the final result to fit in single precision:
280// -150 <= y * log2(x) <= 128,
281// and anything outside that range overflows or underflows.
282//
283// Fast pass:
284// 1. Range reduction for log2(m_x) using a 32-entry lookup table:
285// dx = r * m_x - 1 in [-2^-6, 2^-5],
286// log2(m_x) = log2(1 + dx) - log2(r),
287// where -log2(r) is retrieved from LOG2_R_32[idx_x], and log2(1 + dx)/dx is
288// approximated by a degree-6 polynomial in double precision with Estrin's
289// scheme.
290// 2. Exponent reduction:
291// z = y * log2(x),
292// k = round(z),
293// f = z - k in [-0.5, 0.5].
294// 3. 2^f is evaluated using a degree-7 polynomial.
295// Absorbing +-ERR into the constant term 1.0 +- ERR of the final FMA allows
296// computing the upper and lower bounds for Ziv's rounding test in parallel.
297// 4. If upper == lower in single precision, return upper.
298// 5. If Ziv's test fails, fall back to check_exact_boundary and powf_accurate
299// (DoubleDouble / 128-entry table).
300//
301// Accurate pass:
302// 1. Range reduction using a 128-entry table: -log2(r) = {lo, mid, hi}.
303// log2(1 + dx) is evaluated using a degree-11 polynomial in DoubleDouble.
304// 2. Decompose: log2(x) = (e_x + hi) + log2_tail, where:
305// log2_tail = log2(1 + dx) + mid + lo.
306// Because e_x + hi has <= 28 bits of precision and y has 24 bits,
307// y * (e_x + hi) is exact in double precision.
308// 3. Exponent reduction with 64-entry EXP2_MID1 table and degree-8 polynomial
309// in DoubleDouble.
310LIBC_INLINE float powf(float x, float y) {
311 using FloatBits = fputil::FPBits<float>;
312 using DoubleBits = fputil::FPBits<double>;
313 using fputil::DoubleDouble;
314 using namespace common_constants_internal;
315
316 FloatBits xbits(x), ybits(y);
317 uint32_t x_u = xbits.uintval();
318 uint32_t y_u = ybits.uintval();
319 uint32_t y_a = ybits.abs().uintval();
320
321 // Quick filter for special inputs:
322 // - x: 0, +-1, 2^k, +-Inf, NaN.
323 // - y: 0, +-1, +-2, +-0.5, +-Inf, NaN.
324 if (LIBC_UNLIKELY((x_u & 0x001F'FFFF) == 0 || (y_u & 0x007F'FFFF) == 0)) {
325 if (auto r = powf_internal::check_special_inputs(x, y);
326 LIBC_UNLIKELY(r.has_value()))
327 return r.value();
328 }
329
330#if !defined(LIBC_MATH_HAS_SKIP_ACCURATE_PASS) && \
331 !defined(LIBC_MATH_HAS_SMALL_TABLES)
332 float orig_x_abs = xbits.abs().get_val();
333 float orig_y = y;
334#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS && !LIBC_MATH_HAS_SMALL_TABLES
335
336 int ex = -FloatBits::EXP_BIAS;
337 uint64_t sign = 0;
338
339 // Check for exceptional cases:
340 // - |y| <= 2^-40 or |y| >= 2^40.
341 // - x < 0, x is subnormal, zero, Inf, or NaN.
342 if (LIBC_UNLIKELY(y_a <= powf_internal::Y_LOWER_BOUND ||
343 y_a >= powf_internal::Y_UPPER_BOUND ||
344 x_u >= FloatBits::inf().uintval() ||
345 x_u < FloatBits::min_normal().uintval())) {
346 if (auto r = powf_internal::check_exceptional_cases(x, y, ex, sign);
347 LIBC_UNLIKELY(r.has_value()))
348 return r.value();
349 }
350
351 Sign out_sign = (sign == 0) ? Sign::POS : Sign::NEG;
352
353 // x^y = 2^( y * log2(x) ) = 2^( y * ( e_x + log2(m_x) ) )
354 // Compute log2(x) = e_x + log2(m_x)
355 x_u = FloatBits(x).uintval();
356
357 // Extract exponent field of x.
358 ex += (x_u >> FloatBits::FRACTION_LEN);
359 double e_x = static_cast<double>(ex);
360 uint32_t x_mant = x_u & FloatBits::FRACTION_MASK;
361 // Top 5 bits of mantissa for 32-entry table lookup.
362 int idx_x = static_cast<int>(x_mant >> (FloatBits::FRACTION_LEN - 5));
363 // Embed mantissa into double format: m_x in [1.0, 2.0).
364 uint64_t m_x_u =
365 (static_cast<uint64_t>(x_mant) << 29) | 0x3ff0'0000'0000'0000ULL;
366 double m_x = cpp::bit_cast<double>(from: m_x_u);
367
368 // Range reduction: dx = m_x * r - 1.0 in [-2^-6, 2^-5].
369 // Since r has <= 7 bits and m_x has 24 bits, m_x * r - 1.0 is exact in
370 // double.
371 double dx =
372 fputil::multiply_sub(x: m_x, y: powf_internal::R_32_D[idx_x], z: 1.0); // Exact
373
374 // Evaluate P(dx) ~ log2(1 + dx)/dx with Estrin's scheme:
375 double p = fputil::estrin_polyeval(x: dx, a: POWF_LOG2_COEFFS);
376
377 // log2(x) = e_x - log2(r) + dx * P(dx).
378 double log2_r_ex = powf_internal::LOG2_R_32[idx_x] + e_x;
379 double s = fputil::multiply_add(x: dx, y: p, z: log2_r_ex);
380
381 double y_d = static_cast<double>(y);
382 double z = y_d * s;
383
384 // y * log2(x) = k + f, with k integer and |f| <= 0.5.
385 double kd = fputil::nearest_integer(x: z);
386 int64_t k = static_cast<int64_t>(kd);
387 double f = fputil::multiply_sub(x: y_d, y: s, z: kd);
388
389 // Evaluate P(f) ~ (2^f - 1)/f on [-0.5, 0.5] with Estrin's scheme:
390 double poly = fputil::estrin_polyeval(x: f, a: POWF_EXP2_COEFFS);
391
392 // Normal range: -125 <= k <= 128.
393 // (k + 125) as unsigned <= 253 covers all normal outputs.
394 uint64_t k_u = static_cast<uint64_t>(k + 125);
395
396 if (LIBC_LIKELY(k_u <= 253)) {
397 // Scale by 2^k.
398 uint64_t exp2_k_i =
399 (static_cast<uint64_t>(static_cast<int>(k) + DoubleBits::EXP_BIAS)
400 << DoubleBits::FRACTION_LEN) |
401 sign;
402 double exp2_k = cpp::bit_cast<double>(from: exp2_k_i);
403
404#ifdef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
405 double pp = fputil::multiply_add(f, poly, 1.0);
406 double r_d = pp * exp2_k;
407 return static_cast<float>(r_d);
408#else // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
409#ifdef LIBC_TARGET_CPU_HAS_FMA
410 // Absorbing +-ERR into 1.0 computes both bounds using two parallel FMAs.
411 constexpr double ERR = 0x1.5p-39;
412 double pp_hi = fputil::multiply_add(f, poly, 1.0 + ERR);
413 double pp_lo = fputil::multiply_add(f, poly, 1.0 - ERR);
414 double r_d = pp_hi * exp2_k;
415#else // !LIBC_TARGET_CPU_HAS_FMA
416 // Without FMA, compute unbiased pp = f * poly + 1.0 directly.
417 double pp = fputil::multiply_add(x: f, y: poly, z: 1.0);
418 double r_d = pp * exp2_k;
419#endif // LIBC_TARGET_CPU_HAS_FMA
420
421 float res = static_cast<float>(r_d);
422
423 // Since 2^k is an exact power of 2, testing if the bounds round to the same
424 // float is equivalent to testing the fully scaled bounds.
425#ifdef LIBC_TARGET_CPU_HAS_FMA
426 float upper = static_cast<float>(pp_hi);
427 float lower = static_cast<float>(pp_lo);
428#else // !LIBC_TARGET_CPU_HAS_FMA
429 constexpr double ERR = 0x1.5p-39;
430 float upper = static_cast<float>(pp + ERR);
431 float lower = static_cast<float>(pp - ERR);
432#endif // LIBC_TARGET_CPU_HAS_FMA
433
434 if (LIBC_LIKELY(upper == lower))
435 return res;
436
437 // Accurate fallback when Ziv's test fails:
438#ifndef LIBC_MATH_HAS_SMALL_TABLES
439 if (auto r = check_exact_boundary(x: orig_x_abs, y: orig_y, sign, out_sign);
440 LIBC_UNLIKELY(r.has_value()))
441 return r.value();
442#endif // !LIBC_MATH_HAS_SMALL_TABLES
443
444 double r_dd = powf_accurate(x, y, e_x, sign);
445 res = static_cast<float>(r_dd);
446 if (LIBC_UNLIKELY(FloatBits(res).is_inf()))
447 return fputil::set_overflow<float>(out_sign);
448 return res;
449#endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS
450 }
451
452#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
453 // Exceptional path: k > 128 (overflow), k < -155 (underflow), or denormal.
454 if (k > 128)
455 return fputil::set_overflow<float>(out_sign);
456
457 if (k < -155)
458 return fputil::set_underflow<float>(out_sign);
459
460 // Denormal and underflow path for -155 <= k <= -126:
461#ifndef LIBC_MATH_HAS_SMALL_TABLES
462 if (auto r = check_exact_boundary(x: orig_x_abs, y: orig_y, sign, out_sign);
463 LIBC_UNLIKELY(r.has_value()))
464 return r.value();
465#endif // !LIBC_MATH_HAS_SMALL_TABLES
466
467 int64_t exp2_k_i = (static_cast<uint64_t>(k + DoubleBits::EXP_BIAS)
468 << DoubleBits::FRACTION_LEN) |
469 sign;
470 double exp2_k = cpp::bit_cast<double>(from: exp2_k_i);
471
472 // Add +-2^(-126 + (53 - 24)) = +-2^-97 to mimic single precision denormal
473 // rounding in double precision.
474 // For -155 <= k <= -126, the least significant bit of the double precision
475 // sum (r_d + denorm_bias) is aligned at 2^(-97 - 52) = 2^-149, matching the
476 // least significant bit of single precision denormals 2^(-126 - 23) = 2^-149.
477 // This keeps the values in the double precision normal range while performing
478 // rounding according to the current rounding mode.
479 double denorm_bias = cpp::bit_cast<double>(from: 0x39e0'0000'0000'0000ULL | sign);
480
481 constexpr double ERR = 0x1.5p-39;
482
483#ifdef LIBC_TARGET_CPU_HAS_FMA
484 // Absorbing +-ERR into 1.0 computes both bounds using two parallel FMAs.
485 double pp_hi = fputil::multiply_add(f, poly, 1.0 + ERR);
486 double pp_lo = fputil::multiply_add(f, poly, 1.0 - ERR);
487 double u_hi = fputil::multiply_add(pp_hi, exp2_k, denorm_bias);
488 double u_lo = fputil::multiply_add(pp_lo, exp2_k, denorm_bias);
489#else // !LIBC_TARGET_CPU_HAS_FMA
490 // Without FMA, compute unbiased pp = f * poly + 1.0 directly.
491 double pp = fputil::multiply_add(x: f, y: poly, z: 1.0);
492 double r_d = pp * exp2_k;
493 double err = ERR * r_d;
494 double u_hi = (r_d + err) + denorm_bias;
495 double u_lo = (r_d - err) + denorm_bias;
496#endif // LIBC_TARGET_CPU_HAS_FMA
497
498 if (LIBC_LIKELY(u_hi == u_lo)) {
499 if (LIBC_UNLIKELY(u_hi == denorm_bias))
500 return fputil::set_underflow<float>(out_sign);
501
502 float res = static_cast<float>(u_hi - denorm_bias);
503 if (LIBC_UNLIKELY(FloatBits(res).is_normal())) {
504#ifdef LIBC_TARGET_CPU_HAS_FMA
505 double pp = fputil::multiply_add(f, poly, 1.0);
506#endif // LIBC_TARGET_CPU_HAS_FMA
507 if (static_cast<float>(pp) < 1.0f)
508 fputil::raise_underflow_except_if_required<float>();
509 return res;
510 }
511
512 fputil::set_errno_if_required(ERANGE);
513 fputil::raise_underflow_except_if_required<float>();
514 return res;
515 }
516
517 // Ziv's test failed for denormal input, fall back to accurate pass.
518 double r_dd = powf_accurate(x, y, e_x, sign);
519 float res = static_cast<float>(r_dd);
520 if (LIBC_UNLIKELY(FloatBits(res).is_normal())) {
521 if (static_cast<float>(r_dd * 0x1.0p126) < 1.0f)
522 fputil::raise_underflow_except_if_required<float>();
523 return res;
524 }
525 fputil::set_errno_if_required(ERANGE);
526 fputil::raise_underflow_except_if_required<float>();
527 return res;
528#else // LIBC_MATH_HAS_SKIP_ACCURATE_PASS
529 if (k > 128)
530 return fputil::set_overflow<float>(out_sign);
531 return fputil::set_underflow<float>(out_sign);
532#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
533}
534
535} // namespace double_eval
536} // namespace math
537} // namespace LIBC_NAMESPACE_DECL
538
539#endif // LLVM_LIBC_SRC___SUPPORT_MATH_POWF_DOUBLE_EVAL_H
540