| 1 | //===----------------------------------------------------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | /// |
| 9 | /// \file |
| 10 | /// Double-precision evaluation implementation for powf(x, y). |
| 11 | /// |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
| 14 | #ifndef LLVM_LIBC_SRC___SUPPORT_MATH_POWF_DOUBLE_EVAL_H |
| 15 | #define LLVM_LIBC_SRC___SUPPORT_MATH_POWF_DOUBLE_EVAL_H |
| 16 | |
| 17 | #include "src/__support/CPP/bit.h" |
| 18 | #include "src/__support/CPP/optional.h" |
| 19 | #include "src/__support/FPUtil/FEnvImpl.h" |
| 20 | #include "src/__support/FPUtil/FPBits.h" |
| 21 | #include "src/__support/FPUtil/double_double.h" |
| 22 | #include "src/__support/FPUtil/except_value_utils.h" |
| 23 | #include "src/__support/FPUtil/multiply_add.h" |
| 24 | #include "src/__support/FPUtil/nearest_integer.h" |
| 25 | #include "src/__support/FPUtil/polyeval.h" |
| 26 | #include "src/__support/common.h" |
| 27 | #include "src/__support/macros/attributes.h" |
| 28 | #include "src/__support/macros/config.h" |
| 29 | #include "src/__support/macros/optimization.h" |
| 30 | #include "src/__support/macros/properties/cpu_features.h" |
| 31 | #include "src/__support/math/common_constants.h" |
| 32 | #include "src/__support/math/exp_constants.h" |
| 33 | #include "src/__support/math/powf_utils.h" |
| 34 | |
| 35 | namespace LIBC_NAMESPACE_DECL { |
| 36 | namespace math { |
| 37 | namespace double_eval { |
| 38 | |
| 39 | namespace powf_internal = LIBC_NAMESPACE::math::powf_internal; |
| 40 | |
| 41 | #ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS |
| 42 | // Degree-11 minimax polynomial for log2(1 + x)/x on [-2^-8, 2^-7] in |
| 43 | // double-double, generated by Sollya with: |
| 44 | // > display = hexadecimal; |
| 45 | // > prec = 256; |
| 46 | // > P = fpminimax(log2(1 + x)/x, 11, [|DD...|], [-2^(-8), 2^(-7)]); |
| 47 | // > for i from 0 to 11 do { |
| 48 | // c = coeff(P, i); |
| 49 | // hi = round(c, D, RN); |
| 50 | // lo = round(c - hi, D, RN); |
| 51 | // print("{", lo, ", ", hi, "},"); |
| 52 | // }; |
| 53 | // > dirtyinfnorm((log2(1 + x) - x * P) / x, [-2^(-8), 2^(-7)]); |
| 54 | // 0x1.c598...p-104 < 2^-103. |
| 55 | LIBC_INLINE_VAR constexpr fputil::DoubleDouble POWF_LOG2_COEFFS_DD[12] = { |
| 56 | {.lo: 0x1.777d0ffda0d34p-56, .hi: 0x1.71547652b82fep0}, |
| 57 | {.lo: -0x1.777d0ffd88ca8p-57, .hi: -0x1.71547652b82fep-1}, |
| 58 | {.lo: 0x1.d27f052d9d41dp-56, .hi: 0x1.ec709dc3a03fdp-2}, |
| 59 | {.lo: -0x1.777f259c6c0ap-58, .hi: -0x1.71547652b82fep-2}, |
| 60 | {.lo: 0x1.e5c055599afdbp-56, .hi: 0x1.2776c50ef9bfep-2}, |
| 61 | {.lo: 0x1.e923056eef156p-58, .hi: -0x1.ec709dc3a03fdp-3}, |
| 62 | {.lo: 0x1.1f24e26b5ad0dp-57, .hi: 0x1.a61762a7add76p-3}, |
| 63 | {.lo: 0x1.16d9b5013b55ep-58, .hi: -0x1.71547652bf304p-3}, |
| 64 | {.lo: -0x1.1551148c3d1d8p-57, .hi: 0x1.484b13f00425ep-3}, |
| 65 | {.lo: -0x1.a1cf327c6703p-60, .hi: -0x1.2776ca937f335p-3}, |
| 66 | {.lo: -0x1.d24f6254c2e6ap-57, .hi: 0x1.0c9211efe6d7ep-3}, |
| 67 | {.lo: 0x1.a81cff2946d7cp-59, .hi: -0x1.e1f45baaf7f1dp-4}, |
| 68 | }; |
| 69 | |
| 70 | // Degree-8 minimax polynomial for (2^x - 1)/x on [-2^-7, 2^-7] in |
| 71 | // double-double, generated by Sollya with: |
| 72 | // > display = hexadecimal; |
| 73 | // > prec = 256; |
| 74 | // > P = fpminimax((2^x - 1)/x, 8, [|DD...|], [-2^(-7), 2^(-7)]); |
| 75 | // > for i from 0 to 8 do { |
| 76 | // c = coeff(P, i); |
| 77 | // hi = round(c, D, RN); |
| 78 | // lo = round(c - hi, D, RN); |
| 79 | // print("{", lo, ", ", hi, "},"); |
| 80 | // }; |
| 81 | // > dirtyinfnorm((2^x - (1 + x * P)) / (2^x), [-2^(-7), 2^(-7)]); |
| 82 | // 0x1.e62d...p-106 < 2^-105. |
| 83 | LIBC_INLINE_VAR constexpr fputil::DoubleDouble POWF_EXP2_COEFFS_DD[9] = { |
| 84 | {.lo: 0x1.abc9e3b39803ep-56, .hi: 0x1.62e42fefa39efp-1}, |
| 85 | {.lo: -0x1.5e43a54066432p-57, .hi: 0x1.ebfbdff82c58fp-3}, |
| 86 | {.lo: -0x1.d331626e9fe04p-59, .hi: 0x1.c6b08d704a0cp-5}, |
| 87 | {.lo: 0x1.4f492022eeb74p-62, .hi: 0x1.3b2ab6fba4e77p-7}, |
| 88 | {.lo: 0x1.035733b00abdp-66, .hi: 0x1.5d87fe78a6731p-10}, |
| 89 | {.lo: 0x1.9e4f81ec951dep-68, .hi: 0x1.430912f86c123p-13}, |
| 90 | {.lo: 0x1.7be627c299948p-74, .hi: 0x1.ffcbfc588b997p-17}, |
| 91 | {.lo: -0x1.c8a006d5b39acp-74, .hi: 0x1.62c03345a64f6p-20}, |
| 92 | {.lo: -0x1.fd14495e4b96p-81, .hi: 0x1.b5253eef8c0a9p-24}, |
| 93 | }; |
| 94 | |
| 95 | // Accurate evaluation in DoubleDouble precision when Ziv's test fails. |
| 96 | LIBC_INLINE double powf_accurate(float x, float y, double e_x, uint64_t sign) { |
| 97 | using FloatBits = fputil::FPBits<float>; |
| 98 | using DoubleBits = fputil::FPBits<double>; |
| 99 | using fputil::DoubleDouble; |
| 100 | |
| 101 | uint32_t x_u = FloatBits(x).uintval(); |
| 102 | uint32_t x_mant = x_u & FloatBits::FRACTION_MASK; |
| 103 | int idx_x = static_cast<int>(x_mant >> (FloatBits::FRACTION_LEN - 7)); |
| 104 | // Append hidden bit. |
| 105 | float m_x = cpp::bit_cast<float>(from: x_mant | 0x3f80'0000); |
| 106 | double dx = fputil::multiply_sub( |
| 107 | x: static_cast<double>(m_x), |
| 108 | y: static_cast<double>(common_constants_internal::R[idx_x]), z: 1.0); |
| 109 | |
| 110 | // Evaluate P(dx) ~ log2(1 + dx) / dx using Horner's scheme. |
| 111 | DoubleDouble p = POWF_LOG2_COEFFS_DD[11]; |
| 112 | for (int i = 10; i >= 0; --i) |
| 113 | p = fputil::add(a: POWF_LOG2_COEFFS_DD[i], b: fputil::quick_mult(a: dx, b: p)); |
| 114 | |
| 115 | // log2(1 + dx) = dx * P(dx) in DoubleDouble. |
| 116 | DoubleDouble log2_1p = fputil::quick_mult(a: dx, b: p); |
| 117 | |
| 118 | // -log2(r) is represented in TripleDouble as {lo, mid, hi}. |
| 119 | // We decompose log2(x) = e_x - log2(r) + log2(1 + dx) into: |
| 120 | // log2(x) = (e_x + hi) + (log2_1p + mid + lo) |
| 121 | // = (e_x + hi) + log2_tail. |
| 122 | // The lower part log2_tail accumulates log2_1p, mid, and lo in DoubleDouble: |
| 123 | DoubleDouble log2_tail = |
| 124 | fputil::add<false>(a: log2_1p, b: powf_internal::LOG2_R_TD[idx_x].mid); |
| 125 | log2_tail = fputil::add<false>(a: log2_tail, b: powf_internal::LOG2_R_TD[idx_x].lo); |
| 126 | |
| 127 | // Range reduction for 2^(y * log2(x)): |
| 128 | // Scale by 2^6 = 64 to use the 64-entry EXP2_MID1 table: |
| 129 | // 64 * y * log2(x) = y6 * (e_x + hi) + y6 * log2_tail. |
| 130 | // |
| 131 | // Since e_x + hi has at most 8 + 20 = 28 bits of precision and y has 24 bits, |
| 132 | // the product y6 * (e_x + hi) has at most 28 + 24 = 52 bits, making y6_hi |
| 133 | // exact in double precision. |
| 134 | constexpr double SCALE = 0x1.0p6; |
| 135 | double y6 = static_cast<double>(y) * SCALE; |
| 136 | DoubleDouble prod = fputil::quick_mult(a: y6, b: log2_tail); |
| 137 | |
| 138 | double e_x_hi = e_x + powf_internal::LOG2_R_TD[idx_x].hi; |
| 139 | DoubleDouble y6_hi = fputil::exact_mult(a: y6, b: e_x_hi); |
| 140 | |
| 141 | // hm = round(64 * y * log2(x)) is the nearest integer index. |
| 142 | double hm_d = y6_hi.hi + prod.hi; |
| 143 | double hm = fputil::nearest_integer(x: hm_d); |
| 144 | |
| 145 | // lo6 = 64 * y * log2(x) - hm = (y6_hi - hm) + prod: |
| 146 | DoubleDouble lo6_hi = fputil::exact_add<false>(a: y6_hi.hi, b: -hm); |
| 147 | lo6_hi.lo += y6_hi.lo; |
| 148 | |
| 149 | DoubleDouble lo6 = fputil::add<false>(a: prod, b: lo6_hi); |
| 150 | |
| 151 | // lo = lo6 / 64, with |lo| <= 2^-7. |
| 152 | DoubleDouble lo = fputil::quick_mult(a: 0x1.0p-6, b: lo6); |
| 153 | |
| 154 | int64_t hm_i = static_cast<int64_t>(hm); |
| 155 | int idx_y = static_cast<int>(hm_i & 0x3f); |
| 156 | int hi = static_cast<int>(hm_i >> 6); |
| 157 | int64_t exp_hi_i = static_cast<int64_t>(static_cast<uint64_t>(hi) |
| 158 | << DoubleBits::FRACTION_LEN); |
| 159 | |
| 160 | DoubleDouble exp2_p = fputil::loop_polyeval(x: lo, a: POWF_EXP2_COEFFS_DD); |
| 161 | |
| 162 | DoubleDouble p_lo = fputil::quick_mult(a: lo, b: exp2_p); |
| 163 | DoubleDouble exp2_lo = fputil::exact_add(a: 1.0, b: p_lo.hi); |
| 164 | exp2_lo.lo += p_lo.lo; |
| 165 | |
| 166 | DoubleDouble exp2_hi_mid_dd; |
| 167 | exp2_hi_mid_dd.hi = cpp::bit_cast<double>( |
| 168 | from: exp_hi_i + cpp::bit_cast<int64_t>(from: EXP2_MID1[idx_y].hi) + sign); |
| 169 | exp2_hi_mid_dd.lo = |
| 170 | (idx_y != 0) |
| 171 | ? cpp::bit_cast<double>( |
| 172 | from: exp_hi_i + cpp::bit_cast<int64_t>(from: EXP2_MID1[idx_y].mid) + sign) |
| 173 | : 0.0; |
| 174 | |
| 175 | DoubleDouble rr = fputil::quick_mult(a: exp2_hi_mid_dd, b: exp2_lo); |
| 176 | |
| 177 | DoubleDouble r = fputil::exact_add(a: rr.hi, b: rr.lo); |
| 178 | |
| 179 | // Round to odd to avoid double rounding when converting to float. |
| 180 | uint64_t r_bits = cpp::bit_cast<uint64_t>(from: r.hi); |
| 181 | if (LIBC_UNLIKELY(((r_bits & 0x0fff'ffffULL) == 0) && (r.lo != 0.0))) { |
| 182 | if (DoubleBits(r.hi).sign() == DoubleBits(r.lo).sign()) { |
| 183 | ++r_bits; |
| 184 | } else if ((r_bits & DoubleBits::FRACTION_MASK) > 0) { |
| 185 | --r_bits; |
| 186 | } |
| 187 | } |
| 188 | |
| 189 | return cpp::bit_cast<double>(from: r_bits); |
| 190 | } |
| 191 | #endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS |
| 192 | |
| 193 | #if !defined(LIBC_MATH_HAS_SKIP_ACCURATE_PASS) && \ |
| 194 | !defined(LIBC_MATH_HAS_SMALL_TABLES) |
| 195 | // Check if x^y is an exact rounding boundary. |
| 196 | // When x^y = exact_m * 2^exact_exp with exact_m <= 2^25: |
| 197 | // - exact_m has at most 26 bits, so it fits in double precision without |
| 198 | // rounding. |
| 199 | // - For single precision, exact_exp is in [-200, 127], well within the normal |
| 200 | // range [-1022, 1023] of double precision. |
| 201 | // We compute exact_d = exact_m * 2^exact_exp in double precision and cast to |
| 202 | // float for the final rounded result. |
| 203 | LIBC_INLINE cpp::optional<float> |
| 204 | check_exact_boundary(float x, float y, uint64_t sign, |
| 205 | Sign out_sign = Sign::POS) { |
| 206 | uint32_t exact_m = 0; |
| 207 | int exact_exp = 0; |
| 208 | if (LIBC_UNLIKELY(powf_internal::is_exact_rounding_boundary(x, y, exact_m, |
| 209 | exact_exp))) { |
| 210 | // Number of bits in exact_m: 2^(l - 1) <= exact_m < 2^l. |
| 211 | int l = 32 - cpp::countl_zero(value: exact_m); |
| 212 | // Unbiased exponent of exact_m * 2^exact_exp. |
| 213 | int unbiased_exp = exact_exp + l - 1; |
| 214 | |
| 215 | if (LIBC_UNLIKELY(unbiased_exp > 127)) |
| 216 | return fputil::set_overflow<float>(out_sign); |
| 217 | |
| 218 | // Below the minimum subnormal float (2^-149). |
| 219 | if (LIBC_UNLIKELY(unbiased_exp < -149)) { |
| 220 | fputil::set_errno_if_required(ERANGE); |
| 221 | fputil::raise_underflow_except_if_required<float>(); |
| 222 | } |
| 223 | |
| 224 | // Scale factor = sign * 2^exact_exp. |
| 225 | using DoubleBits = fputil::FPBits<double>; |
| 226 | double scale = cpp::bit_cast<double>( |
| 227 | from: (static_cast<uint64_t>(exact_exp + DoubleBits::EXP_BIAS) |
| 228 | << DoubleBits::FRACTION_LEN) | |
| 229 | sign); |
| 230 | double exact_d = static_cast<double>(exact_m) * scale; |
| 231 | return static_cast<float>(exact_d); |
| 232 | } |
| 233 | return cpp::nullopt; |
| 234 | } |
| 235 | #endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS && !LIBC_MATH_HAS_SMALL_TABLES |
| 236 | |
| 237 | // Degree-6 minimax polynomial approximation for log2(1 + dx)/dx on |
| 238 | // [-2^-6, 2^-5]. Generated by Sollya with: |
| 239 | // > display = hexadecimal; |
| 240 | // > p6 = fpminimax(log2(1 + x)/x, 6, [|D...|], [-2^-6, 2^-5]); |
| 241 | // > for i from 0 to 6 do print(round(coeff(p6, i), D, RN), ","); |
| 242 | // > dirtyinfnorm((log2(1 + x) - x * p6) / log2(1 + x), [-2^-6, 2^-5]); |
| 243 | // 0x1.050a...p-47 |
| 244 | // > dirtyinfnorm(log2(1 + x) - x * p6, [-2^-6, 2^-5]); |
| 245 | // 0x1.712b...p-52 |
| 246 | LIBC_INLINE_VAR constexpr double POWF_LOG2_COEFFS[7] = { |
| 247 | 0x1.71547652b831fp0, -0x1.71547652b2ff5p-1, 0x1.ec709dbd02463p-2, |
| 248 | -0x1.7154787f73adep-2, 0x1.2777b7f57b29fp-2, -0x1.ec55ea50eaba7p-3, |
| 249 | 0x1.92c78e7d5a931p-3, |
| 250 | }; |
| 251 | |
| 252 | // Degree-7 polynomial approximation P(f) ~ (2^f - 1)/f on [-0.5, 0.5] |
| 253 | // Generated by Sollya with: |
| 254 | // > display = hexadecimal; |
| 255 | // > p = fpminimax((2^x - 1)/x, 7, [|D...|], [-0.5, 0.5]); |
| 256 | // > for i from 0 to 7 do print(round(coeff(p, i), D, RN), ","); |
| 257 | // > dirtyinfnorm(2^x - (1 + x * p), [-0.5, 0.5]); |
| 258 | // 0x1.0496...p-39 |
| 259 | // > dirtyinfnorm((2^x - (1 + x * p)) / 2^x, [-0.5, 0.5]); |
| 260 | // 0x1.0496...p-39 |
| 261 | LIBC_INLINE_VAR constexpr double POWF_EXP2_COEFFS[8] = { |
| 262 | 0x1.62e42fef9cdf7p-1, 0x1.ebfbdff85f49p-3, 0x1.c6b08da69c963p-5, |
| 263 | 0x1.3b2ab6a0e1172p-7, 0x1.5d87762dfc134p-10, 0x1.43099b4e829a9p-13, |
| 264 | 0x1.00c080f7699bep-16, 0x1.62c0108a065d8p-20, |
| 265 | }; |
| 266 | |
| 267 | // Overview of powf(x, y) = x^y computation in double precision: |
| 268 | // |
| 269 | // Let x = 2^(e_x) * m_x > 0. Then: |
| 270 | // x^y = 2^( y * log2(x) ) |
| 271 | // = 2^( y * ( e_x + log2(m_x) ) ) |
| 272 | // = 2^( k + f ) |
| 273 | // = 2^k * 2^f, |
| 274 | // where: |
| 275 | // k = round(y * log2(x)), |
| 276 | // f = y * log2(x) - k. |
| 277 | // |
| 278 | // In particular, k is an integer, and |f| <= 0.5. |
| 279 | // For the final result to fit in single precision: |
| 280 | // -150 <= y * log2(x) <= 128, |
| 281 | // and anything outside that range overflows or underflows. |
| 282 | // |
| 283 | // Fast pass: |
| 284 | // 1. Range reduction for log2(m_x) using a 32-entry lookup table: |
| 285 | // dx = r * m_x - 1 in [-2^-6, 2^-5], |
| 286 | // log2(m_x) = log2(1 + dx) - log2(r), |
| 287 | // where -log2(r) is retrieved from LOG2_R_32[idx_x], and log2(1 + dx)/dx is |
| 288 | // approximated by a degree-6 polynomial in double precision with Estrin's |
| 289 | // scheme. |
| 290 | // 2. Exponent reduction: |
| 291 | // z = y * log2(x), |
| 292 | // k = round(z), |
| 293 | // f = z - k in [-0.5, 0.5]. |
| 294 | // 3. 2^f is evaluated using a degree-7 polynomial. |
| 295 | // Absorbing +-ERR into the constant term 1.0 +- ERR of the final FMA allows |
| 296 | // computing the upper and lower bounds for Ziv's rounding test in parallel. |
| 297 | // 4. If upper == lower in single precision, return upper. |
| 298 | // 5. If Ziv's test fails, fall back to check_exact_boundary and powf_accurate |
| 299 | // (DoubleDouble / 128-entry table). |
| 300 | // |
| 301 | // Accurate pass: |
| 302 | // 1. Range reduction using a 128-entry table: -log2(r) = {lo, mid, hi}. |
| 303 | // log2(1 + dx) is evaluated using a degree-11 polynomial in DoubleDouble. |
| 304 | // 2. Decompose: log2(x) = (e_x + hi) + log2_tail, where: |
| 305 | // log2_tail = log2(1 + dx) + mid + lo. |
| 306 | // Because e_x + hi has <= 28 bits of precision and y has 24 bits, |
| 307 | // y * (e_x + hi) is exact in double precision. |
| 308 | // 3. Exponent reduction with 64-entry EXP2_MID1 table and degree-8 polynomial |
| 309 | // in DoubleDouble. |
| 310 | LIBC_INLINE float powf(float x, float y) { |
| 311 | using FloatBits = fputil::FPBits<float>; |
| 312 | using DoubleBits = fputil::FPBits<double>; |
| 313 | using fputil::DoubleDouble; |
| 314 | using namespace common_constants_internal; |
| 315 | |
| 316 | FloatBits xbits(x), ybits(y); |
| 317 | uint32_t x_u = xbits.uintval(); |
| 318 | uint32_t y_u = ybits.uintval(); |
| 319 | uint32_t y_a = ybits.abs().uintval(); |
| 320 | |
| 321 | // Quick filter for special inputs: |
| 322 | // - x: 0, +-1, 2^k, +-Inf, NaN. |
| 323 | // - y: 0, +-1, +-2, +-0.5, +-Inf, NaN. |
| 324 | if (LIBC_UNLIKELY((x_u & 0x001F'FFFF) == 0 || (y_u & 0x007F'FFFF) == 0)) { |
| 325 | if (auto r = powf_internal::check_special_inputs(x, y); |
| 326 | LIBC_UNLIKELY(r.has_value())) |
| 327 | return r.value(); |
| 328 | } |
| 329 | |
| 330 | #if !defined(LIBC_MATH_HAS_SKIP_ACCURATE_PASS) && \ |
| 331 | !defined(LIBC_MATH_HAS_SMALL_TABLES) |
| 332 | float orig_x_abs = xbits.abs().get_val(); |
| 333 | float orig_y = y; |
| 334 | #endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS && !LIBC_MATH_HAS_SMALL_TABLES |
| 335 | |
| 336 | int ex = -FloatBits::EXP_BIAS; |
| 337 | uint64_t sign = 0; |
| 338 | |
| 339 | // Check for exceptional cases: |
| 340 | // - |y| <= 2^-40 or |y| >= 2^40. |
| 341 | // - x < 0, x is subnormal, zero, Inf, or NaN. |
| 342 | if (LIBC_UNLIKELY(y_a <= powf_internal::Y_LOWER_BOUND || |
| 343 | y_a >= powf_internal::Y_UPPER_BOUND || |
| 344 | x_u >= FloatBits::inf().uintval() || |
| 345 | x_u < FloatBits::min_normal().uintval())) { |
| 346 | if (auto r = powf_internal::check_exceptional_cases(x, y, ex, sign); |
| 347 | LIBC_UNLIKELY(r.has_value())) |
| 348 | return r.value(); |
| 349 | } |
| 350 | |
| 351 | Sign out_sign = (sign == 0) ? Sign::POS : Sign::NEG; |
| 352 | |
| 353 | // x^y = 2^( y * log2(x) ) = 2^( y * ( e_x + log2(m_x) ) ) |
| 354 | // Compute log2(x) = e_x + log2(m_x) |
| 355 | x_u = FloatBits(x).uintval(); |
| 356 | |
| 357 | // Extract exponent field of x. |
| 358 | ex += (x_u >> FloatBits::FRACTION_LEN); |
| 359 | double e_x = static_cast<double>(ex); |
| 360 | uint32_t x_mant = x_u & FloatBits::FRACTION_MASK; |
| 361 | // Top 5 bits of mantissa for 32-entry table lookup. |
| 362 | int idx_x = static_cast<int>(x_mant >> (FloatBits::FRACTION_LEN - 5)); |
| 363 | // Embed mantissa into double format: m_x in [1.0, 2.0). |
| 364 | uint64_t m_x_u = |
| 365 | (static_cast<uint64_t>(x_mant) << 29) | 0x3ff0'0000'0000'0000ULL; |
| 366 | double m_x = cpp::bit_cast<double>(from: m_x_u); |
| 367 | |
| 368 | // Range reduction: dx = m_x * r - 1.0 in [-2^-6, 2^-5]. |
| 369 | // Since r has <= 7 bits and m_x has 24 bits, m_x * r - 1.0 is exact in |
| 370 | // double. |
| 371 | double dx = |
| 372 | fputil::multiply_sub(x: m_x, y: powf_internal::R_32_D[idx_x], z: 1.0); // Exact |
| 373 | |
| 374 | // Evaluate P(dx) ~ log2(1 + dx)/dx with Estrin's scheme: |
| 375 | double p = fputil::estrin_polyeval(x: dx, a: POWF_LOG2_COEFFS); |
| 376 | |
| 377 | // log2(x) = e_x - log2(r) + dx * P(dx). |
| 378 | double log2_r_ex = powf_internal::LOG2_R_32[idx_x] + e_x; |
| 379 | double s = fputil::multiply_add(x: dx, y: p, z: log2_r_ex); |
| 380 | |
| 381 | double y_d = static_cast<double>(y); |
| 382 | double z = y_d * s; |
| 383 | |
| 384 | // y * log2(x) = k + f, with k integer and |f| <= 0.5. |
| 385 | double kd = fputil::nearest_integer(x: z); |
| 386 | int64_t k = static_cast<int64_t>(kd); |
| 387 | double f = fputil::multiply_sub(x: y_d, y: s, z: kd); |
| 388 | |
| 389 | // Evaluate P(f) ~ (2^f - 1)/f on [-0.5, 0.5] with Estrin's scheme: |
| 390 | double poly = fputil::estrin_polyeval(x: f, a: POWF_EXP2_COEFFS); |
| 391 | |
| 392 | // Normal range: -125 <= k <= 128. |
| 393 | // (k + 125) as unsigned <= 253 covers all normal outputs. |
| 394 | uint64_t k_u = static_cast<uint64_t>(k + 125); |
| 395 | |
| 396 | if (LIBC_LIKELY(k_u <= 253)) { |
| 397 | // Scale by 2^k. |
| 398 | uint64_t exp2_k_i = |
| 399 | (static_cast<uint64_t>(static_cast<int>(k) + DoubleBits::EXP_BIAS) |
| 400 | << DoubleBits::FRACTION_LEN) | |
| 401 | sign; |
| 402 | double exp2_k = cpp::bit_cast<double>(from: exp2_k_i); |
| 403 | |
| 404 | #ifdef LIBC_MATH_HAS_SKIP_ACCURATE_PASS |
| 405 | double pp = fputil::multiply_add(f, poly, 1.0); |
| 406 | double r_d = pp * exp2_k; |
| 407 | return static_cast<float>(r_d); |
| 408 | #else // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS |
| 409 | #ifdef LIBC_TARGET_CPU_HAS_FMA |
| 410 | // Absorbing +-ERR into 1.0 computes both bounds using two parallel FMAs. |
| 411 | constexpr double ERR = 0x1.5p-39; |
| 412 | double pp_hi = fputil::multiply_add(f, poly, 1.0 + ERR); |
| 413 | double pp_lo = fputil::multiply_add(f, poly, 1.0 - ERR); |
| 414 | double r_d = pp_hi * exp2_k; |
| 415 | #else // !LIBC_TARGET_CPU_HAS_FMA |
| 416 | // Without FMA, compute unbiased pp = f * poly + 1.0 directly. |
| 417 | double pp = fputil::multiply_add(x: f, y: poly, z: 1.0); |
| 418 | double r_d = pp * exp2_k; |
| 419 | #endif // LIBC_TARGET_CPU_HAS_FMA |
| 420 | |
| 421 | float res = static_cast<float>(r_d); |
| 422 | |
| 423 | // Since 2^k is an exact power of 2, testing if the bounds round to the same |
| 424 | // float is equivalent to testing the fully scaled bounds. |
| 425 | #ifdef LIBC_TARGET_CPU_HAS_FMA |
| 426 | float upper = static_cast<float>(pp_hi); |
| 427 | float lower = static_cast<float>(pp_lo); |
| 428 | #else // !LIBC_TARGET_CPU_HAS_FMA |
| 429 | constexpr double ERR = 0x1.5p-39; |
| 430 | float upper = static_cast<float>(pp + ERR); |
| 431 | float lower = static_cast<float>(pp - ERR); |
| 432 | #endif // LIBC_TARGET_CPU_HAS_FMA |
| 433 | |
| 434 | if (LIBC_LIKELY(upper == lower)) |
| 435 | return res; |
| 436 | |
| 437 | // Accurate fallback when Ziv's test fails: |
| 438 | #ifndef LIBC_MATH_HAS_SMALL_TABLES |
| 439 | if (auto r = check_exact_boundary(x: orig_x_abs, y: orig_y, sign, out_sign); |
| 440 | LIBC_UNLIKELY(r.has_value())) |
| 441 | return r.value(); |
| 442 | #endif // !LIBC_MATH_HAS_SMALL_TABLES |
| 443 | |
| 444 | double r_dd = powf_accurate(x, y, e_x, sign); |
| 445 | res = static_cast<float>(r_dd); |
| 446 | if (LIBC_UNLIKELY(FloatBits(res).is_inf())) |
| 447 | return fputil::set_overflow<float>(out_sign); |
| 448 | return res; |
| 449 | #endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS |
| 450 | } |
| 451 | |
| 452 | #ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS |
| 453 | // Exceptional path: k > 128 (overflow), k < -155 (underflow), or denormal. |
| 454 | if (k > 128) |
| 455 | return fputil::set_overflow<float>(out_sign); |
| 456 | |
| 457 | if (k < -155) |
| 458 | return fputil::set_underflow<float>(out_sign); |
| 459 | |
| 460 | // Denormal and underflow path for -155 <= k <= -126: |
| 461 | #ifndef LIBC_MATH_HAS_SMALL_TABLES |
| 462 | if (auto r = check_exact_boundary(x: orig_x_abs, y: orig_y, sign, out_sign); |
| 463 | LIBC_UNLIKELY(r.has_value())) |
| 464 | return r.value(); |
| 465 | #endif // !LIBC_MATH_HAS_SMALL_TABLES |
| 466 | |
| 467 | int64_t exp2_k_i = (static_cast<uint64_t>(k + DoubleBits::EXP_BIAS) |
| 468 | << DoubleBits::FRACTION_LEN) | |
| 469 | sign; |
| 470 | double exp2_k = cpp::bit_cast<double>(from: exp2_k_i); |
| 471 | |
| 472 | // Add +-2^(-126 + (53 - 24)) = +-2^-97 to mimic single precision denormal |
| 473 | // rounding in double precision. |
| 474 | // For -155 <= k <= -126, the least significant bit of the double precision |
| 475 | // sum (r_d + denorm_bias) is aligned at 2^(-97 - 52) = 2^-149, matching the |
| 476 | // least significant bit of single precision denormals 2^(-126 - 23) = 2^-149. |
| 477 | // This keeps the values in the double precision normal range while performing |
| 478 | // rounding according to the current rounding mode. |
| 479 | double denorm_bias = cpp::bit_cast<double>(from: 0x39e0'0000'0000'0000ULL | sign); |
| 480 | |
| 481 | constexpr double ERR = 0x1.5p-39; |
| 482 | |
| 483 | #ifdef LIBC_TARGET_CPU_HAS_FMA |
| 484 | // Absorbing +-ERR into 1.0 computes both bounds using two parallel FMAs. |
| 485 | double pp_hi = fputil::multiply_add(f, poly, 1.0 + ERR); |
| 486 | double pp_lo = fputil::multiply_add(f, poly, 1.0 - ERR); |
| 487 | double u_hi = fputil::multiply_add(pp_hi, exp2_k, denorm_bias); |
| 488 | double u_lo = fputil::multiply_add(pp_lo, exp2_k, denorm_bias); |
| 489 | #else // !LIBC_TARGET_CPU_HAS_FMA |
| 490 | // Without FMA, compute unbiased pp = f * poly + 1.0 directly. |
| 491 | double pp = fputil::multiply_add(x: f, y: poly, z: 1.0); |
| 492 | double r_d = pp * exp2_k; |
| 493 | double err = ERR * r_d; |
| 494 | double u_hi = (r_d + err) + denorm_bias; |
| 495 | double u_lo = (r_d - err) + denorm_bias; |
| 496 | #endif // LIBC_TARGET_CPU_HAS_FMA |
| 497 | |
| 498 | if (LIBC_LIKELY(u_hi == u_lo)) { |
| 499 | if (LIBC_UNLIKELY(u_hi == denorm_bias)) |
| 500 | return fputil::set_underflow<float>(out_sign); |
| 501 | |
| 502 | float res = static_cast<float>(u_hi - denorm_bias); |
| 503 | if (LIBC_UNLIKELY(FloatBits(res).is_normal())) { |
| 504 | #ifdef LIBC_TARGET_CPU_HAS_FMA |
| 505 | double pp = fputil::multiply_add(f, poly, 1.0); |
| 506 | #endif // LIBC_TARGET_CPU_HAS_FMA |
| 507 | if (static_cast<float>(pp) < 1.0f) |
| 508 | fputil::raise_underflow_except_if_required<float>(); |
| 509 | return res; |
| 510 | } |
| 511 | |
| 512 | fputil::set_errno_if_required(ERANGE); |
| 513 | fputil::raise_underflow_except_if_required<float>(); |
| 514 | return res; |
| 515 | } |
| 516 | |
| 517 | // Ziv's test failed for denormal input, fall back to accurate pass. |
| 518 | double r_dd = powf_accurate(x, y, e_x, sign); |
| 519 | float res = static_cast<float>(r_dd); |
| 520 | if (LIBC_UNLIKELY(FloatBits(res).is_normal())) { |
| 521 | if (static_cast<float>(r_dd * 0x1.0p126) < 1.0f) |
| 522 | fputil::raise_underflow_except_if_required<float>(); |
| 523 | return res; |
| 524 | } |
| 525 | fputil::set_errno_if_required(ERANGE); |
| 526 | fputil::raise_underflow_except_if_required<float>(); |
| 527 | return res; |
| 528 | #else // LIBC_MATH_HAS_SKIP_ACCURATE_PASS |
| 529 | if (k > 128) |
| 530 | return fputil::set_overflow<float>(out_sign); |
| 531 | return fputil::set_underflow<float>(out_sign); |
| 532 | #endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS |
| 533 | } |
| 534 | |
| 535 | } // namespace double_eval |
| 536 | } // namespace math |
| 537 | } // namespace LIBC_NAMESPACE_DECL |
| 538 | |
| 539 | #endif // LLVM_LIBC_SRC___SUPPORT_MATH_POWF_DOUBLE_EVAL_H |
| 540 | |