1//===-- Implementation header for expbf16 -----------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#ifndef LLVM_LIBC_SRC___SUPPORT_MATH_EXPBF16_H
10#define LLVM_LIBC_SRC___SUPPORT_MATH_EXPBF16_H
11
12#include "hdr/errno_macros.h"
13#include "hdr/fenv_macros.h"
14#include "src/__support/FPUtil/FEnvImpl.h"
15#include "src/__support/FPUtil/FPBits.h"
16#include "src/__support/FPUtil/bfloat16.h"
17#include "src/__support/FPUtil/cast.h"
18#include "src/__support/FPUtil/except_value_utils.h"
19#include "src/__support/FPUtil/multiply_add.h"
20#include "src/__support/FPUtil/nearest_integer.h"
21#include "src/__support/FPUtil/polyeval.h"
22#include "src/__support/FPUtil/rounding_mode.h"
23#include "src/__support/macros/config.h"
24#include "src/__support/macros/optimization.h"
25
26namespace LIBC_NAMESPACE_DECL {
27
28namespace math {
29
30namespace expbf16_internal {
31
32// Generated by Sollya with the following commands:
33// > display = hexadecimal;
34// > for i from -96 to 88 by 8 do print(i, round(exp(i), SG, RN) @ "f,");
35LIBC_INLINE_VAR constexpr float EXP_HI[24] = {
36 0x1.6a4p-139f, 0x1.07b71p-127f, 0x1.7fd974p-116f, 0x1.175afp-104f,
37 0x1.969d48p-93f, 0x1.27ec46p-81f, 0x1.aebabap-70f, 0x1.397924p-58f,
38 0x1.c8465p-47f, 0x1.4c1078p-35f, 0x1.e355bcp-24f, 0x1.5fc21p-12f,
39 0x1p0f, 0x1.749ea8p11f, 0x1.0f2ebep23f, 0x1.8ab7fcp34f,
40 0x1.1f43fcp46f, 0x1.a220d4p57f, 0x1.304d6ap69f, 0x1.baed16p80f,
41 0x1.425982p92f, 0x1.d531d8p103f, 0x1.55779cp115f, 0x1.f1056ep126f,
42};
43
44// Generated by Sollya with the following commands:
45// > display = hexadecimal;
46// > for i from 0 to 7.75 by 0.25 do print(round(exp(i), SG, RN) @ "f,");
47LIBC_INLINE_VAR constexpr float EXP_MID[32] = {
48 0x1p0f, 0x1.48b5e4p0f, 0x1.a61298p0f, 0x1.0ef9dcp1f,
49 0x1.5bf0a8p1f, 0x1.bec38ep1f, 0x1.1ed3fep2f, 0x1.704b6ap2f,
50 0x1.d8e64cp2f, 0x1.2f9b88p3f, 0x1.85d6fep3f, 0x1.f4907p3f,
51 0x1.415e5cp4f, 0x1.9ca53cp4f, 0x1.08ec72p5f, 0x1.542b2ep5f,
52 0x1.b4c902p5f, 0x1.186bf2p6f, 0x1.68118ap6f, 0x1.ce564ep6f,
53 0x1.28d38ap7f, 0x1.7d21eep7f, 0x1.e96244p7f, 0x1.3a30dp8f,
54 0x1.936dc6p8f, 0x1.0301a4p9f, 0x1.4c9222p9f, 0x1.ab0786p9f,
55 0x1.122886p10f, 0x1.6006b6p10f, 0x1.c402b6p10f, 0x1.223252p11f,
56};
57
58} // namespace expbf16_internal
59
60LIBC_INLINE bfloat16 expbf16(bfloat16 x) {
61 using FPBits = fputil::FPBits<bfloat16>;
62 FPBits x_bits(x);
63
64 uint16_t x_u = x_bits.uintval();
65 uint16_t x_abs = x_u & 0x7fffU;
66
67 // 0 <= |x| <= 2^(-3), or |x| >= 89, or x is NaN
68 if (LIBC_UNLIKELY(x_abs <= 0x3e00U || x_abs >= 0x42b2U)) {
69
70 // exp(NaN) = NaN
71 if (x_bits.is_nan()) {
72 if (x_bits.is_signaling_nan()) {
73 fputil::raise_except_if_required(FE_INVALID);
74 return FPBits::quiet_nan().get_val();
75 }
76
77 return x;
78 }
79
80 // if x >= 89
81 if (x_bits.is_pos() && x_abs >= 0x42b2U) {
82 // exp(inf) = inf
83 if (x_bits.is_inf())
84 return x;
85
86 return fputil::set_overflow<bfloat16>();
87 }
88
89 // x <= -92.5
90 if (x_u >= 0xc2b9U) {
91 // exp(-inf) = +0
92 if (x_bits.is_inf())
93 return FPBits::zero().get_val();
94
95 if (LIBC_UNLIKELY(x_u == 0xc2b9U)) {
96 fputil::set_errno_if_required(ERANGE);
97 fputil::raise_underflow_except_if_required<bfloat16>();
98#ifdef LIBC_MATH_HAS_ASSUME_ROUND_NEAREST_ONLY
99 return FPBits::min_subnormal().get_val();
100#else
101 int rounding = fputil::quick_get_round();
102 if (rounding == FE_TONEAREST || rounding == FE_UPWARD)
103 return FPBits::min_subnormal().get_val();
104 return FPBits::zero().get_val();
105#endif // LIBC_MATH_HAS_ASSUME_ROUND_NEAREST_ONLY
106 }
107
108 return fputil::set_underflow<bfloat16>();
109 }
110
111 // exp(0) = 1
112 if (x_bits.is_zero())
113 return bfloat16(1.0f);
114
115 // 0 < |x| <= 2^(-3)
116 if (x_abs <= 0x3e00U) {
117 // For |x| <= 2^-12, exp(x) rounds like 1 + x in bfloat16.
118 // Use double so tiny products in the float polynomial cannot underflow.
119 if (x_abs <= 0x3980U)
120 return fputil::cast<bfloat16>(
121 x: 1.0 + static_cast<double>(static_cast<float>(x)));
122
123 float xf = static_cast<float>(x);
124 // Degree-3 minimax polynomial generated by Sollya with the following
125 // commands:
126 // > display = hexadecimal;
127 // > P = fpminimax(expm1(x)/x, 2, [|SG...|], [-2^-7, 2^-7]);
128 // > 1 + x * P;
129 // 0x1p0 + x * (0x1p0 + x * (0x1.00004p-1 + x * 0x1.555578p-3))
130 return fputil::cast<bfloat16>(x: fputil::polyeval(
131 x: xf, a0: 0x1p+0f, a: 0x1p+0f, a: 0x1.00004p-1f, a: 0x1.555578p-3f));
132 }
133 }
134
135#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
136 constexpr fputil::ExceptValues<bfloat16, 4> EXPBF16_EXCEPTS = {.values: {
137 // (input, RZ output, RU offset, RD offset, RN offset)
138 // x = 0x40DB (6.84375)
139 // MPFR: RU=0x446B, RD=0x446A, RZ=0x446A, RN=0x446B
140 {.input: 0x40DBU, .rnd_towardzero_result: 0x446AU, .rnd_upward_offset: 1U, .rnd_downward_offset: 0U, .rnd_tonearest_offset: 1U},
141 // x = 0x419D, keep original
142 {.input: 0x419DU, .rnd_towardzero_result: 0x4D9FU, .rnd_upward_offset: 1U, .rnd_downward_offset: 0U, .rnd_tonearest_offset: 0U},
143 // x = 0x41F9 (31.125)
144 // MPFR: RU=0x55F0, RD=0x55EF, RZ=0x55EF, RN=0x55F0
145 {.input: 0x41F9U, .rnd_towardzero_result: 0x55EFU, .rnd_upward_offset: 1U, .rnd_downward_offset: 0U, .rnd_tonearest_offset: 1U},
146 // x = 0xC19F (-19.875)
147 // MPFR: RU=0x3121, RD=0x3120, RZ=0x3120, RN=0x3121
148 {.input: 0xC19FU, .rnd_towardzero_result: 0x3120U, .rnd_upward_offset: 1U, .rnd_downward_offset: 0U, .rnd_tonearest_offset: 1U},
149 }};
150
151 if (auto r = EXPBF16_EXCEPTS.lookup(x_bits: x_u); LIBC_UNLIKELY(r.has_value()))
152 return r.value();
153#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
154
155 // For -93 < x < 89, we do the following range reduction:
156 // x = hi + mid + lo
157 // where,
158 // hi is an integer
159 // mid * 2^2 is an integer
160 // -2^3 <= lo <= 2^3
161 // also, hi + mid = round(4 * x) / x
162 // then,
163 // exp(x) = exp(hi + mid + lo)
164 // = exp(hi) * exp(mid) * exp(lo)
165 // we store 184/8 + 1 = 24 values for looking up exp(hi)
166 // from -96 to 88
167 // we store 8*4 = 32 values for looking up exp(mid) since
168 // mid will always have the bit pattern |bbb.bb| where
169 // b can be either 0 or 1
170
171 float xf = static_cast<float>(x);
172 float kf = fputil::nearest_integer(x: xf * 4.0f);
173 int x_hi_mid = static_cast<int>(kf);
174 int x_hi = x_hi_mid >> 5;
175 int x_mid = x_hi_mid & 0b11111;
176 // lo = x - (hi + mid) = round(x * 4) / (-4) + x
177 float lo = fputil::sub_multiply(x: xf, y: kf, z: 0.25f);
178
179 float exp_hi = expbf16_internal::EXP_HI[x_hi + 12];
180 float exp_mid = expbf16_internal::EXP_MID[x_mid];
181
182 // Degree-3 minimax polynomial generated by Sollya with the following
183 // commands:
184 // > display = hexadecimal;
185 // > P = fpminimax(expm1(x)/x, 2, [|SG...|], [-2^-7, 2^-7]);
186 // > 1 + x * P;
187 // 0x1p0 + x * (0x1p0 + x * (0x1.00004p-1 + x * 0x1.555578p-3))
188 float exp_lo =
189 fputil::polyeval(x: lo, a0: 0x1p+0f, a: 0x1p+0f, a: 0x1.00004p-1f, a: 0x1.555578p-3f);
190
191 return fputil::cast<bfloat16>(x: exp_hi * exp_mid * exp_lo);
192}
193
194} // namespace math
195} // namespace LIBC_NAMESPACE_DECL
196
197#endif // LLVM_LIBC_SRC___SUPPORT_MATH_EXPBF16_H
198