1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// Bitwise operations on floating-point numbers.
11///
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_LIBC_SRC___SUPPORT_FPUTIL_FP_BITWISE_H
15#define LLVM_LIBC_SRC___SUPPORT_FPUTIL_FP_BITWISE_H
16
17#include "src/__support/CPP/type_traits.h"
18#include "src/__support/FPUtil/FPBits.h"
19#include "src/__support/macros/attributes.h"
20#include "src/__support/macros/config.h"
21#include "src/__support/macros/properties/architectures.h"
22#include "src/__support/macros/properties/compiler.h"
23#include "src/__support/macros/properties/cpu_features.h"
24
25#ifdef LIBC_TARGET_CPU_HAS_SSE2
26#include <immintrin.h>
27#elif defined(LIBC_TARGET_ARCH_IS_AARCH64) && \
28 defined(LIBC_TARGET_CPU_HAS_ARM_NEON)
29#ifdef LIBC_COMPILER_IS_MSVC
30#include <arm64_neon.h>
31#else
32#include <arm_neon.h>
33#endif
34#endif // LIBC_TARGET_CPU_HAS_SSE2
35
36namespace LIBC_NAMESPACE_DECL {
37namespace fputil {
38
39// Bitwise AND of two floating-point values (a & b), performed directly in
40// FP/SIMD registers when SSE2 or AArch64 NEON is available to avoid GPR
41// round-trips.
42template <typename T>
43LIBC_ALWAYS_INLINE constexpr cpp::enable_if_t<cpp::is_floating_point_v<T>, T>
44fp_and(T a, T b) {
45 if (!cpp::is_constant_evaluated()) {
46#ifdef LIBC_TARGET_CPU_HAS_SSE2
47 if constexpr (cpp::is_same_v<T, float>)
48 return _mm_cvtss_f32(_mm_and_ps(_mm_set_ss(a), _mm_set_ss(b)));
49 if constexpr (cpp::is_same_v<T, double>)
50 return _mm_cvtsd_f64(_mm_and_pd(_mm_set_sd(a), _mm_set_sd(b)));
51#elif defined(LIBC_TARGET_ARCH_IS_AARCH64) && \
52 defined(LIBC_TARGET_CPU_HAS_ARM_NEON)
53 if constexpr (cpp::is_same_v<T, float>)
54 return vget_lane_f32(
55 vreinterpret_f32_u32(vand_u32(vreinterpret_u32_f32(vdup_n_f32(a)),
56 vreinterpret_u32_f32(vdup_n_f32(b)))),
57 0);
58 if constexpr (cpp::is_same_v<T, double>)
59 return vget_lane_f64(
60 vreinterpret_f64_u64(vand_u64(vreinterpret_u64_f64(vdup_n_f64(a)),
61 vreinterpret_u64_f64(vdup_n_f64(b)))),
62 0);
63#endif
64 }
65 using StorageType = typename FPBits<T>::StorageType;
66 return FPBits<T>(static_cast<StorageType>(FPBits<T>(a).uintval() &
67 FPBits<T>(b).uintval()))
68 .get_val();
69}
70
71// Bitwise OR of two floating-point values (a | b), performed directly in
72// FP/SIMD registers when SSE2 or AArch64 NEON is available to avoid GPR
73// round-trips.
74template <typename T>
75LIBC_ALWAYS_INLINE constexpr cpp::enable_if_t<cpp::is_floating_point_v<T>, T>
76fp_or(T a, T b) {
77 if (!cpp::is_constant_evaluated()) {
78#ifdef LIBC_TARGET_CPU_HAS_SSE2
79 if constexpr (cpp::is_same_v<T, float>)
80 return _mm_cvtss_f32(_mm_or_ps(_mm_set_ss(a), _mm_set_ss(b)));
81 if constexpr (cpp::is_same_v<T, double>)
82 return _mm_cvtsd_f64(_mm_or_pd(_mm_set_sd(a), _mm_set_sd(b)));
83#elif defined(LIBC_TARGET_ARCH_IS_AARCH64) && \
84 defined(LIBC_TARGET_CPU_HAS_ARM_NEON)
85 if constexpr (cpp::is_same_v<T, float>)
86 return vget_lane_f32(
87 vreinterpret_f32_u32(vorr_u32(vreinterpret_u32_f32(vdup_n_f32(a)),
88 vreinterpret_u32_f32(vdup_n_f32(b)))),
89 0);
90 if constexpr (cpp::is_same_v<T, double>)
91 return vget_lane_f64(
92 vreinterpret_f64_u64(vorr_u64(vreinterpret_u64_f64(vdup_n_f64(a)),
93 vreinterpret_u64_f64(vdup_n_f64(b)))),
94 0);
95#endif
96 }
97 using StorageType = typename FPBits<T>::StorageType;
98 return FPBits<T>(static_cast<StorageType>(FPBits<T>(a).uintval() |
99 FPBits<T>(b).uintval()))
100 .get_val();
101}
102
103// Bitwise XOR of two floating-point values (a ^ b), performed directly in
104// FP/SIMD registers when SSE2 or AArch64 NEON is available to avoid GPR
105// round-trips.
106template <typename T>
107LIBC_ALWAYS_INLINE constexpr cpp::enable_if_t<cpp::is_floating_point_v<T>, T>
108fp_xor(T a, T b) {
109 if (!cpp::is_constant_evaluated()) {
110#ifdef LIBC_TARGET_CPU_HAS_SSE2
111 if constexpr (cpp::is_same_v<T, float>)
112 return _mm_cvtss_f32(_mm_xor_ps(_mm_set_ss(a), _mm_set_ss(b)));
113 if constexpr (cpp::is_same_v<T, double>)
114 return _mm_cvtsd_f64(_mm_xor_pd(_mm_set_sd(a), _mm_set_sd(b)));
115#elif defined(LIBC_TARGET_ARCH_IS_AARCH64) && \
116 defined(LIBC_TARGET_CPU_HAS_ARM_NEON)
117 if constexpr (cpp::is_same_v<T, float>)
118 return vget_lane_f32(
119 vreinterpret_f32_u32(veor_u32(vreinterpret_u32_f32(vdup_n_f32(a)),
120 vreinterpret_u32_f32(vdup_n_f32(b)))),
121 0);
122 if constexpr (cpp::is_same_v<T, double>)
123 return vget_lane_f64(
124 vreinterpret_f64_u64(veor_u64(vreinterpret_u64_f64(vdup_n_f64(a)),
125 vreinterpret_u64_f64(vdup_n_f64(b)))),
126 0);
127#endif
128 }
129 using StorageType = typename FPBits<T>::StorageType;
130 return FPBits<T>(static_cast<StorageType>(FPBits<T>(a).uintval() ^
131 FPBits<T>(b).uintval()))
132 .get_val();
133}
134
135// Bitwise AND-NOT of two floating-point values (a & ~b), clearing the bits set
136// in b from a. Performed directly in FP/SIMD registers when SSE2 or AArch64
137// NEON is available to avoid GPR round-trips.
138template <typename T>
139LIBC_ALWAYS_INLINE constexpr cpp::enable_if_t<cpp::is_floating_point_v<T>, T>
140fp_andnot(T a, T b) {
141 if (!cpp::is_constant_evaluated()) {
142#ifdef LIBC_TARGET_CPU_HAS_SSE2
143 if constexpr (cpp::is_same_v<T, float>)
144 return _mm_cvtss_f32(_mm_andnot_ps(_mm_set_ss(b), _mm_set_ss(a)));
145 if constexpr (cpp::is_same_v<T, double>)
146 return _mm_cvtsd_f64(_mm_andnot_pd(_mm_set_sd(b), _mm_set_sd(a)));
147#elif defined(LIBC_TARGET_ARCH_IS_AARCH64) && \
148 defined(LIBC_TARGET_CPU_HAS_ARM_NEON)
149 if constexpr (cpp::is_same_v<T, float>)
150 return vget_lane_f32(
151 vreinterpret_f32_u32(vbic_u32(vreinterpret_u32_f32(vdup_n_f32(a)),
152 vreinterpret_u32_f32(vdup_n_f32(b)))),
153 0);
154 if constexpr (cpp::is_same_v<T, double>)
155 return vget_lane_f64(
156 vreinterpret_f64_u64(vbic_u64(vreinterpret_u64_f64(vdup_n_f64(a)),
157 vreinterpret_u64_f64(vdup_n_f64(b)))),
158 0);
159#endif
160 }
161 using StorageType = typename FPBits<T>::StorageType;
162 return FPBits<T>(static_cast<StorageType>(FPBits<T>(a).uintval() &
163 ~FPBits<T>(b).uintval()))
164 .get_val();
165}
166
167// Bitwise NOT of a floating-point value (~a), performed directly in FP/SIMD
168// registers when SSE2 or AArch64 NEON is available to avoid GPR round-trips.
169template <typename T>
170LIBC_ALWAYS_INLINE constexpr cpp::enable_if_t<cpp::is_floating_point_v<T>, T>
171fp_not(T a) {
172 if (!cpp::is_constant_evaluated()) {
173#ifdef LIBC_TARGET_CPU_HAS_SSE2
174 if constexpr (cpp::is_same_v<T, float>)
175 return _mm_cvtss_f32(
176 _mm_xor_ps(_mm_set_ss(a), _mm_castsi128_ps(a: _mm_set1_epi32(i: -1))));
177 if constexpr (cpp::is_same_v<T, double>)
178 return _mm_cvtsd_f64(
179 _mm_xor_pd(_mm_set_sd(a), _mm_castsi128_pd(a: _mm_set1_epi64x(q: -1))));
180#elif defined(LIBC_TARGET_ARCH_IS_AARCH64) && \
181 defined(LIBC_TARGET_CPU_HAS_ARM_NEON)
182 if constexpr (cpp::is_same_v<T, float>)
183 return vget_lane_f32(
184 vreinterpret_f32_u32(vmvn_u32(vreinterpret_u32_f32(vdup_n_f32(a)))),
185 0);
186 if constexpr (cpp::is_same_v<T, double>)
187 return vget_lane_f64(
188 vreinterpret_f64_u32(vmvn_u32(vreinterpret_u32_f64(vdup_n_f64(a)))),
189 0);
190#endif
191 }
192 using StorageType = typename FPBits<T>::StorageType;
193 FPBits<T> res(a);
194 res.set_uintval(static_cast<StorageType>(~res.uintval()));
195 return res.get_val();
196}
197
198// Directly add an integer `hi` to the exponent field of a floating-point value
199// `r`. It is equivalent to scaling `r` by 2^hi when both `r` and the resulting
200// bits are normal.
201// The default implementation bit-casts `r` to its unsigned storage type, shifts
202// `hi` to the exponent field, adds the shifted `hi` to `r`'s bits, and
203// bit-casts the result back to `T`.
204// When SSE2 or AArch64 NEON is available, `float` and `double` use SIMD
205// intrinsics to keep `r` in the FP/SIMD register and perform the shift and
206// integer addition directly in the FP/SIMD registers, avoiding a GPR
207// round-trip.
208template <typename T>
209LIBC_ALWAYS_INLINE constexpr cpp::enable_if_t<cpp::is_floating_point_v<T>, T>
210scale_exp2(T r, int hi) {
211 if (!cpp::is_constant_evaluated()) {
212#ifdef LIBC_TARGET_CPU_HAS_SSE2
213 if constexpr (cpp::is_same_v<T, float>) {
214 __m128i v_exp =
215 _mm_slli_epi32(_mm_cvtsi32_si128(a: hi), FPBits<T>::FRACTION_LEN);
216 __m128i v_r = _mm_castps_si128(_mm_set_ss(r));
217 return _mm_cvtss_f32(a: _mm_castsi128_ps(a: _mm_add_epi32(a: v_exp, b: v_r)));
218 }
219 if constexpr (cpp::is_same_v<T, double>) {
220 __m128i v_exp =
221 _mm_slli_epi64(_mm_cvtsi32_si128(a: hi), FPBits<T>::FRACTION_LEN);
222 __m128i v_r = _mm_castpd_si128(_mm_set_sd(r));
223 return _mm_cvtsd_f64(a: _mm_castsi128_pd(a: _mm_add_epi64(a: v_exp, b: v_r)));
224 }
225#elif defined(LIBC_TARGET_ARCH_IS_AARCH64) && defined(__ARM_NEON)
226 if constexpr (cpp::is_same_v<T, float>) {
227 uint32x2_t v_exp = vshl_n_u32(vdup_n_u32(static_cast<uint32_t>(hi)),
228 FPBits<T>::FRACTION_LEN);
229 uint32x2_t v_r = vreinterpret_u32_f32(vdup_n_f32(r));
230 return vget_lane_f32(vreinterpret_f32_u32(vadd_u32(v_exp, v_r)), 0);
231 }
232 if constexpr (cpp::is_same_v<T, double>) {
233 uint64x1_t v_exp = vshl_n_u64(vdup_n_u64(static_cast<uint64_t>(hi)),
234 FPBits<T>::FRACTION_LEN);
235 uint64x1_t v_r = vreinterpret_u64_f64(vdup_n_f64(r));
236 return vget_lane_f64(vreinterpret_f64_u64(vadd_u64(v_exp, v_r)), 0);
237 }
238#endif
239 }
240 using StorageType = typename FPBits<T>::StorageType;
241 StorageType exp_hi = static_cast<StorageType>(static_cast<StorageType>(hi)
242 << FPBits<T>::FRACTION_LEN);
243 return FPBits<T>(static_cast<StorageType>(exp_hi + FPBits<T>(r).uintval()))
244 .get_val();
245}
246
247} // namespace fputil
248} // namespace LIBC_NAMESPACE_DECL
249
250#endif // LLVM_LIBC_SRC___SUPPORT_FPUTIL_FP_BITWISE_H
251