| 1 | //===----------------------------------------------------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | /// |
| 9 | /// \file |
| 10 | /// Bitwise operations on floating-point numbers. |
| 11 | /// |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
| 14 | #ifndef LLVM_LIBC_SRC___SUPPORT_FPUTIL_FP_BITWISE_H |
| 15 | #define LLVM_LIBC_SRC___SUPPORT_FPUTIL_FP_BITWISE_H |
| 16 | |
| 17 | #include "src/__support/CPP/type_traits.h" |
| 18 | #include "src/__support/FPUtil/FPBits.h" |
| 19 | #include "src/__support/macros/attributes.h" |
| 20 | #include "src/__support/macros/config.h" |
| 21 | #include "src/__support/macros/properties/architectures.h" |
| 22 | #include "src/__support/macros/properties/compiler.h" |
| 23 | #include "src/__support/macros/properties/cpu_features.h" |
| 24 | |
| 25 | #ifdef LIBC_TARGET_CPU_HAS_SSE2 |
| 26 | #include <immintrin.h> |
| 27 | #elif defined(LIBC_TARGET_ARCH_IS_AARCH64) && \ |
| 28 | defined(LIBC_TARGET_CPU_HAS_ARM_NEON) |
| 29 | #ifdef LIBC_COMPILER_IS_MSVC |
| 30 | #include <arm64_neon.h> |
| 31 | #else |
| 32 | #include <arm_neon.h> |
| 33 | #endif |
| 34 | #endif // LIBC_TARGET_CPU_HAS_SSE2 |
| 35 | |
| 36 | namespace LIBC_NAMESPACE_DECL { |
| 37 | namespace fputil { |
| 38 | |
| 39 | // Bitwise AND of two floating-point values (a & b), performed directly in |
| 40 | // FP/SIMD registers when SSE2 or AArch64 NEON is available to avoid GPR |
| 41 | // round-trips. |
| 42 | template <typename T> |
| 43 | LIBC_ALWAYS_INLINE constexpr cpp::enable_if_t<cpp::is_floating_point_v<T>, T> |
| 44 | fp_and(T a, T b) { |
| 45 | if (!cpp::is_constant_evaluated()) { |
| 46 | #ifdef LIBC_TARGET_CPU_HAS_SSE2 |
| 47 | if constexpr (cpp::is_same_v<T, float>) |
| 48 | return _mm_cvtss_f32(_mm_and_ps(_mm_set_ss(a), _mm_set_ss(b))); |
| 49 | if constexpr (cpp::is_same_v<T, double>) |
| 50 | return _mm_cvtsd_f64(_mm_and_pd(_mm_set_sd(a), _mm_set_sd(b))); |
| 51 | #elif defined(LIBC_TARGET_ARCH_IS_AARCH64) && \ |
| 52 | defined(LIBC_TARGET_CPU_HAS_ARM_NEON) |
| 53 | if constexpr (cpp::is_same_v<T, float>) |
| 54 | return vget_lane_f32( |
| 55 | vreinterpret_f32_u32(vand_u32(vreinterpret_u32_f32(vdup_n_f32(a)), |
| 56 | vreinterpret_u32_f32(vdup_n_f32(b)))), |
| 57 | 0); |
| 58 | if constexpr (cpp::is_same_v<T, double>) |
| 59 | return vget_lane_f64( |
| 60 | vreinterpret_f64_u64(vand_u64(vreinterpret_u64_f64(vdup_n_f64(a)), |
| 61 | vreinterpret_u64_f64(vdup_n_f64(b)))), |
| 62 | 0); |
| 63 | #endif |
| 64 | } |
| 65 | using StorageType = typename FPBits<T>::StorageType; |
| 66 | return FPBits<T>(static_cast<StorageType>(FPBits<T>(a).uintval() & |
| 67 | FPBits<T>(b).uintval())) |
| 68 | .get_val(); |
| 69 | } |
| 70 | |
| 71 | // Bitwise OR of two floating-point values (a | b), performed directly in |
| 72 | // FP/SIMD registers when SSE2 or AArch64 NEON is available to avoid GPR |
| 73 | // round-trips. |
| 74 | template <typename T> |
| 75 | LIBC_ALWAYS_INLINE constexpr cpp::enable_if_t<cpp::is_floating_point_v<T>, T> |
| 76 | fp_or(T a, T b) { |
| 77 | if (!cpp::is_constant_evaluated()) { |
| 78 | #ifdef LIBC_TARGET_CPU_HAS_SSE2 |
| 79 | if constexpr (cpp::is_same_v<T, float>) |
| 80 | return _mm_cvtss_f32(_mm_or_ps(_mm_set_ss(a), _mm_set_ss(b))); |
| 81 | if constexpr (cpp::is_same_v<T, double>) |
| 82 | return _mm_cvtsd_f64(_mm_or_pd(_mm_set_sd(a), _mm_set_sd(b))); |
| 83 | #elif defined(LIBC_TARGET_ARCH_IS_AARCH64) && \ |
| 84 | defined(LIBC_TARGET_CPU_HAS_ARM_NEON) |
| 85 | if constexpr (cpp::is_same_v<T, float>) |
| 86 | return vget_lane_f32( |
| 87 | vreinterpret_f32_u32(vorr_u32(vreinterpret_u32_f32(vdup_n_f32(a)), |
| 88 | vreinterpret_u32_f32(vdup_n_f32(b)))), |
| 89 | 0); |
| 90 | if constexpr (cpp::is_same_v<T, double>) |
| 91 | return vget_lane_f64( |
| 92 | vreinterpret_f64_u64(vorr_u64(vreinterpret_u64_f64(vdup_n_f64(a)), |
| 93 | vreinterpret_u64_f64(vdup_n_f64(b)))), |
| 94 | 0); |
| 95 | #endif |
| 96 | } |
| 97 | using StorageType = typename FPBits<T>::StorageType; |
| 98 | return FPBits<T>(static_cast<StorageType>(FPBits<T>(a).uintval() | |
| 99 | FPBits<T>(b).uintval())) |
| 100 | .get_val(); |
| 101 | } |
| 102 | |
| 103 | // Bitwise XOR of two floating-point values (a ^ b), performed directly in |
| 104 | // FP/SIMD registers when SSE2 or AArch64 NEON is available to avoid GPR |
| 105 | // round-trips. |
| 106 | template <typename T> |
| 107 | LIBC_ALWAYS_INLINE constexpr cpp::enable_if_t<cpp::is_floating_point_v<T>, T> |
| 108 | fp_xor(T a, T b) { |
| 109 | if (!cpp::is_constant_evaluated()) { |
| 110 | #ifdef LIBC_TARGET_CPU_HAS_SSE2 |
| 111 | if constexpr (cpp::is_same_v<T, float>) |
| 112 | return _mm_cvtss_f32(_mm_xor_ps(_mm_set_ss(a), _mm_set_ss(b))); |
| 113 | if constexpr (cpp::is_same_v<T, double>) |
| 114 | return _mm_cvtsd_f64(_mm_xor_pd(_mm_set_sd(a), _mm_set_sd(b))); |
| 115 | #elif defined(LIBC_TARGET_ARCH_IS_AARCH64) && \ |
| 116 | defined(LIBC_TARGET_CPU_HAS_ARM_NEON) |
| 117 | if constexpr (cpp::is_same_v<T, float>) |
| 118 | return vget_lane_f32( |
| 119 | vreinterpret_f32_u32(veor_u32(vreinterpret_u32_f32(vdup_n_f32(a)), |
| 120 | vreinterpret_u32_f32(vdup_n_f32(b)))), |
| 121 | 0); |
| 122 | if constexpr (cpp::is_same_v<T, double>) |
| 123 | return vget_lane_f64( |
| 124 | vreinterpret_f64_u64(veor_u64(vreinterpret_u64_f64(vdup_n_f64(a)), |
| 125 | vreinterpret_u64_f64(vdup_n_f64(b)))), |
| 126 | 0); |
| 127 | #endif |
| 128 | } |
| 129 | using StorageType = typename FPBits<T>::StorageType; |
| 130 | return FPBits<T>(static_cast<StorageType>(FPBits<T>(a).uintval() ^ |
| 131 | FPBits<T>(b).uintval())) |
| 132 | .get_val(); |
| 133 | } |
| 134 | |
| 135 | // Bitwise AND-NOT of two floating-point values (a & ~b), clearing the bits set |
| 136 | // in b from a. Performed directly in FP/SIMD registers when SSE2 or AArch64 |
| 137 | // NEON is available to avoid GPR round-trips. |
| 138 | template <typename T> |
| 139 | LIBC_ALWAYS_INLINE constexpr cpp::enable_if_t<cpp::is_floating_point_v<T>, T> |
| 140 | fp_andnot(T a, T b) { |
| 141 | if (!cpp::is_constant_evaluated()) { |
| 142 | #ifdef LIBC_TARGET_CPU_HAS_SSE2 |
| 143 | if constexpr (cpp::is_same_v<T, float>) |
| 144 | return _mm_cvtss_f32(_mm_andnot_ps(_mm_set_ss(b), _mm_set_ss(a))); |
| 145 | if constexpr (cpp::is_same_v<T, double>) |
| 146 | return _mm_cvtsd_f64(_mm_andnot_pd(_mm_set_sd(b), _mm_set_sd(a))); |
| 147 | #elif defined(LIBC_TARGET_ARCH_IS_AARCH64) && \ |
| 148 | defined(LIBC_TARGET_CPU_HAS_ARM_NEON) |
| 149 | if constexpr (cpp::is_same_v<T, float>) |
| 150 | return vget_lane_f32( |
| 151 | vreinterpret_f32_u32(vbic_u32(vreinterpret_u32_f32(vdup_n_f32(a)), |
| 152 | vreinterpret_u32_f32(vdup_n_f32(b)))), |
| 153 | 0); |
| 154 | if constexpr (cpp::is_same_v<T, double>) |
| 155 | return vget_lane_f64( |
| 156 | vreinterpret_f64_u64(vbic_u64(vreinterpret_u64_f64(vdup_n_f64(a)), |
| 157 | vreinterpret_u64_f64(vdup_n_f64(b)))), |
| 158 | 0); |
| 159 | #endif |
| 160 | } |
| 161 | using StorageType = typename FPBits<T>::StorageType; |
| 162 | return FPBits<T>(static_cast<StorageType>(FPBits<T>(a).uintval() & |
| 163 | ~FPBits<T>(b).uintval())) |
| 164 | .get_val(); |
| 165 | } |
| 166 | |
| 167 | // Bitwise NOT of a floating-point value (~a), performed directly in FP/SIMD |
| 168 | // registers when SSE2 or AArch64 NEON is available to avoid GPR round-trips. |
| 169 | template <typename T> |
| 170 | LIBC_ALWAYS_INLINE constexpr cpp::enable_if_t<cpp::is_floating_point_v<T>, T> |
| 171 | fp_not(T a) { |
| 172 | if (!cpp::is_constant_evaluated()) { |
| 173 | #ifdef LIBC_TARGET_CPU_HAS_SSE2 |
| 174 | if constexpr (cpp::is_same_v<T, float>) |
| 175 | return _mm_cvtss_f32( |
| 176 | _mm_xor_ps(_mm_set_ss(a), _mm_castsi128_ps(a: _mm_set1_epi32(i: -1)))); |
| 177 | if constexpr (cpp::is_same_v<T, double>) |
| 178 | return _mm_cvtsd_f64( |
| 179 | _mm_xor_pd(_mm_set_sd(a), _mm_castsi128_pd(a: _mm_set1_epi64x(q: -1)))); |
| 180 | #elif defined(LIBC_TARGET_ARCH_IS_AARCH64) && \ |
| 181 | defined(LIBC_TARGET_CPU_HAS_ARM_NEON) |
| 182 | if constexpr (cpp::is_same_v<T, float>) |
| 183 | return vget_lane_f32( |
| 184 | vreinterpret_f32_u32(vmvn_u32(vreinterpret_u32_f32(vdup_n_f32(a)))), |
| 185 | 0); |
| 186 | if constexpr (cpp::is_same_v<T, double>) |
| 187 | return vget_lane_f64( |
| 188 | vreinterpret_f64_u32(vmvn_u32(vreinterpret_u32_f64(vdup_n_f64(a)))), |
| 189 | 0); |
| 190 | #endif |
| 191 | } |
| 192 | using StorageType = typename FPBits<T>::StorageType; |
| 193 | FPBits<T> res(a); |
| 194 | res.set_uintval(static_cast<StorageType>(~res.uintval())); |
| 195 | return res.get_val(); |
| 196 | } |
| 197 | |
| 198 | // Directly add an integer `hi` to the exponent field of a floating-point value |
| 199 | // `r`. It is equivalent to scaling `r` by 2^hi when both `r` and the resulting |
| 200 | // bits are normal. |
| 201 | // The default implementation bit-casts `r` to its unsigned storage type, shifts |
| 202 | // `hi` to the exponent field, adds the shifted `hi` to `r`'s bits, and |
| 203 | // bit-casts the result back to `T`. |
| 204 | // When SSE2 or AArch64 NEON is available, `float` and `double` use SIMD |
| 205 | // intrinsics to keep `r` in the FP/SIMD register and perform the shift and |
| 206 | // integer addition directly in the FP/SIMD registers, avoiding a GPR |
| 207 | // round-trip. |
| 208 | template <typename T> |
| 209 | LIBC_ALWAYS_INLINE constexpr cpp::enable_if_t<cpp::is_floating_point_v<T>, T> |
| 210 | scale_exp2(T r, int hi) { |
| 211 | if (!cpp::is_constant_evaluated()) { |
| 212 | #ifdef LIBC_TARGET_CPU_HAS_SSE2 |
| 213 | if constexpr (cpp::is_same_v<T, float>) { |
| 214 | __m128i v_exp = |
| 215 | _mm_slli_epi32(_mm_cvtsi32_si128(a: hi), FPBits<T>::FRACTION_LEN); |
| 216 | __m128i v_r = _mm_castps_si128(_mm_set_ss(r)); |
| 217 | return _mm_cvtss_f32(a: _mm_castsi128_ps(a: _mm_add_epi32(a: v_exp, b: v_r))); |
| 218 | } |
| 219 | if constexpr (cpp::is_same_v<T, double>) { |
| 220 | __m128i v_exp = |
| 221 | _mm_slli_epi64(_mm_cvtsi32_si128(a: hi), FPBits<T>::FRACTION_LEN); |
| 222 | __m128i v_r = _mm_castpd_si128(_mm_set_sd(r)); |
| 223 | return _mm_cvtsd_f64(a: _mm_castsi128_pd(a: _mm_add_epi64(a: v_exp, b: v_r))); |
| 224 | } |
| 225 | #elif defined(LIBC_TARGET_ARCH_IS_AARCH64) && defined(__ARM_NEON) |
| 226 | if constexpr (cpp::is_same_v<T, float>) { |
| 227 | uint32x2_t v_exp = vshl_n_u32(vdup_n_u32(static_cast<uint32_t>(hi)), |
| 228 | FPBits<T>::FRACTION_LEN); |
| 229 | uint32x2_t v_r = vreinterpret_u32_f32(vdup_n_f32(r)); |
| 230 | return vget_lane_f32(vreinterpret_f32_u32(vadd_u32(v_exp, v_r)), 0); |
| 231 | } |
| 232 | if constexpr (cpp::is_same_v<T, double>) { |
| 233 | uint64x1_t v_exp = vshl_n_u64(vdup_n_u64(static_cast<uint64_t>(hi)), |
| 234 | FPBits<T>::FRACTION_LEN); |
| 235 | uint64x1_t v_r = vreinterpret_u64_f64(vdup_n_f64(r)); |
| 236 | return vget_lane_f64(vreinterpret_f64_u64(vadd_u64(v_exp, v_r)), 0); |
| 237 | } |
| 238 | #endif |
| 239 | } |
| 240 | using StorageType = typename FPBits<T>::StorageType; |
| 241 | StorageType exp_hi = static_cast<StorageType>(static_cast<StorageType>(hi) |
| 242 | << FPBits<T>::FRACTION_LEN); |
| 243 | return FPBits<T>(static_cast<StorageType>(exp_hi + FPBits<T>(r).uintval())) |
| 244 | .get_val(); |
| 245 | } |
| 246 | |
| 247 | } // namespace fputil |
| 248 | } // namespace LIBC_NAMESPACE_DECL |
| 249 | |
| 250 | #endif // LLVM_LIBC_SRC___SUPPORT_FPUTIL_FP_BITWISE_H |
| 251 | |