1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// Common header for multiply-add and multiply-sub implementations.
11///
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_LIBC_SRC___SUPPORT_FPUTIL_MULTIPLY_ADD_H
15#define LLVM_LIBC_SRC___SUPPORT_FPUTIL_MULTIPLY_ADD_H
16
17#include "src/__support/CPP/type_traits.h"
18#include "src/__support/common.h"
19#include "src/__support/macros/config.h"
20#include "src/__support/macros/optimization.h"
21#include "src/__support/macros/properties/architectures.h"
22#include "src/__support/macros/properties/compiler.h"
23#include "src/__support/macros/properties/cpu_features.h" // LIBC_TARGET_CPU_HAS_FMA
24
25#if defined(LIBC_TARGET_CPU_HAS_FMA)
26#if !__has_builtin(__builtin_elementwise_fma) && defined(LIBC_COMPILER_IS_MSVC)
27#if defined(LIBC_TARGET_ARCH_IS_X86)
28#include <immintrin.h>
29#elif defined(LIBC_TARGET_ARCH_IS_AARCH64)
30#include <arm64_neon.h>
31#endif
32#endif
33#endif // LIBC_TARGET_CPU_HAS_FMA
34
35namespace LIBC_NAMESPACE_DECL {
36namespace fputil {
37
38namespace internal {
39
40template <typename T> struct HasHardwareFma : cpp::false_type {};
41
42template <typename T> T fma_impl(T, T, T) = delete;
43template <typename T> T fms_impl(T, T, T) = delete;
44template <typename T> T fnma_impl(T, T, T) = delete;
45
46// Hardware FMA helpers use builtins directly instead of including FMA.h to
47// avoid a circular dependency:
48// multiply_add.h -> FMA.h -> generic/FMA.h -> dyadic_float.h.
49#if defined(LIBC_TARGET_CPU_HAS_FMA)
50
51#ifdef LIBC_TARGET_CPU_HAS_FMA_FLOAT
52template <> struct HasHardwareFma<float> : cpp::true_type {};
53
54LIBC_ALWAYS_INLINE float fma_impl(float x, float y, float z) {
55#if __has_builtin(__builtin_elementwise_fma)
56 return __builtin_elementwise_fma(x, y, z);
57#elif defined(LIBC_COMPILER_IS_MSVC) && defined(LIBC_TARGET_ARCH_IS_X86)
58 return _mm_cvtss_f32(
59 _mm_fmadd_ss(_mm_set_ss(x), _mm_set_ss(y), _mm_set_ss(z)));
60#elif defined(LIBC_COMPILER_IS_MSVC) && defined(LIBC_TARGET_ARCH_IS_AARCH64)
61 return vget_lane_f32(vfma_f32(vdup_n_f32(z), vdup_n_f32(x), vdup_n_f32(y)),
62 0);
63#else
64 return __builtin_fmaf(x, y, z);
65#endif
66}
67
68LIBC_ALWAYS_INLINE float fms_impl(float x, float y, float z) {
69#if __has_builtin(__builtin_elementwise_fma)
70 return __builtin_elementwise_fma(x, y, -z);
71#elif defined(LIBC_COMPILER_IS_MSVC) && defined(LIBC_TARGET_ARCH_IS_X86)
72 return _mm_cvtss_f32(
73 _mm_fmsub_ss(_mm_set_ss(x), _mm_set_ss(y), _mm_set_ss(z)));
74#elif defined(LIBC_COMPILER_IS_MSVC) && defined(LIBC_TARGET_ARCH_IS_AARCH64)
75 return vget_lane_f32(vfma_f32(vdup_n_f32(-z), vdup_n_f32(x), vdup_n_f32(y)),
76 0);
77#else
78 return __builtin_fmaf(x, y, -z);
79#endif
80}
81
82LIBC_ALWAYS_INLINE float fnma_impl(float x, float y, float z) {
83#if __has_builtin(__builtin_elementwise_fma)
84 return __builtin_elementwise_fma(-x, y, z);
85#elif defined(LIBC_COMPILER_IS_MSVC) && defined(LIBC_TARGET_ARCH_IS_X86)
86 return _mm_cvtss_f32(
87 _mm_fnmadd_ss(_mm_set_ss(x), _mm_set_ss(y), _mm_set_ss(z)));
88#elif defined(LIBC_COMPILER_IS_MSVC) && defined(LIBC_TARGET_ARCH_IS_AARCH64)
89 return vget_lane_f32(vfms_f32(vdup_n_f32(z), vdup_n_f32(x), vdup_n_f32(y)),
90 0);
91#else
92 return __builtin_fmaf(-x, y, z);
93#endif
94}
95#endif // LIBC_TARGET_CPU_HAS_FMA_FLOAT
96
97#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE
98template <> struct HasHardwareFma<double> : cpp::true_type {};
99
100LIBC_ALWAYS_INLINE double fma_impl(double x, double y, double z) {
101#if __has_builtin(__builtin_elementwise_fma)
102 return __builtin_elementwise_fma(x, y, z);
103#elif defined(LIBC_COMPILER_IS_MSVC) && defined(LIBC_TARGET_ARCH_IS_X86)
104 return _mm_cvtsd_f64(
105 _mm_fmadd_sd(_mm_set_sd(x), _mm_set_sd(y), _mm_set_sd(z)));
106#elif defined(LIBC_COMPILER_IS_MSVC) && defined(LIBC_TARGET_ARCH_IS_AARCH64)
107 return vget_lane_f64(vfma_f64(vdup_n_f64(z), vdup_n_f64(x), vdup_n_f64(y)),
108 0);
109#else
110 return __builtin_fma(x, y, z);
111#endif
112}
113
114LIBC_ALWAYS_INLINE double fms_impl(double x, double y, double z) {
115#if __has_builtin(__builtin_elementwise_fma)
116 return __builtin_elementwise_fma(x, y, -z);
117#elif defined(LIBC_COMPILER_IS_MSVC) && defined(LIBC_TARGET_ARCH_IS_X86)
118 return _mm_cvtsd_f64(
119 _mm_fmsub_sd(_mm_set_sd(x), _mm_set_sd(y), _mm_set_sd(z)));
120#elif defined(LIBC_COMPILER_IS_MSVC) && defined(LIBC_TARGET_ARCH_IS_AARCH64)
121 return vget_lane_f64(vfma_f64(vdup_n_f64(-z), vdup_n_f64(x), vdup_n_f64(y)),
122 0);
123#else
124 return __builtin_fma(x, y, -z);
125#endif
126}
127
128LIBC_ALWAYS_INLINE double fnma_impl(double x, double y, double z) {
129#if __has_builtin(__builtin_elementwise_fma)
130 return __builtin_elementwise_fma(-x, y, z);
131#elif defined(LIBC_COMPILER_IS_MSVC) && defined(LIBC_TARGET_ARCH_IS_X86)
132 return _mm_cvtsd_f64(
133 _mm_fnmadd_sd(_mm_set_sd(x), _mm_set_sd(y), _mm_set_sd(z)));
134#elif defined(LIBC_COMPILER_IS_MSVC) && defined(LIBC_TARGET_ARCH_IS_AARCH64)
135 return vget_lane_f64(vfms_f64(vdup_n_f64(z), vdup_n_f64(x), vdup_n_f64(y)),
136 0);
137#else
138 return __builtin_fma(-x, y, z);
139#endif
140}
141#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE
142
143#endif // LIBC_TARGET_CPU_HAS_FMA
144
145} // namespace internal
146
147// Compute x * y + z, using hardware FMA when available.
148template <typename T>
149LIBC_INLINE constexpr cpp::enable_if_t<(sizeof(T) > sizeof(double)), T>
150multiply_add(const T &x, const T &y, const T &z) {
151 T prod = x * y;
152 return prod + z;
153}
154
155template <typename T>
156LIBC_ALWAYS_INLINE constexpr cpp::enable_if_t<(sizeof(T) <= sizeof(double)), T>
157multiply_add(T x, T y, T z) {
158 if constexpr (internal::HasHardwareFma<T>::value) {
159 if (!cpp::is_constant_evaluated())
160 return internal::fma_impl(x, y, z);
161 }
162 T prod = x * y;
163 return prod + z;
164}
165
166// Compute x * y - z, using hardware FMA when available.
167template <typename T>
168LIBC_INLINE constexpr cpp::enable_if_t<(sizeof(T) > sizeof(double)), T>
169multiply_sub(const T &x, const T &y, const T &z) {
170 T prod = x * y;
171 return prod - z;
172}
173
174template <typename T>
175LIBC_ALWAYS_INLINE constexpr cpp::enable_if_t<(sizeof(T) <= sizeof(double)), T>
176multiply_sub(T x, T y, T z) {
177 if constexpr (internal::HasHardwareFma<T>::value) {
178 if (!cpp::is_constant_evaluated())
179 return internal::fms_impl(x, y, z);
180 }
181 T prod = x * y;
182 return prod - z;
183}
184
185// Compute x - y * z, using hardware FMA when available.
186// Without FMA, when directed rounding modes (FE_UPWARD / FE_DOWNWARD) may be
187// active, evaluate x + ((-y) * z) instead of x - (y * z) for floating-point
188// types so that (-y) * z is rounded in the same direction as the outer sum
189// when y or z is constant (e.g., for x = min_denormal and 0 < z < 1 in
190// FE_UPWARD, RU(x + RU((-x) * z)) = x, whereas RU(x - RU(x * z)) = 0).
191template <typename T>
192LIBC_INLINE constexpr cpp::enable_if_t<(sizeof(T) > sizeof(double)), T>
193sub_multiply(const T &x, const T &y, const T &z) {
194#ifndef LIBC_MATH_HAS_ASSUME_ROUND_NEAREST_ONLY
195 if constexpr (cpp::is_floating_point_v<T>) {
196 T prod = (-y) * z;
197 return x + prod;
198 }
199#endif // !LIBC_MATH_HAS_ASSUME_ROUND_NEAREST_ONLY
200 T prod = y * z;
201 return x - prod;
202}
203
204template <typename T>
205LIBC_ALWAYS_INLINE constexpr cpp::enable_if_t<(sizeof(T) <= sizeof(double)), T>
206sub_multiply(T x, T y, T z) {
207 if constexpr (internal::HasHardwareFma<T>::value) {
208 if (!cpp::is_constant_evaluated())
209 return internal::fnma_impl(y, z, x);
210 }
211#ifndef LIBC_MATH_HAS_ASSUME_ROUND_NEAREST_ONLY
212 if constexpr (cpp::is_floating_point_v<T>) {
213 T prod = (-y) * z;
214 return x + prod;
215 }
216#endif // !LIBC_MATH_HAS_ASSUME_ROUND_NEAREST_ONLY
217 T prod = y * z;
218 return x - prod;
219}
220
221} // namespace fputil
222} // namespace LIBC_NAMESPACE_DECL
223
224#endif // LLVM_LIBC_SRC___SUPPORT_FPUTIL_MULTIPLY_ADD_H
225