1//===-- Common header for FMA implementations -------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#ifndef LLVM_LIBC_SRC___SUPPORT_FPUTIL_FMA_H
10#define LLVM_LIBC_SRC___SUPPORT_FPUTIL_FMA_H
11
12#include "src/__support/CPP/type_traits.h"
13#include "src/__support/FPUtil/generic/FMA.h"
14#include "src/__support/macros/config.h"
15#include "src/__support/macros/properties/architectures.h"
16#include "src/__support/macros/properties/compiler.h"
17#include "src/__support/macros/properties/cpu_features.h" // LIBC_TARGET_CPU_HAS_FMA
18
19#if defined(LIBC_TARGET_CPU_HAS_FMA) && \
20 !__has_builtin(__builtin_elementwise_fma) && \
21 defined(LIBC_COMPILER_IS_MSVC)
22#if defined(LIBC_TARGET_ARCH_IS_X86)
23#include <immintrin.h>
24#elif defined(LIBC_TARGET_ARCH_IS_AARCH64)
25#include <arm64_neon.h>
26#endif
27#endif
28
29namespace LIBC_NAMESPACE_DECL {
30namespace fputil {
31
32template <typename OutType, typename InType>
33LIBC_INLINE constexpr OutType fma(InType x, InType y, InType z) {
34 return generic::fma<OutType>(x, y, z);
35}
36
37#ifdef LIBC_TARGET_CPU_HAS_FMA
38
39#ifdef LIBC_TARGET_CPU_HAS_FMA_FLOAT
40template <> LIBC_INLINE float fma(float x, float y, float z) {
41#if __has_builtin(__builtin_elementwise_fma)
42 return __builtin_elementwise_fma(x, y, z);
43#elif defined(LIBC_COMPILER_IS_MSVC) && defined(LIBC_TARGET_ARCH_IS_X86)
44 return _mm_cvtss_f32(
45 _mm_fmadd_ss(_mm_set_ss(x), _mm_set_ss(y), _mm_set_ss(z)));
46#elif defined(LIBC_COMPILER_IS_MSVC) && defined(LIBC_TARGET_ARCH_IS_AARCH64)
47 return vget_lane_f32(vfma_f32(vdup_n_f32(z), vdup_n_f32(x), vdup_n_f32(y)),
48 0);
49#else
50 return __builtin_fmaf(x, y, z);
51#endif
52}
53#endif // LIBC_TARGET_CPU_HAS_FMA_FLOAT
54
55#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE
56template <> LIBC_INLINE double fma(double x, double y, double z) {
57#if __has_builtin(__builtin_elementwise_fma)
58 return __builtin_elementwise_fma(x, y, z);
59#elif defined(LIBC_COMPILER_IS_MSVC) && defined(LIBC_TARGET_ARCH_IS_X86)
60 return _mm_cvtsd_f64(
61 _mm_fmadd_sd(_mm_set_sd(x), _mm_set_sd(y), _mm_set_sd(z)));
62#elif defined(LIBC_COMPILER_IS_MSVC) && defined(LIBC_TARGET_ARCH_IS_AARCH64)
63 return vget_lane_f64(vfma_f64(vdup_n_f64(z), vdup_n_f64(x), vdup_n_f64(y)),
64 0);
65#else
66 return __builtin_fma(x, y, z);
67#endif
68}
69#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE
70#endif // LIBC_TARGET_CPU_HAS_FMA
71
72} // namespace fputil
73} // namespace LIBC_NAMESPACE_DECL
74
75#endif // LLVM_LIBC_SRC___SUPPORT_FPUTIL_FMA_H
76