| 1 | //===-- Portable optimization macros ----------------------------*- C++ -*-===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // This header file defines portable macros for performance optimization. |
| 9 | |
| 10 | #ifndef LLVM_LIBC_SRC___SUPPORT_MACROS_OPTIMIZATION_H |
| 11 | #define LLVM_LIBC_SRC___SUPPORT_MACROS_OPTIMIZATION_H |
| 12 | |
| 13 | #include "src/__support/macros/attributes.h" // LIBC_INLINE |
| 14 | #include "src/__support/macros/config.h" |
| 15 | #include "src/__support/macros/properties/architectures.h" |
| 16 | #include "src/__support/macros/properties/compiler.h" // LIBC_COMPILER_IS_CLANG |
| 17 | #include "src/__support/macros/properties/cpu_features.h" |
| 18 | |
| 19 | // We use a template to implement likely/unlikely to make sure that we don't |
| 20 | // accidentally pass an integer. |
| 21 | // Note that `__builtin_expect` requires the second argument to be a constant |
| 22 | // expression. |
| 23 | namespace LIBC_NAMESPACE_DECL { |
| 24 | namespace details { |
| 25 | template <typename T> |
| 26 | LIBC_ALWAYS_INLINE constexpr bool expects_bool_condition(T value) = delete; |
| 27 | |
| 28 | template <> |
| 29 | LIBC_ALWAYS_INLINE constexpr bool expects_bool_condition<bool>(bool value) { |
| 30 | return value; |
| 31 | } |
| 32 | |
| 33 | template <typename T> void do_not_optimize_impl(T &) = delete; |
| 34 | |
| 35 | // Zero-cost SSA value barrier for float and double: keeps `val` in its native |
| 36 | // register class without spilling to memory or moving across GPR/FPR banks, |
| 37 | // while preventing the optimizer from looking through or duplicating `val`. |
| 38 | // Kept in non-constexpr helpers because C++17 rejects inline `asm` statements |
| 39 | // directly inside `constexpr` functions (`-Wc++20-extensions`). Skipped on |
| 40 | // `cl.exe` (`LIBC_COMPILER_IS_MSVC`, which excludes `clang-cl`) because MSVC |
| 41 | // does not support GNU extended `asm` register constraints. |
| 42 | LIBC_ALWAYS_INLINE void do_not_optimize_impl(LIBC_UNUSED float &val) { |
| 43 | #if !defined(LIBC_COMPILER_IS_MSVC) |
| 44 | #if defined(LIBC_TARGET_CPU_HAS_ARM_FPU_FLOAT) |
| 45 | asm("" : "+w" (val)); |
| 46 | #elif defined(LIBC_TARGET_ARCH_IS_X86) && defined(LIBC_TARGET_CPU_HAS_SSE2) |
| 47 | asm("" : "+x" (val)); |
| 48 | #elif defined(LIBC_TARGET_ARCH_IS_X86) |
| 49 | asm("" : "+t" (val)); |
| 50 | #elif defined(LIBC_TARGET_CPU_HAS_RISCV_FPU_FLOAT) |
| 51 | asm("" : "+f" (val)); |
| 52 | #elif defined(LIBC_TARGET_ARCH_IS_AMDGPU) |
| 53 | asm("" : "+v" (val)); |
| 54 | #elif defined(LIBC_TARGET_ARCH_IS_NVPTX) |
| 55 | asm("" : "+f" (val)); |
| 56 | #elif defined(LIBC_TARGET_ARCH_IS_ARM) || \ |
| 57 | defined(LIBC_TARGET_ARCH_IS_ANY_RISCV) || \ |
| 58 | defined(LIBC_TARGET_ARCH_IS_WASM) || defined(LIBC_TARGET_ARCH_IS_HEXAGON) |
| 59 | asm("" : "+r" (val)); |
| 60 | #endif |
| 61 | #endif // !LIBC_COMPILER_IS_MSVC |
| 62 | } |
| 63 | |
| 64 | LIBC_ALWAYS_INLINE void do_not_optimize_impl(LIBC_UNUSED double &val) { |
| 65 | #if !defined(LIBC_COMPILER_IS_MSVC) |
| 66 | #if defined(LIBC_TARGET_CPU_HAS_ARM_FPU_DOUBLE) |
| 67 | asm("" : "+w" (val)); |
| 68 | #elif defined(LIBC_TARGET_ARCH_IS_X86) && defined(LIBC_TARGET_CPU_HAS_SSE2) |
| 69 | asm("" : "+x" (val)); |
| 70 | #elif defined(LIBC_TARGET_ARCH_IS_X86) |
| 71 | asm("" : "+t" (val)); |
| 72 | #elif defined(LIBC_TARGET_CPU_HAS_RISCV_FPU_DOUBLE) |
| 73 | asm("" : "+f" (val)); |
| 74 | #elif defined(LIBC_TARGET_ARCH_IS_AMDGPU) |
| 75 | asm("" : "+v" (val)); |
| 76 | #elif defined(LIBC_TARGET_ARCH_IS_NVPTX) |
| 77 | asm("" : "+d" (val)); |
| 78 | #elif defined(LIBC_TARGET_ARCH_IS_RISCV64) || defined(LIBC_TARGET_ARCH_IS_WASM) |
| 79 | asm("" : "+r" (val)); |
| 80 | #endif |
| 81 | #endif // !LIBC_COMPILER_IS_MSVC |
| 82 | } |
| 83 | |
| 84 | template <typename T> |
| 85 | LIBC_ALWAYS_INLINE LIBC_CONSTEXPR_DEFAULT void do_not_optimize(T &val) { |
| 86 | #if LIBC_HAS_BUILTIN_IS_CONSTANT_EVALUATED |
| 87 | if (!__builtin_is_constant_evaluated()) |
| 88 | #endif |
| 89 | do_not_optimize_impl(val); |
| 90 | } |
| 91 | |
| 92 | } // namespace details |
| 93 | } // namespace LIBC_NAMESPACE_DECL |
| 94 | |
| 95 | #define LIBC_DO_NOT_OPTIMIZE(var) LIBC_NAMESPACE::details::do_not_optimize(var) |
| 96 | |
| 97 | #define LIBC_LIKELY(x) \ |
| 98 | (__builtin_expect(LIBC_NAMESPACE::details::expects_bool_condition((x)), true)) |
| 99 | #define LIBC_UNLIKELY(x) \ |
| 100 | (__builtin_expect(LIBC_NAMESPACE::details::expects_bool_condition((x)), \ |
| 101 | false)) |
| 102 | |
| 103 | #if defined(LIBC_COMPILER_IS_CLANG) |
| 104 | #define LIBC_LOOP_NOUNROLL _Pragma("nounroll") |
| 105 | #define LIBC_LOOP_UNROLL _Pragma("unroll") |
| 106 | #elif defined(LIBC_COMPILER_IS_GCC) |
| 107 | #if LIBC_COMPILER_GCC_VER >= 800 |
| 108 | #define LIBC_LOOP_NOUNROLL _Pragma("GCC unroll 0") |
| 109 | #define LIBC_LOOP_UNROLL _Pragma("GCC unroll 2048") |
| 110 | #else |
| 111 | #define LIBC_LOOP_NOUNROLL |
| 112 | #define LIBC_LOOP_UNROLL |
| 113 | #endif |
| 114 | #elif defined(LIBC_COMPILER_IS_MSVC) |
| 115 | #define LIBC_LOOP_NOUNROLL |
| 116 | #define LIBC_LOOP_UNROLL |
| 117 | #else |
| 118 | #error "Unhandled compiler" |
| 119 | #endif |
| 120 | |
| 121 | // Defining optimization options for math functions. |
| 122 | // TODO: Exporting this to public generated headers? |
| 123 | #define LIBC_MATH_SKIP_ACCURATE_PASS 0x01 |
| 124 | #define LIBC_MATH_SMALL_TABLES 0x02 |
| 125 | #define LIBC_MATH_NO_ERRNO 0x04 |
| 126 | #define LIBC_MATH_NO_EXCEPT 0x08 |
| 127 | #define LIBC_MATH_INTERMEDIATE_COMP_IN_FLOAT 0x10 |
| 128 | #define LIBC_MATH_ASSUME_ROUND_NEAREST_ONLY 0x20 |
| 129 | #define LIBC_MATH_FAST \ |
| 130 | (LIBC_MATH_SKIP_ACCURATE_PASS | LIBC_MATH_SMALL_TABLES | \ |
| 131 | LIBC_MATH_NO_ERRNO | LIBC_MATH_NO_EXCEPT | \ |
| 132 | LIBC_MATH_ASSUME_ROUND_NEAREST_ONLY) |
| 133 | |
| 134 | #ifndef LIBC_MATH |
| 135 | #define LIBC_MATH 0 |
| 136 | #endif // LIBC_MATH |
| 137 | |
| 138 | #if ((LIBC_MATH) & LIBC_MATH_SKIP_ACCURATE_PASS) |
| 139 | #define LIBC_MATH_HAS_SKIP_ACCURATE_PASS |
| 140 | #endif |
| 141 | |
| 142 | #if ((LIBC_MATH) & LIBC_MATH_SMALL_TABLES) |
| 143 | #define LIBC_MATH_HAS_SMALL_TABLES |
| 144 | #define LIBC_MATH_SMALL_TABLES_INLINE LIBC_NOINLINE LIBC_INLINE |
| 145 | #else |
| 146 | #define LIBC_MATH_SMALL_TABLES_INLINE LIBC_ALWAYS_INLINE |
| 147 | #endif |
| 148 | |
| 149 | #if ((LIBC_MATH) & LIBC_MATH_INTERMEDIATE_COMP_IN_FLOAT) |
| 150 | #define LIBC_MATH_HAS_INTERMEDIATE_COMP_IN_FLOAT |
| 151 | #endif |
| 152 | |
| 153 | #if ((LIBC_MATH) & LIBC_MATH_NO_ERRNO) |
| 154 | #define LIBC_MATH_HAS_NO_ERRNO |
| 155 | #endif |
| 156 | |
| 157 | #if ((LIBC_MATH) & LIBC_MATH_NO_EXCEPT) |
| 158 | #define LIBC_MATH_HAS_NO_EXCEPT |
| 159 | #endif |
| 160 | |
| 161 | #if ((LIBC_MATH) & LIBC_MATH_ASSUME_ROUND_NEAREST_ONLY) |
| 162 | #define LIBC_MATH_HAS_ASSUME_ROUND_NEAREST_ONLY |
| 163 | #define LIBC_DO_NOT_OPTIMIZE_ROUNDING(var) ((void)(var)) |
| 164 | #else |
| 165 | #define LIBC_DO_NOT_OPTIMIZE_ROUNDING(var) LIBC_DO_NOT_OPTIMIZE(var) |
| 166 | #endif |
| 167 | |
| 168 | // Helper macros to compare math implementation namespace tokens in #if |
| 169 | // directives (e.g. when LIBC_MATH_<FUNC>_IMPL is overridden via -D). |
| 170 | #define LIBC_MATH_IMPL_ID_double_eval 1 |
| 171 | #define LIBC_MATH_IMPL_ID_float_eval 2 |
| 172 | #define LIBC_MATH_IMPL_ID_float_eval_cr_rn 3 |
| 173 | #define LIBC_MATH_IMPL_ID_integer_eval 4 |
| 174 | #define LIBC_MATH_IMPL_CONCAT_INNER(a, b) a##b |
| 175 | #define LIBC_MATH_IMPL_CONCAT(a, b) LIBC_MATH_IMPL_CONCAT_INNER(a, b) |
| 176 | #define LIBC_MATH_IMPL_IS(impl, target) \ |
| 177 | (LIBC_MATH_IMPL_CONCAT(LIBC_MATH_IMPL_ID_, impl) == \ |
| 178 | LIBC_MATH_IMPL_CONCAT(LIBC_MATH_IMPL_ID_, target)) |
| 179 | |
| 180 | #endif // LLVM_LIBC_SRC___SUPPORT_MACROS_OPTIMIZATION_H |
| 181 | |