1//===-- Portable optimization macros ----------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8// This header file defines portable macros for performance optimization.
9
10#ifndef LLVM_LIBC_SRC___SUPPORT_MACROS_OPTIMIZATION_H
11#define LLVM_LIBC_SRC___SUPPORT_MACROS_OPTIMIZATION_H
12
13#include "src/__support/macros/attributes.h" // LIBC_INLINE
14#include "src/__support/macros/config.h"
15#include "src/__support/macros/properties/architectures.h"
16#include "src/__support/macros/properties/compiler.h" // LIBC_COMPILER_IS_CLANG
17#include "src/__support/macros/properties/cpu_features.h"
18
19// We use a template to implement likely/unlikely to make sure that we don't
20// accidentally pass an integer.
21// Note that `__builtin_expect` requires the second argument to be a constant
22// expression.
23namespace LIBC_NAMESPACE_DECL {
24namespace details {
25template <typename T>
26LIBC_ALWAYS_INLINE constexpr bool expects_bool_condition(T value) = delete;
27
28template <>
29LIBC_ALWAYS_INLINE constexpr bool expects_bool_condition<bool>(bool value) {
30 return value;
31}
32
33template <typename T> void do_not_optimize_impl(T &) = delete;
34
35// Zero-cost SSA value barrier for float and double: keeps `val` in its native
36// register class without spilling to memory or moving across GPR/FPR banks,
37// while preventing the optimizer from looking through or duplicating `val`.
38// Kept in non-constexpr helpers because C++17 rejects inline `asm` statements
39// directly inside `constexpr` functions (`-Wc++20-extensions`). Skipped on
40// `cl.exe` (`LIBC_COMPILER_IS_MSVC`, which excludes `clang-cl`) because MSVC
41// does not support GNU extended `asm` register constraints.
42LIBC_ALWAYS_INLINE void do_not_optimize_impl(LIBC_UNUSED float &val) {
43#if !defined(LIBC_COMPILER_IS_MSVC)
44#if defined(LIBC_TARGET_CPU_HAS_ARM_FPU_FLOAT)
45 asm("" : "+w"(val));
46#elif defined(LIBC_TARGET_ARCH_IS_X86) && defined(LIBC_TARGET_CPU_HAS_SSE2)
47 asm("" : "+x"(val));
48#elif defined(LIBC_TARGET_ARCH_IS_X86)
49 asm("" : "+t"(val));
50#elif defined(LIBC_TARGET_CPU_HAS_RISCV_FPU_FLOAT)
51 asm("" : "+f"(val));
52#elif defined(LIBC_TARGET_ARCH_IS_AMDGPU)
53 asm("" : "+v"(val));
54#elif defined(LIBC_TARGET_ARCH_IS_NVPTX)
55 asm("" : "+f"(val));
56#elif defined(LIBC_TARGET_ARCH_IS_ARM) || \
57 defined(LIBC_TARGET_ARCH_IS_ANY_RISCV) || \
58 defined(LIBC_TARGET_ARCH_IS_WASM) || defined(LIBC_TARGET_ARCH_IS_HEXAGON)
59 asm("" : "+r"(val));
60#endif
61#endif // !LIBC_COMPILER_IS_MSVC
62}
63
64LIBC_ALWAYS_INLINE void do_not_optimize_impl(LIBC_UNUSED double &val) {
65#if !defined(LIBC_COMPILER_IS_MSVC)
66#if defined(LIBC_TARGET_CPU_HAS_ARM_FPU_DOUBLE)
67 asm("" : "+w"(val));
68#elif defined(LIBC_TARGET_ARCH_IS_X86) && defined(LIBC_TARGET_CPU_HAS_SSE2)
69 asm("" : "+x"(val));
70#elif defined(LIBC_TARGET_ARCH_IS_X86)
71 asm("" : "+t"(val));
72#elif defined(LIBC_TARGET_CPU_HAS_RISCV_FPU_DOUBLE)
73 asm("" : "+f"(val));
74#elif defined(LIBC_TARGET_ARCH_IS_AMDGPU)
75 asm("" : "+v"(val));
76#elif defined(LIBC_TARGET_ARCH_IS_NVPTX)
77 asm("" : "+d"(val));
78#elif defined(LIBC_TARGET_ARCH_IS_RISCV64) || defined(LIBC_TARGET_ARCH_IS_WASM)
79 asm("" : "+r"(val));
80#endif
81#endif // !LIBC_COMPILER_IS_MSVC
82}
83
84template <typename T>
85LIBC_ALWAYS_INLINE LIBC_CONSTEXPR_DEFAULT void do_not_optimize(T &val) {
86#if LIBC_HAS_BUILTIN_IS_CONSTANT_EVALUATED
87 if (!__builtin_is_constant_evaluated())
88#endif
89 do_not_optimize_impl(val);
90}
91
92} // namespace details
93} // namespace LIBC_NAMESPACE_DECL
94
95#define LIBC_DO_NOT_OPTIMIZE(var) LIBC_NAMESPACE::details::do_not_optimize(var)
96
97#define LIBC_LIKELY(x) \
98 (__builtin_expect(LIBC_NAMESPACE::details::expects_bool_condition((x)), true))
99#define LIBC_UNLIKELY(x) \
100 (__builtin_expect(LIBC_NAMESPACE::details::expects_bool_condition((x)), \
101 false))
102
103#if defined(LIBC_COMPILER_IS_CLANG)
104#define LIBC_LOOP_NOUNROLL _Pragma("nounroll")
105#define LIBC_LOOP_UNROLL _Pragma("unroll")
106#elif defined(LIBC_COMPILER_IS_GCC)
107#if LIBC_COMPILER_GCC_VER >= 800
108#define LIBC_LOOP_NOUNROLL _Pragma("GCC unroll 0")
109#define LIBC_LOOP_UNROLL _Pragma("GCC unroll 2048")
110#else
111#define LIBC_LOOP_NOUNROLL
112#define LIBC_LOOP_UNROLL
113#endif
114#elif defined(LIBC_COMPILER_IS_MSVC)
115#define LIBC_LOOP_NOUNROLL
116#define LIBC_LOOP_UNROLL
117#else
118#error "Unhandled compiler"
119#endif
120
121// Defining optimization options for math functions.
122// TODO: Exporting this to public generated headers?
123#define LIBC_MATH_SKIP_ACCURATE_PASS 0x01
124#define LIBC_MATH_SMALL_TABLES 0x02
125#define LIBC_MATH_NO_ERRNO 0x04
126#define LIBC_MATH_NO_EXCEPT 0x08
127#define LIBC_MATH_INTERMEDIATE_COMP_IN_FLOAT 0x10
128#define LIBC_MATH_ASSUME_ROUND_NEAREST_ONLY 0x20
129#define LIBC_MATH_FAST \
130 (LIBC_MATH_SKIP_ACCURATE_PASS | LIBC_MATH_SMALL_TABLES | \
131 LIBC_MATH_NO_ERRNO | LIBC_MATH_NO_EXCEPT | \
132 LIBC_MATH_ASSUME_ROUND_NEAREST_ONLY)
133
134#ifndef LIBC_MATH
135#define LIBC_MATH 0
136#endif // LIBC_MATH
137
138#if ((LIBC_MATH) & LIBC_MATH_SKIP_ACCURATE_PASS)
139#define LIBC_MATH_HAS_SKIP_ACCURATE_PASS
140#endif
141
142#if ((LIBC_MATH) & LIBC_MATH_SMALL_TABLES)
143#define LIBC_MATH_HAS_SMALL_TABLES
144#define LIBC_MATH_SMALL_TABLES_INLINE LIBC_NOINLINE LIBC_INLINE
145#else
146#define LIBC_MATH_SMALL_TABLES_INLINE LIBC_ALWAYS_INLINE
147#endif
148
149#if ((LIBC_MATH) & LIBC_MATH_INTERMEDIATE_COMP_IN_FLOAT)
150#define LIBC_MATH_HAS_INTERMEDIATE_COMP_IN_FLOAT
151#endif
152
153#if ((LIBC_MATH) & LIBC_MATH_NO_ERRNO)
154#define LIBC_MATH_HAS_NO_ERRNO
155#endif
156
157#if ((LIBC_MATH) & LIBC_MATH_NO_EXCEPT)
158#define LIBC_MATH_HAS_NO_EXCEPT
159#endif
160
161#if ((LIBC_MATH) & LIBC_MATH_ASSUME_ROUND_NEAREST_ONLY)
162#define LIBC_MATH_HAS_ASSUME_ROUND_NEAREST_ONLY
163#define LIBC_DO_NOT_OPTIMIZE_ROUNDING(var) ((void)(var))
164#else
165#define LIBC_DO_NOT_OPTIMIZE_ROUNDING(var) LIBC_DO_NOT_OPTIMIZE(var)
166#endif
167
168// Helper macros to compare math implementation namespace tokens in #if
169// directives (e.g. when LIBC_MATH_<FUNC>_IMPL is overridden via -D).
170#define LIBC_MATH_IMPL_ID_double_eval 1
171#define LIBC_MATH_IMPL_ID_float_eval 2
172#define LIBC_MATH_IMPL_ID_float_eval_cr_rn 3
173#define LIBC_MATH_IMPL_ID_integer_eval 4
174#define LIBC_MATH_IMPL_CONCAT_INNER(a, b) a##b
175#define LIBC_MATH_IMPL_CONCAT(a, b) LIBC_MATH_IMPL_CONCAT_INNER(a, b)
176#define LIBC_MATH_IMPL_IS(impl, target) \
177 (LIBC_MATH_IMPL_CONCAT(LIBC_MATH_IMPL_ID_, impl) == \
178 LIBC_MATH_IMPL_CONCAT(LIBC_MATH_IMPL_ID_, target))
179
180#endif // LLVM_LIBC_SRC___SUPPORT_MACROS_OPTIMIZATION_H
181