1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#ifndef _LIBCPP___ATOMIC_SUPPORT_C11_H
10#define _LIBCPP___ATOMIC_SUPPORT_C11_H
11
12#include <__atomic/memory_order.h>
13#include <__config>
14#include <__cstddef/ptrdiff_t.h>
15#include <__memory/addressof.h>
16#include <__type_traits/enable_if.h>
17#include <__type_traits/is_pointer.h>
18#include <__type_traits/remove_const.h>
19
20#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
21# pragma GCC system_header
22#endif
23
24//
25// This file implements support for C11-style atomics
26//
27
28_LIBCPP_BEGIN_NAMESPACE_STD
29
30template <typename _Tp>
31struct __cxx_atomic_base_impl {
32 _LIBCPP_HIDE_FROM_ABI
33#ifndef _LIBCPP_CXX03_LANG
34 __cxx_atomic_base_impl() _NOEXCEPT = default;
35#else
36 __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {
37 }
38#endif // _LIBCPP_CXX03_LANG
39 _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp __value) _NOEXCEPT : __a_value(__value) {}
40 _Atomic(_Tp) __a_value;
41};
42
43#define __cxx_atomic_is_lock_free(__s) __c11_atomic_is_lock_free(__s)
44
45_LIBCPP_HIDE_FROM_ABI inline void __cxx_atomic_thread_fence(memory_order __order) _NOEXCEPT {
46 __c11_atomic_thread_fence(static_cast<__memory_order_underlying_t>(__order));
47}
48
49_LIBCPP_HIDE_FROM_ABI inline void __cxx_atomic_signal_fence(memory_order __order) _NOEXCEPT {
50 __c11_atomic_signal_fence(static_cast<__memory_order_underlying_t>(__order));
51}
52
53template <class _Tp>
54_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val) _NOEXCEPT {
55 __c11_atomic_init(std::addressof(__a->__a_value), __val);
56}
57template <class _Tp>
58_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val) _NOEXCEPT {
59 __c11_atomic_init(std::addressof(__a->__a_value), __val);
60}
61
62template <class _Tp>
63_LIBCPP_HIDE_FROM_ABI void
64__cxx_atomic_store(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val, memory_order __order) _NOEXCEPT {
65 __c11_atomic_store(std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order));
66}
67template <class _Tp>
68_LIBCPP_HIDE_FROM_ABI void
69__cxx_atomic_store(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) _NOEXCEPT {
70 __c11_atomic_store(std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order));
71}
72
73template <class _Tp>
74_LIBCPP_HIDE_FROM_ABI _Tp
75__cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const volatile* __a, memory_order __order) _NOEXCEPT {
76 using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*;
77 return __c11_atomic_load(
78 const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
79}
80template <class _Tp>
81_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const* __a, memory_order __order) _NOEXCEPT {
82 using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*;
83 return __c11_atomic_load(
84 const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
85}
86
87template <class _Tp>
88_LIBCPP_HIDE_FROM_ABI void
89__cxx_atomic_load_inplace(__cxx_atomic_base_impl<_Tp> const volatile* __a, _Tp* __dst, memory_order __order) _NOEXCEPT {
90 using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*;
91 *__dst = __c11_atomic_load(
92 const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
93}
94template <class _Tp>
95_LIBCPP_HIDE_FROM_ABI void
96__cxx_atomic_load_inplace(__cxx_atomic_base_impl<_Tp> const* __a, _Tp* __dst, memory_order __order) _NOEXCEPT {
97 using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*;
98 *__dst = __c11_atomic_load(
99 const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
100}
101
102template <class _Tp>
103_LIBCPP_HIDE_FROM_ABI _Tp
104__cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __value, memory_order __order) _NOEXCEPT {
105 return __c11_atomic_exchange(
106 std::addressof(__a->__a_value), __value, static_cast<__memory_order_underlying_t>(__order));
107}
108template <class _Tp>
109_LIBCPP_HIDE_FROM_ABI _Tp
110__cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp>* __a, _Tp __value, memory_order __order) _NOEXCEPT {
111 return __c11_atomic_exchange(
112 std::addressof(__a->__a_value), __value, static_cast<__memory_order_underlying_t>(__order));
113}
114
115_LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR memory_order __to_failure_order(memory_order __order) {
116 // Avoid switch statement to make this a constexpr.
117 return __order == memory_order_release
118 ? memory_order_relaxed
119 : (__order == memory_order_acq_rel ? memory_order_acquire : __order);
120}
121
122template <class _Tp>
123_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong(
124 __cxx_atomic_base_impl<_Tp> volatile* __a,
125 _Tp* __expected,
126 _Tp __value,
127 memory_order __success,
128 memory_order __failure) _NOEXCEPT {
129 return __c11_atomic_compare_exchange_strong(
130 std::addressof(__a->__a_value),
131 __expected,
132 __value,
133 static_cast<__memory_order_underlying_t>(__success),
134 static_cast<__memory_order_underlying_t>(__to_failure_order(order: __failure)));
135}
136template <class _Tp>
137_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong(
138 __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure)
139 _NOEXCEPT {
140 return __c11_atomic_compare_exchange_strong(
141 std::addressof(__a->__a_value),
142 __expected,
143 __value,
144 static_cast<__memory_order_underlying_t>(__success),
145 static_cast<__memory_order_underlying_t>(__to_failure_order(order: __failure)));
146}
147
148template <class _Tp>
149_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak(
150 __cxx_atomic_base_impl<_Tp> volatile* __a,
151 _Tp* __expected,
152 _Tp __value,
153 memory_order __success,
154 memory_order __failure) _NOEXCEPT {
155 return __c11_atomic_compare_exchange_weak(
156 std::addressof(__a->__a_value),
157 __expected,
158 __value,
159 static_cast<__memory_order_underlying_t>(__success),
160 static_cast<__memory_order_underlying_t>(__to_failure_order(order: __failure)));
161}
162template <class _Tp>
163_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak(
164 __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure)
165 _NOEXCEPT {
166 return __c11_atomic_compare_exchange_weak(
167 std::addressof(__a->__a_value),
168 __expected,
169 __value,
170 static_cast<__memory_order_underlying_t>(__success),
171 static_cast<__memory_order_underlying_t>(__to_failure_order(order: __failure)));
172}
173
174template <class _Tp>
175_LIBCPP_HIDE_FROM_ABI _Tp
176__cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT {
177 return __c11_atomic_fetch_add(
178 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
179}
180template <class _Tp>
181_LIBCPP_HIDE_FROM_ABI _Tp
182__cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp>* __a, _Tp __delta, memory_order __order) _NOEXCEPT {
183 return __c11_atomic_fetch_add(
184 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
185}
186
187template <class _Tp>
188_LIBCPP_HIDE_FROM_ABI _Tp*
189__cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
190 return __c11_atomic_fetch_add(
191 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
192}
193template <class _Tp>
194_LIBCPP_HIDE_FROM_ABI _Tp*
195__cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*>* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
196 return __c11_atomic_fetch_add(
197 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
198}
199
200template <class _Tp>
201_LIBCPP_HIDE_FROM_ABI _Tp
202__cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT {
203 return __c11_atomic_fetch_sub(
204 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
205}
206template <class _Tp>
207_LIBCPP_HIDE_FROM_ABI _Tp
208__cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp>* __a, _Tp __delta, memory_order __order) _NOEXCEPT {
209 return __c11_atomic_fetch_sub(
210 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
211}
212template <class _Tp>
213_LIBCPP_HIDE_FROM_ABI _Tp*
214__cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
215 return __c11_atomic_fetch_sub(
216 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
217}
218template <class _Tp>
219_LIBCPP_HIDE_FROM_ABI _Tp*
220__cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*>* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
221 return __c11_atomic_fetch_sub(
222 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
223}
224
225template <class _Tp>
226_LIBCPP_HIDE_FROM_ABI _Tp
227__cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
228 return __c11_atomic_fetch_and(
229 std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
230}
231template <class _Tp>
232_LIBCPP_HIDE_FROM_ABI _Tp
233__cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
234 return __c11_atomic_fetch_and(
235 std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
236}
237
238template <class _Tp>
239_LIBCPP_HIDE_FROM_ABI _Tp
240__cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
241 return __c11_atomic_fetch_or(
242 std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
243}
244template <class _Tp>
245_LIBCPP_HIDE_FROM_ABI _Tp
246__cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
247 return __c11_atomic_fetch_or(
248 std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
249}
250
251template <class _Tp>
252_LIBCPP_HIDE_FROM_ABI _Tp
253__cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
254 return __c11_atomic_fetch_xor(
255 std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
256}
257template <class _Tp>
258_LIBCPP_HIDE_FROM_ABI _Tp
259__cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
260 return __c11_atomic_fetch_xor(
261 std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
262}
263
264// Clang's __c11_atomic_fetch_max/min builtins do not accept pointer
265// arguments, so dispatch to a CAS loop for pointer types.
266// TODO: Use the builtin for pointer types once Clang accepts them.
267template <class _Tp, __enable_if_t<!is_pointer<_Tp>::value, int> = 0>
268_LIBCPP_HIDE_FROM_ABI _Tp
269__cxx_atomic_fetch_max(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val, memory_order __order) _NOEXCEPT {
270 return __c11_atomic_fetch_max(
271 std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order));
272}
273template <class _Tp, __enable_if_t<is_pointer<_Tp>::value, int> = 0>
274_LIBCPP_HIDE_FROM_ABI _Tp
275__cxx_atomic_fetch_max(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val, memory_order __order) _NOEXCEPT {
276 _Tp __ret = std::__cxx_atomic_load(__a, memory_order_relaxed);
277 _Tp __value;
278 do {
279 __value = __ret > __val ? __ret : __val;
280 } while (
281 !std::__cxx_atomic_compare_exchange_weak(__a, std::addressof(__ret), __value, __order, memory_order_relaxed));
282 return __ret;
283}
284
285template <class _Tp, __enable_if_t<!is_pointer<_Tp>::value, int> = 0>
286_LIBCPP_HIDE_FROM_ABI _Tp
287__cxx_atomic_fetch_max(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) _NOEXCEPT {
288 return __c11_atomic_fetch_max(
289 std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order));
290}
291template <class _Tp, __enable_if_t<is_pointer<_Tp>::value, int> = 0>
292_LIBCPP_HIDE_FROM_ABI _Tp
293__cxx_atomic_fetch_max(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) _NOEXCEPT {
294 _Tp __ret = std::__cxx_atomic_load(__a, memory_order_relaxed);
295 _Tp __value;
296 do {
297 __value = __ret > __val ? __ret : __val;
298 } while (
299 !std::__cxx_atomic_compare_exchange_weak(__a, std::addressof(__ret), __value, __order, memory_order_relaxed));
300 return __ret;
301}
302
303template <class _Tp, __enable_if_t<!is_pointer<_Tp>::value, int> = 0>
304_LIBCPP_HIDE_FROM_ABI _Tp
305__cxx_atomic_fetch_min(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val, memory_order __order) _NOEXCEPT {
306 return __c11_atomic_fetch_min(
307 std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order));
308}
309template <class _Tp, __enable_if_t<is_pointer<_Tp>::value, int> = 0>
310_LIBCPP_HIDE_FROM_ABI _Tp
311__cxx_atomic_fetch_min(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val, memory_order __order) _NOEXCEPT {
312 _Tp __ret = std::__cxx_atomic_load(__a, memory_order_relaxed);
313 _Tp __value;
314 do {
315 __value = __ret < __val ? __ret : __val;
316 } while (
317 !std::__cxx_atomic_compare_exchange_weak(__a, std::addressof(__ret), __value, __order, memory_order_relaxed));
318 return __ret;
319}
320
321template <class _Tp, __enable_if_t<!is_pointer<_Tp>::value, int> = 0>
322_LIBCPP_HIDE_FROM_ABI _Tp
323__cxx_atomic_fetch_min(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) _NOEXCEPT {
324 return __c11_atomic_fetch_min(
325 std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order));
326}
327template <class _Tp, __enable_if_t<is_pointer<_Tp>::value, int> = 0>
328_LIBCPP_HIDE_FROM_ABI _Tp
329__cxx_atomic_fetch_min(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) _NOEXCEPT {
330 _Tp __ret = std::__cxx_atomic_load(__a, memory_order_relaxed);
331 _Tp __value;
332 do {
333 __value = __ret < __val ? __ret : __val;
334 } while (
335 !std::__cxx_atomic_compare_exchange_weak(__a, std::addressof(__ret), __value, __order, memory_order_relaxed));
336 return __ret;
337}
338
339_LIBCPP_END_NAMESPACE_STD
340
341#endif // _LIBCPP___ATOMIC_SUPPORT_C11_H
342