1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#ifndef _LIBCPP___ATOMIC_SUPPORT_C11_H
10#define _LIBCPP___ATOMIC_SUPPORT_C11_H
11
12#include <__atomic/clear_padding.h>
13#include <__atomic/memory_order.h>
14#include <__config>
15#include <__cstddef/ptrdiff_t.h>
16#include <__memory/addressof.h>
17#include <__type_traits/enable_if.h>
18#include <__type_traits/is_pointer.h>
19#include <__type_traits/remove_const.h>
20
21#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
22# pragma GCC system_header
23#endif
24
25//
26// This file implements support for C11-style atomics
27//
28
29_LIBCPP_BEGIN_NAMESPACE_STD
30
31template <typename _Tp>
32struct __cxx_atomic_base_impl {
33 _LIBCPP_HIDE_FROM_ABI
34#ifndef _LIBCPP_CXX03_LANG
35 __cxx_atomic_base_impl() _NOEXCEPT = default;
36#else
37 __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {
38 }
39#endif // _LIBCPP_CXX03_LANG
40
41 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp __value) _NOEXCEPT
42 : __a_value(std::__clear_padding_if_needed(__value)) {}
43
44 _Atomic(_Tp) __a_value;
45};
46
47#define __cxx_atomic_is_lock_free(__s) __c11_atomic_is_lock_free(__s)
48
49_LIBCPP_HIDE_FROM_ABI inline void __cxx_atomic_thread_fence(memory_order __order) _NOEXCEPT {
50 __c11_atomic_thread_fence(static_cast<__memory_order_underlying_t>(__order));
51}
52
53_LIBCPP_HIDE_FROM_ABI inline void __cxx_atomic_signal_fence(memory_order __order) _NOEXCEPT {
54 __c11_atomic_signal_fence(static_cast<__memory_order_underlying_t>(__order));
55}
56
57template <class _Tp>
58_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val) _NOEXCEPT {
59 std::__clear_padding_if_needed(__val);
60 __c11_atomic_init(std::addressof(__a->__a_value), __val);
61}
62template <class _Tp>
63_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val) _NOEXCEPT {
64 std::__clear_padding_if_needed(__val);
65 __c11_atomic_init(std::addressof(__a->__a_value), __val);
66}
67
68template <class _Tp>
69_LIBCPP_HIDE_FROM_ABI void
70__cxx_atomic_store(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val, memory_order __order) _NOEXCEPT {
71 std::__clear_padding_if_needed(__val);
72 __c11_atomic_store(std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order));
73}
74template <class _Tp>
75_LIBCPP_HIDE_FROM_ABI void
76__cxx_atomic_store(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) _NOEXCEPT {
77 std::__clear_padding_if_needed(__val);
78 __c11_atomic_store(std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order));
79}
80
81template <class _Tp>
82_LIBCPP_HIDE_FROM_ABI _Tp
83__cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const volatile* __a, memory_order __order) _NOEXCEPT {
84 using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*;
85 return __c11_atomic_load(
86 const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
87}
88template <class _Tp>
89_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const* __a, memory_order __order) _NOEXCEPT {
90 using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*;
91 return __c11_atomic_load(
92 const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
93}
94
95template <class _Tp>
96_LIBCPP_HIDE_FROM_ABI void
97__cxx_atomic_load_inplace(__cxx_atomic_base_impl<_Tp> const volatile* __a, _Tp* __dst, memory_order __order) _NOEXCEPT {
98 using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*;
99 *__dst = __c11_atomic_load(
100 const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
101}
102template <class _Tp>
103_LIBCPP_HIDE_FROM_ABI void
104__cxx_atomic_load_inplace(__cxx_atomic_base_impl<_Tp> const* __a, _Tp* __dst, memory_order __order) _NOEXCEPT {
105 using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*;
106 *__dst = __c11_atomic_load(
107 const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
108}
109
110template <class _Tp>
111_LIBCPP_HIDE_FROM_ABI _Tp
112__cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __value, memory_order __order) _NOEXCEPT {
113 std::__clear_padding_if_needed(__value);
114 return __c11_atomic_exchange(
115 std::addressof(__a->__a_value), __value, static_cast<__memory_order_underlying_t>(__order));
116}
117template <class _Tp>
118_LIBCPP_HIDE_FROM_ABI _Tp
119__cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp>* __a, _Tp __value, memory_order __order) _NOEXCEPT {
120 std::__clear_padding_if_needed(__value);
121 return __c11_atomic_exchange(
122 std::addressof(__a->__a_value), __value, static_cast<__memory_order_underlying_t>(__order));
123}
124
125_LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR memory_order __to_failure_order(memory_order __order) {
126 // Avoid switch statement to make this a constexpr.
127 return __order == memory_order_release
128 ? memory_order_relaxed
129 : (__order == memory_order_acq_rel ? memory_order_acquire : __order);
130}
131
132template <class _Tp>
133_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong(
134 __cxx_atomic_base_impl<_Tp> volatile* __a,
135 _Tp* __expected,
136 _Tp __value,
137 memory_order __success,
138 memory_order __failure) _NOEXCEPT {
139 return std::__atomic_cas_with_clear_padding(
140 __expected,
141 __value,
142 // Use always_inline since Clang fails to inline the lambda, even when the memory order is a constant
143 [__a, __success, __failure](_Tp* __expected_or_copy, _Tp __value_maybe_padding_cleared) _LIBCPP_ALWAYS_INLINE {
144 return __c11_atomic_compare_exchange_strong(
145 std::addressof(__a->__a_value),
146 __expected_or_copy,
147 __value_maybe_padding_cleared,
148 static_cast<__memory_order_underlying_t>(__success),
149 static_cast<__memory_order_underlying_t>(std::__to_failure_order(order: __failure)));
150 });
151}
152template <class _Tp>
153_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong(
154 __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure)
155 _NOEXCEPT {
156 return std::__atomic_cas_with_clear_padding(
157 __expected,
158 __value,
159 [__a, __success, __failure](_Tp* __expected_or_copy, _Tp __value_maybe_padding_cleared) _LIBCPP_ALWAYS_INLINE {
160 return __c11_atomic_compare_exchange_strong(
161 std::addressof(__a->__a_value),
162 __expected_or_copy,
163 __value_maybe_padding_cleared,
164 static_cast<__memory_order_underlying_t>(__success),
165 static_cast<__memory_order_underlying_t>(std::__to_failure_order(order: __failure)));
166 });
167}
168
169template <class _Tp>
170_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak(
171 __cxx_atomic_base_impl<_Tp> volatile* __a,
172 _Tp* __expected,
173 _Tp __value,
174 memory_order __success,
175 memory_order __failure) _NOEXCEPT {
176 return std::__atomic_cas_with_clear_padding(
177 __expected,
178 __value,
179 [__a, __success, __failure](_Tp* __expected_or_copy, _Tp __value_maybe_padding_cleared) _LIBCPP_ALWAYS_INLINE {
180 return __c11_atomic_compare_exchange_weak(
181 std::addressof(__a->__a_value),
182 __expected_or_copy,
183 __value_maybe_padding_cleared,
184 static_cast<__memory_order_underlying_t>(__success),
185 static_cast<__memory_order_underlying_t>(std::__to_failure_order(order: __failure)));
186 });
187}
188
189template <class _Tp>
190_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak(
191 __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure)
192 _NOEXCEPT {
193 return std::__atomic_cas_with_clear_padding(
194 __expected,
195 __value,
196 [__a, __success, __failure](_Tp* __expected_or_copy, _Tp __value_maybe_padding_cleared) _LIBCPP_ALWAYS_INLINE {
197 return __c11_atomic_compare_exchange_weak(
198 std::addressof(__a->__a_value),
199 __expected_or_copy,
200 __value_maybe_padding_cleared,
201 static_cast<__memory_order_underlying_t>(__success),
202 static_cast<__memory_order_underlying_t>(std::__to_failure_order(order: __failure)));
203 });
204}
205
206template <class _Tp>
207_LIBCPP_HIDE_FROM_ABI _Tp
208__cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT {
209 return __c11_atomic_fetch_add(
210 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
211}
212template <class _Tp>
213_LIBCPP_HIDE_FROM_ABI _Tp
214__cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp>* __a, _Tp __delta, memory_order __order) _NOEXCEPT {
215 return __c11_atomic_fetch_add(
216 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
217}
218
219template <class _Tp>
220_LIBCPP_HIDE_FROM_ABI _Tp*
221__cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
222 return __c11_atomic_fetch_add(
223 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
224}
225template <class _Tp>
226_LIBCPP_HIDE_FROM_ABI _Tp*
227__cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*>* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
228 return __c11_atomic_fetch_add(
229 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
230}
231
232template <class _Tp>
233_LIBCPP_HIDE_FROM_ABI _Tp
234__cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT {
235 return __c11_atomic_fetch_sub(
236 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
237}
238template <class _Tp>
239_LIBCPP_HIDE_FROM_ABI _Tp
240__cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp>* __a, _Tp __delta, memory_order __order) _NOEXCEPT {
241 return __c11_atomic_fetch_sub(
242 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
243}
244template <class _Tp>
245_LIBCPP_HIDE_FROM_ABI _Tp*
246__cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
247 return __c11_atomic_fetch_sub(
248 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
249}
250template <class _Tp>
251_LIBCPP_HIDE_FROM_ABI _Tp*
252__cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*>* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
253 return __c11_atomic_fetch_sub(
254 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
255}
256
257template <class _Tp>
258_LIBCPP_HIDE_FROM_ABI _Tp
259__cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
260 return __c11_atomic_fetch_and(
261 std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
262}
263template <class _Tp>
264_LIBCPP_HIDE_FROM_ABI _Tp
265__cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
266 return __c11_atomic_fetch_and(
267 std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
268}
269
270template <class _Tp>
271_LIBCPP_HIDE_FROM_ABI _Tp
272__cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
273 return __c11_atomic_fetch_or(
274 std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
275}
276template <class _Tp>
277_LIBCPP_HIDE_FROM_ABI _Tp
278__cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
279 return __c11_atomic_fetch_or(
280 std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
281}
282
283template <class _Tp>
284_LIBCPP_HIDE_FROM_ABI _Tp
285__cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
286 return __c11_atomic_fetch_xor(
287 std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
288}
289template <class _Tp>
290_LIBCPP_HIDE_FROM_ABI _Tp
291__cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
292 return __c11_atomic_fetch_xor(
293 std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
294}
295
296// Clang's __c11_atomic_fetch_max/min builtins do not accept pointer
297// arguments, so dispatch to a CAS loop for pointer types.
298// TODO: Use the builtin for pointer types once Clang accepts them.
299template <class _Tp, __enable_if_t<!is_pointer<_Tp>::value, int> = 0>
300_LIBCPP_HIDE_FROM_ABI _Tp
301__cxx_atomic_fetch_max(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val, memory_order __order) _NOEXCEPT {
302 return __c11_atomic_fetch_max(
303 std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order));
304}
305template <class _Tp, __enable_if_t<is_pointer<_Tp>::value, int> = 0>
306_LIBCPP_HIDE_FROM_ABI _Tp
307__cxx_atomic_fetch_max(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val, memory_order __order) _NOEXCEPT {
308 _Tp __ret = std::__cxx_atomic_load(__a, memory_order_relaxed);
309 _Tp __value;
310 do {
311 __value = __ret > __val ? __ret : __val;
312 } while (
313 !std::__cxx_atomic_compare_exchange_weak(__a, std::addressof(__ret), __value, __order, memory_order_relaxed));
314 return __ret;
315}
316
317template <class _Tp, __enable_if_t<!is_pointer<_Tp>::value, int> = 0>
318_LIBCPP_HIDE_FROM_ABI _Tp
319__cxx_atomic_fetch_max(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) _NOEXCEPT {
320 return __c11_atomic_fetch_max(
321 std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order));
322}
323template <class _Tp, __enable_if_t<is_pointer<_Tp>::value, int> = 0>
324_LIBCPP_HIDE_FROM_ABI _Tp
325__cxx_atomic_fetch_max(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) _NOEXCEPT {
326 _Tp __ret = std::__cxx_atomic_load(__a, memory_order_relaxed);
327 _Tp __value;
328 do {
329 __value = __ret > __val ? __ret : __val;
330 } while (
331 !std::__cxx_atomic_compare_exchange_weak(__a, std::addressof(__ret), __value, __order, memory_order_relaxed));
332 return __ret;
333}
334
335template <class _Tp, __enable_if_t<!is_pointer<_Tp>::value, int> = 0>
336_LIBCPP_HIDE_FROM_ABI _Tp
337__cxx_atomic_fetch_min(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val, memory_order __order) _NOEXCEPT {
338 return __c11_atomic_fetch_min(
339 std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order));
340}
341template <class _Tp, __enable_if_t<is_pointer<_Tp>::value, int> = 0>
342_LIBCPP_HIDE_FROM_ABI _Tp
343__cxx_atomic_fetch_min(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val, memory_order __order) _NOEXCEPT {
344 _Tp __ret = std::__cxx_atomic_load(__a, memory_order_relaxed);
345 _Tp __value;
346 do {
347 __value = __ret < __val ? __ret : __val;
348 } while (
349 !std::__cxx_atomic_compare_exchange_weak(__a, std::addressof(__ret), __value, __order, memory_order_relaxed));
350 return __ret;
351}
352
353template <class _Tp, __enable_if_t<!is_pointer<_Tp>::value, int> = 0>
354_LIBCPP_HIDE_FROM_ABI _Tp
355__cxx_atomic_fetch_min(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) _NOEXCEPT {
356 return __c11_atomic_fetch_min(
357 std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order));
358}
359template <class _Tp, __enable_if_t<is_pointer<_Tp>::value, int> = 0>
360_LIBCPP_HIDE_FROM_ABI _Tp
361__cxx_atomic_fetch_min(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) _NOEXCEPT {
362 _Tp __ret = std::__cxx_atomic_load(__a, memory_order_relaxed);
363 _Tp __value;
364 do {
365 __value = __ret < __val ? __ret : __val;
366 } while (
367 !std::__cxx_atomic_compare_exchange_weak(__a, std::addressof(__ret), __value, __order, memory_order_relaxed));
368 return __ret;
369}
370
371_LIBCPP_END_NAMESPACE_STD
372
373#endif // _LIBCPP___ATOMIC_SUPPORT_C11_H
374