1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#ifndef _LIBCPP___ATOMIC_SUPPORT_C11_H
10#define _LIBCPP___ATOMIC_SUPPORT_C11_H
11
12#include <__atomic/clear_padding.h>
13#include <__atomic/memory_order.h>
14#include <__config>
15#include <__cstddef/ptrdiff_t.h>
16#include <__memory/addressof.h>
17#include <__type_traits/enable_if.h>
18#include <__type_traits/is_pointer.h>
19#include <__type_traits/remove_const.h>
20
21#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
22# pragma GCC system_header
23#endif
24
25//
26// This file implements support for C11-style atomics
27//
28
29_LIBCPP_BEGIN_NAMESPACE_STD
30
31template <typename _Tp>
32struct __cxx_atomic_base_impl {
33#ifndef _LIBCPP_CXX03_LANG
34 __cxx_atomic_base_impl() _NOEXCEPT = default;
35#else
36 __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {}
37#endif // _LIBCPP_CXX03_LANG
38
39 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp __value) _NOEXCEPT
40 : __a_value(std::__clear_padding_if_needed(__value)) {}
41
42 _Atomic(_Tp) __a_value;
43};
44
45#define __cxx_atomic_is_lock_free(__s) __c11_atomic_is_lock_free(__s)
46
47inline void __cxx_atomic_thread_fence(memory_order __order) _NOEXCEPT {
48 __c11_atomic_thread_fence(static_cast<__memory_order_underlying_t>(__order));
49}
50
51inline void __cxx_atomic_signal_fence(memory_order __order) _NOEXCEPT {
52 __c11_atomic_signal_fence(static_cast<__memory_order_underlying_t>(__order));
53}
54
55template <class _Tp>
56void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val) _NOEXCEPT {
57 std::__clear_padding_if_needed(__val);
58 __c11_atomic_init(std::addressof(__a->__a_value), __val);
59}
60template <class _Tp>
61void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val) _NOEXCEPT {
62 std::__clear_padding_if_needed(__val);
63 __c11_atomic_init(std::addressof(__a->__a_value), __val);
64}
65
66template <class _Tp>
67void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val, memory_order __order) _NOEXCEPT {
68 std::__clear_padding_if_needed(__val);
69 __c11_atomic_store(std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order));
70}
71template <class _Tp>
72void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) _NOEXCEPT {
73 std::__clear_padding_if_needed(__val);
74 __c11_atomic_store(std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order));
75}
76
77template <class _Tp>
78_Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const volatile* __a, memory_order __order) _NOEXCEPT {
79 using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*;
80 return __c11_atomic_load(
81 const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
82}
83template <class _Tp>
84_Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const* __a, memory_order __order) _NOEXCEPT {
85 using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*;
86 return __c11_atomic_load(
87 const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
88}
89
90template <class _Tp>
91void __cxx_atomic_load_inplace(
92 __cxx_atomic_base_impl<_Tp> const volatile* __a, _Tp* __dst, memory_order __order) _NOEXCEPT {
93 using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*;
94 *__dst = __c11_atomic_load(
95 const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
96}
97template <class _Tp>
98void __cxx_atomic_load_inplace(__cxx_atomic_base_impl<_Tp> const* __a, _Tp* __dst, memory_order __order) _NOEXCEPT {
99 using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*;
100 *__dst = __c11_atomic_load(
101 const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
102}
103
104template <class _Tp>
105_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __value, memory_order __order) _NOEXCEPT {
106 std::__clear_padding_if_needed(__value);
107 return __c11_atomic_exchange(
108 std::addressof(__a->__a_value), __value, static_cast<__memory_order_underlying_t>(__order));
109}
110template <class _Tp>
111_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp>* __a, _Tp __value, memory_order __order) _NOEXCEPT {
112 std::__clear_padding_if_needed(__value);
113 return __c11_atomic_exchange(
114 std::addressof(__a->__a_value), __value, static_cast<__memory_order_underlying_t>(__order));
115}
116
117inline _LIBCPP_CONSTEXPR memory_order __to_failure_order(memory_order __order) {
118 // Avoid switch statement to make this a constexpr.
119 return __order == memory_order_release
120 ? memory_order_relaxed
121 : (__order == memory_order_acq_rel ? memory_order_acquire : __order);
122}
123
124template <class _Tp>
125bool __cxx_atomic_compare_exchange_strong(
126 __cxx_atomic_base_impl<_Tp> volatile* __a,
127 _Tp* __expected,
128 _Tp __value,
129 memory_order __success,
130 memory_order __failure) _NOEXCEPT {
131 return std::__atomic_cas_with_clear_padding(
132 __expected,
133 __value,
134 // Use always_inline since Clang fails to inline the lambda, even when the memory order is a constant
135 [__a, __success, __failure](_Tp* __expected_or_copy, _Tp __value_maybe_padding_cleared) _LIBCPP_ALWAYS_INLINE {
136 return __c11_atomic_compare_exchange_strong(
137 std::addressof(__a->__a_value),
138 __expected_or_copy,
139 __value_maybe_padding_cleared,
140 static_cast<__memory_order_underlying_t>(__success),
141 static_cast<__memory_order_underlying_t>(std::__to_failure_order(order: __failure)));
142 });
143}
144template <class _Tp>
145bool __cxx_atomic_compare_exchange_strong(
146 __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure)
147 _NOEXCEPT {
148 return std::__atomic_cas_with_clear_padding(
149 __expected,
150 __value,
151 [__a, __success, __failure](_Tp* __expected_or_copy, _Tp __value_maybe_padding_cleared) _LIBCPP_ALWAYS_INLINE {
152 return __c11_atomic_compare_exchange_strong(
153 std::addressof(__a->__a_value),
154 __expected_or_copy,
155 __value_maybe_padding_cleared,
156 static_cast<__memory_order_underlying_t>(__success),
157 static_cast<__memory_order_underlying_t>(std::__to_failure_order(order: __failure)));
158 });
159}
160
161template <class _Tp>
162bool __cxx_atomic_compare_exchange_weak(
163 __cxx_atomic_base_impl<_Tp> volatile* __a,
164 _Tp* __expected,
165 _Tp __value,
166 memory_order __success,
167 memory_order __failure) _NOEXCEPT {
168 return std::__atomic_cas_with_clear_padding(
169 __expected,
170 __value,
171 [__a, __success, __failure](_Tp* __expected_or_copy, _Tp __value_maybe_padding_cleared) _LIBCPP_ALWAYS_INLINE {
172 return __c11_atomic_compare_exchange_weak(
173 std::addressof(__a->__a_value),
174 __expected_or_copy,
175 __value_maybe_padding_cleared,
176 static_cast<__memory_order_underlying_t>(__success),
177 static_cast<__memory_order_underlying_t>(std::__to_failure_order(order: __failure)));
178 });
179}
180
181template <class _Tp>
182bool __cxx_atomic_compare_exchange_weak(
183 __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure)
184 _NOEXCEPT {
185 return std::__atomic_cas_with_clear_padding(
186 __expected,
187 __value,
188 [__a, __success, __failure](_Tp* __expected_or_copy, _Tp __value_maybe_padding_cleared) _LIBCPP_ALWAYS_INLINE {
189 return __c11_atomic_compare_exchange_weak(
190 std::addressof(__a->__a_value),
191 __expected_or_copy,
192 __value_maybe_padding_cleared,
193 static_cast<__memory_order_underlying_t>(__success),
194 static_cast<__memory_order_underlying_t>(std::__to_failure_order(order: __failure)));
195 });
196}
197
198template <class _Tp>
199_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT {
200 return __c11_atomic_fetch_add(
201 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
202}
203template <class _Tp>
204_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp>* __a, _Tp __delta, memory_order __order) _NOEXCEPT {
205 return __c11_atomic_fetch_add(
206 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
207}
208
209template <class _Tp>
210_Tp* __cxx_atomic_fetch_add(
211 __cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
212 return __c11_atomic_fetch_add(
213 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
214}
215template <class _Tp>
216_Tp* __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*>* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
217 return __c11_atomic_fetch_add(
218 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
219}
220
221template <class _Tp>
222_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT {
223 return __c11_atomic_fetch_sub(
224 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
225}
226template <class _Tp>
227_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp>* __a, _Tp __delta, memory_order __order) _NOEXCEPT {
228 return __c11_atomic_fetch_sub(
229 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
230}
231template <class _Tp>
232_Tp* __cxx_atomic_fetch_sub(
233 __cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
234 return __c11_atomic_fetch_sub(
235 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
236}
237template <class _Tp>
238_Tp* __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*>* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
239 return __c11_atomic_fetch_sub(
240 std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
241}
242
243template <class _Tp>
244_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
245 return __c11_atomic_fetch_and(
246 std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
247}
248template <class _Tp>
249_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
250 return __c11_atomic_fetch_and(
251 std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
252}
253
254template <class _Tp>
255_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
256 return __c11_atomic_fetch_or(
257 std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
258}
259template <class _Tp>
260_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
261 return __c11_atomic_fetch_or(
262 std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
263}
264
265template <class _Tp>
266_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
267 return __c11_atomic_fetch_xor(
268 std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
269}
270template <class _Tp>
271_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
272 return __c11_atomic_fetch_xor(
273 std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
274}
275
276// Clang's __c11_atomic_fetch_max/min builtins do not accept pointer
277// arguments, so dispatch to a CAS loop for pointer types.
278// TODO: Use the builtin for pointer types once Clang accepts them.
279template <class _Tp, __enable_if_t<!is_pointer<_Tp>::value, int> = 0>
280_LIBCPP_HIDE_FROM_ABI _Tp
281__cxx_atomic_fetch_max(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val, memory_order __order) _NOEXCEPT {
282 return __c11_atomic_fetch_max(
283 std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order));
284}
285template <class _Tp, __enable_if_t<is_pointer<_Tp>::value, int> = 0>
286_LIBCPP_HIDE_FROM_ABI _Tp
287__cxx_atomic_fetch_max(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val, memory_order __order) _NOEXCEPT {
288 _Tp __ret = std::__cxx_atomic_load(__a, memory_order_relaxed);
289 _Tp __value;
290 do {
291 __value = __ret > __val ? __ret : __val;
292 } while (
293 !std::__cxx_atomic_compare_exchange_weak(__a, std::addressof(__ret), __value, __order, memory_order_relaxed));
294 return __ret;
295}
296
297template <class _Tp, __enable_if_t<!is_pointer<_Tp>::value, int> = 0>
298_LIBCPP_HIDE_FROM_ABI _Tp
299__cxx_atomic_fetch_max(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) _NOEXCEPT {
300 return __c11_atomic_fetch_max(
301 std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order));
302}
303template <class _Tp, __enable_if_t<is_pointer<_Tp>::value, int> = 0>
304_LIBCPP_HIDE_FROM_ABI _Tp
305__cxx_atomic_fetch_max(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) _NOEXCEPT {
306 _Tp __ret = std::__cxx_atomic_load(__a, memory_order_relaxed);
307 _Tp __value;
308 do {
309 __value = __ret > __val ? __ret : __val;
310 } while (
311 !std::__cxx_atomic_compare_exchange_weak(__a, std::addressof(__ret), __value, __order, memory_order_relaxed));
312 return __ret;
313}
314
315template <class _Tp, __enable_if_t<!is_pointer<_Tp>::value, int> = 0>
316_LIBCPP_HIDE_FROM_ABI _Tp
317__cxx_atomic_fetch_min(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val, memory_order __order) _NOEXCEPT {
318 return __c11_atomic_fetch_min(
319 std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order));
320}
321template <class _Tp, __enable_if_t<is_pointer<_Tp>::value, int> = 0>
322_LIBCPP_HIDE_FROM_ABI _Tp
323__cxx_atomic_fetch_min(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val, memory_order __order) _NOEXCEPT {
324 _Tp __ret = std::__cxx_atomic_load(__a, memory_order_relaxed);
325 _Tp __value;
326 do {
327 __value = __ret < __val ? __ret : __val;
328 } while (
329 !std::__cxx_atomic_compare_exchange_weak(__a, std::addressof(__ret), __value, __order, memory_order_relaxed));
330 return __ret;
331}
332
333template <class _Tp, __enable_if_t<!is_pointer<_Tp>::value, int> = 0>
334_LIBCPP_HIDE_FROM_ABI _Tp
335__cxx_atomic_fetch_min(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) _NOEXCEPT {
336 return __c11_atomic_fetch_min(
337 std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order));
338}
339template <class _Tp, __enable_if_t<is_pointer<_Tp>::value, int> = 0>
340_LIBCPP_HIDE_FROM_ABI _Tp
341__cxx_atomic_fetch_min(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) _NOEXCEPT {
342 _Tp __ret = std::__cxx_atomic_load(__a, memory_order_relaxed);
343 _Tp __value;
344 do {
345 __value = __ret < __val ? __ret : __val;
346 } while (
347 !std::__cxx_atomic_compare_exchange_weak(__a, std::addressof(__ret), __value, __order, memory_order_relaxed));
348 return __ret;
349}
350
351_LIBCPP_END_NAMESPACE_STD
352
353#endif // _LIBCPP___ATOMIC_SUPPORT_C11_H
354