1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#ifndef _LIBCPP___ATOMIC_ATOMIC_H
10#define _LIBCPP___ATOMIC_ATOMIC_H
11
12#include <__atomic/atomic_sync.h>
13#include <__atomic/atomic_waitable_traits.h>
14#include <__atomic/check_memory_order.h>
15#include <__atomic/floating_point_helper.h>
16#include <__atomic/is_always_lock_free.h>
17#include <__atomic/memory_order.h>
18#include <__atomic/support.h>
19#include <__config>
20#include <__cstddef/ptrdiff_t.h>
21#include <__memory/addressof.h>
22#include <__type_traits/enable_if.h>
23#include <__type_traits/is_floating_point.h>
24#include <__type_traits/is_function.h>
25#include <__type_traits/is_integral.h>
26#include <__type_traits/is_nothrow_constructible.h>
27#include <__type_traits/is_same.h>
28#include <__type_traits/is_trivially_copyable.h>
29#include <__type_traits/remove_pointer.h>
30#include <__utility/forward.h>
31#include <cstring>
32
33#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
34# pragma GCC system_header
35#endif
36
37_LIBCPP_BEGIN_NAMESPACE_STD
38
39template <class _Tp, bool = is_integral<_Tp>::value && !is_same<_Tp, bool>::value>
40struct __atomic_base // false
41{
42 mutable __cxx_atomic_impl<_Tp> __a_;
43
44 using value_type = _Tp;
45
46#if _LIBCPP_STD_VER >= 17
47 static constexpr bool is_always_lock_free = __libcpp_is_always_lock_free<__cxx_atomic_impl<_Tp> >::__value;
48#endif
49
50 [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool is_lock_free() const volatile _NOEXCEPT {
51 return __cxx_atomic_is_lock_free(sizeof(__cxx_atomic_impl<_Tp>));
52 }
53 [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool is_lock_free() const _NOEXCEPT {
54 return static_cast<__atomic_base const volatile*>(this)->is_lock_free();
55 }
56 _LIBCPP_HIDE_FROM_ABI void store(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
57 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) {
58 std::__cxx_atomic_store(std::addressof(__a_), __d, __m);
59 }
60 _LIBCPP_HIDE_FROM_ABI void store(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT
61 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) {
62 std::__cxx_atomic_store(std::addressof(__a_), __d, __m);
63 }
64 [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _Tp load(memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT
65 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
66 return std::__cxx_atomic_load(std::addressof(__a_), __m);
67 }
68 [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _Tp load(memory_order __m = memory_order_seq_cst) const _NOEXCEPT
69 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
70 return std::__cxx_atomic_load(std::addressof(__a_), __m);
71 }
72 _LIBCPP_HIDE_FROM_ABI operator _Tp() const volatile _NOEXCEPT { return load(); }
73 _LIBCPP_HIDE_FROM_ABI operator _Tp() const _NOEXCEPT { return load(); }
74 _LIBCPP_HIDE_FROM_ABI _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
75 return std::__cxx_atomic_exchange(std::addressof(__a_), __d, __m);
76 }
77 _LIBCPP_HIDE_FROM_ABI _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
78 return std::__cxx_atomic_exchange(std::addressof(__a_), __d, __m);
79 }
80 _LIBCPP_HIDE_FROM_ABI bool
81 compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) volatile _NOEXCEPT
82 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {
83 return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __s, __f);
84 }
85 _LIBCPP_HIDE_FROM_ABI bool compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) _NOEXCEPT
86 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {
87 return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __s, __f);
88 }
89 _LIBCPP_HIDE_FROM_ABI bool
90 compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) volatile _NOEXCEPT
91 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {
92 return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __s, __f);
93 }
94 _LIBCPP_HIDE_FROM_ABI bool compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) _NOEXCEPT
95 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {
96 return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __s, __f);
97 }
98 _LIBCPP_HIDE_FROM_ABI bool
99 compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
100 return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __m, __m);
101 }
102 _LIBCPP_HIDE_FROM_ABI bool
103 compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
104 return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __m, __m);
105 }
106 _LIBCPP_HIDE_FROM_ABI bool
107 compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
108 return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __m, __m);
109 }
110 _LIBCPP_HIDE_FROM_ABI bool
111 compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
112 return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __m, __m);
113 }
114
115#if _LIBCPP_STD_VER >= 20
116 _LIBCPP_HIDE_FROM_ABI void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT {
117 std::__atomic_wait(*this, __v, __m);
118 }
119 _LIBCPP_HIDE_FROM_ABI void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT {
120 std::__atomic_wait(*this, __v, __m);
121 }
122 _LIBCPP_HIDE_FROM_ABI void notify_one() volatile _NOEXCEPT { std::__atomic_notify_one(*this); }
123 _LIBCPP_HIDE_FROM_ABI void notify_one() _NOEXCEPT { std::__atomic_notify_one(*this); }
124 _LIBCPP_HIDE_FROM_ABI void notify_all() volatile _NOEXCEPT { std::__atomic_notify_all(*this); }
125 _LIBCPP_HIDE_FROM_ABI void notify_all() _NOEXCEPT { std::__atomic_notify_all(*this); }
126#endif // _LIBCPP_STD_VER >= 20
127
128#if _LIBCPP_STD_VER >= 20
129 _LIBCPP_HIDE_FROM_ABI constexpr __atomic_base() noexcept(is_nothrow_default_constructible_v<_Tp>) : __a_(_Tp()) {}
130#else
131 _LIBCPP_HIDE_FROM_ABI __atomic_base() _NOEXCEPT = default;
132#endif
133
134 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __atomic_base(_Tp __d) _NOEXCEPT : __a_(__d) {}
135
136 __atomic_base(const __atomic_base&) = delete;
137};
138
139// atomic<Integral>
140
141template <class _Tp>
142struct __atomic_base<_Tp, true> : public __atomic_base<_Tp, false> {
143 using __base _LIBCPP_NODEBUG = __atomic_base<_Tp, false>;
144
145 using difference_type = typename __base::value_type;
146
147 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __atomic_base() _NOEXCEPT = default;
148
149 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __atomic_base(_Tp __d) _NOEXCEPT : __base(__d) {}
150
151 _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
152 return std::__cxx_atomic_fetch_add(std::addressof(this->__a_), __op, __m);
153 }
154 _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
155 return std::__cxx_atomic_fetch_add(std::addressof(this->__a_), __op, __m);
156 }
157 _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
158 return std::__cxx_atomic_fetch_sub(std::addressof(this->__a_), __op, __m);
159 }
160 _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
161 return std::__cxx_atomic_fetch_sub(std::addressof(this->__a_), __op, __m);
162 }
163 _LIBCPP_HIDE_FROM_ABI _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
164 return std::__cxx_atomic_fetch_and(std::addressof(this->__a_), __op, __m);
165 }
166 _LIBCPP_HIDE_FROM_ABI _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
167 return std::__cxx_atomic_fetch_and(std::addressof(this->__a_), __op, __m);
168 }
169 _LIBCPP_HIDE_FROM_ABI _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
170 return std::__cxx_atomic_fetch_or(std::addressof(this->__a_), __op, __m);
171 }
172 _LIBCPP_HIDE_FROM_ABI _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
173 return std::__cxx_atomic_fetch_or(std::addressof(this->__a_), __op, __m);
174 }
175 _LIBCPP_HIDE_FROM_ABI _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
176 return std::__cxx_atomic_fetch_xor(std::addressof(this->__a_), __op, __m);
177 }
178 _LIBCPP_HIDE_FROM_ABI _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
179 return std::__cxx_atomic_fetch_xor(std::addressof(this->__a_), __op, __m);
180 }
181
182 _LIBCPP_HIDE_FROM_ABI _Tp operator++(int) volatile _NOEXCEPT { return fetch_add(_Tp(1)); }
183 _LIBCPP_HIDE_FROM_ABI _Tp operator++(int) _NOEXCEPT { return fetch_add(_Tp(1)); }
184 _LIBCPP_HIDE_FROM_ABI _Tp operator--(int) volatile _NOEXCEPT { return fetch_sub(_Tp(1)); }
185 _LIBCPP_HIDE_FROM_ABI _Tp operator--(int) _NOEXCEPT { return fetch_sub(_Tp(1)); }
186 _LIBCPP_HIDE_FROM_ABI _Tp operator++() volatile _NOEXCEPT { return fetch_add(_Tp(1)) + _Tp(1); }
187 _LIBCPP_HIDE_FROM_ABI _Tp operator++() _NOEXCEPT { return fetch_add(_Tp(1)) + _Tp(1); }
188 _LIBCPP_HIDE_FROM_ABI _Tp operator--() volatile _NOEXCEPT { return fetch_sub(_Tp(1)) - _Tp(1); }
189 _LIBCPP_HIDE_FROM_ABI _Tp operator--() _NOEXCEPT { return fetch_sub(_Tp(1)) - _Tp(1); }
190 _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __op) volatile _NOEXCEPT { return fetch_add(__op) + __op; }
191 _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __op) _NOEXCEPT { return fetch_add(__op) + __op; }
192 _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __op) volatile _NOEXCEPT { return fetch_sub(__op) - __op; }
193 _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __op) _NOEXCEPT { return fetch_sub(__op) - __op; }
194 _LIBCPP_HIDE_FROM_ABI _Tp operator&=(_Tp __op) volatile _NOEXCEPT { return fetch_and(__op) & __op; }
195 _LIBCPP_HIDE_FROM_ABI _Tp operator&=(_Tp __op) _NOEXCEPT { return fetch_and(__op) & __op; }
196 _LIBCPP_HIDE_FROM_ABI _Tp operator|=(_Tp __op) volatile _NOEXCEPT { return fetch_or(__op) | __op; }
197 _LIBCPP_HIDE_FROM_ABI _Tp operator|=(_Tp __op) _NOEXCEPT { return fetch_or(__op) | __op; }
198 _LIBCPP_HIDE_FROM_ABI _Tp operator^=(_Tp __op) volatile _NOEXCEPT { return fetch_xor(__op) ^ __op; }
199 _LIBCPP_HIDE_FROM_ABI _Tp operator^=(_Tp __op) _NOEXCEPT { return fetch_xor(__op) ^ __op; }
200};
201
202#if _LIBCPP_STD_VER >= 20
203// Here we need _IsIntegral because the default template argument is not enough
204// e.g __atomic_base<int> is __atomic_base<int, true>, which inherits from
205// __atomic_base<int, false> and the caller of the wait function is
206// __atomic_base<int, false>. So specializing __atomic_base<_Tp> does not work
207template <class _Tp, bool _IsIntegral>
208struct __atomic_waitable_traits<__atomic_base<_Tp, _IsIntegral> > {
209 using __value_type _LIBCPP_NODEBUG = _Tp;
210
211 static _LIBCPP_HIDE_FROM_ABI _Tp __atomic_load(const __atomic_base<_Tp, _IsIntegral>& __a, memory_order __order) {
212 return __a.load(__order);
213 }
214
215 static _LIBCPP_HIDE_FROM_ABI _Tp
216 __atomic_load(const volatile __atomic_base<_Tp, _IsIntegral>& __this, memory_order __order) {
217 return __this.load(__order);
218 }
219
220 static _LIBCPP_HIDE_FROM_ABI const __cxx_atomic_impl<_Tp>*
221 __atomic_contention_address(const __atomic_base<_Tp, _IsIntegral>& __a) {
222 return std::addressof(__a.__a_);
223 }
224
225 static _LIBCPP_HIDE_FROM_ABI const volatile __cxx_atomic_impl<_Tp>*
226 __atomic_contention_address(const volatile __atomic_base<_Tp, _IsIntegral>& __this) {
227 return std::addressof(__this.__a_);
228 }
229};
230
231#endif // _LIBCPP_STD_VER >= 20
232
233template <typename _Tp>
234struct __check_atomic_mandates {
235 using type _LIBCPP_NODEBUG = _Tp;
236 static_assert(is_trivially_copyable<_Tp>::value, "std::atomic<T> requires that 'T' be a trivially copyable type");
237};
238
239template <class _Tp>
240struct atomic : public __atomic_base<typename __check_atomic_mandates<_Tp>::type> {
241 using __base _LIBCPP_NODEBUG = __atomic_base<_Tp>;
242
243#if _LIBCPP_STD_VER >= 20
244 _LIBCPP_HIDE_FROM_ABI atomic() = default;
245#else
246 _LIBCPP_HIDE_FROM_ABI atomic() _NOEXCEPT = default;
247#endif
248
249 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR atomic(_Tp __d) _NOEXCEPT : __base(__d) {}
250
251 _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __d) volatile _NOEXCEPT {
252 __base::store(__d);
253 return __d;
254 }
255 _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __d) _NOEXCEPT {
256 __base::store(__d);
257 return __d;
258 }
259
260 atomic& operator=(const atomic&) = delete;
261 atomic& operator=(const atomic&) volatile = delete;
262};
263
264// atomic<T*>
265
266template <class _Tp>
267struct atomic<_Tp*> : public __atomic_base<_Tp*> {
268 using __base _LIBCPP_NODEBUG = __atomic_base<_Tp*>;
269
270 using difference_type = ptrdiff_t;
271
272 _LIBCPP_HIDE_FROM_ABI atomic() _NOEXCEPT = default;
273
274 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR atomic(_Tp* __d) _NOEXCEPT : __base(__d) {}
275
276 _LIBCPP_HIDE_FROM_ABI _Tp* operator=(_Tp* __d) volatile _NOEXCEPT {
277 __base::store(__d);
278 return __d;
279 }
280 _LIBCPP_HIDE_FROM_ABI _Tp* operator=(_Tp* __d) _NOEXCEPT {
281 __base::store(__d);
282 return __d;
283 }
284
285 _LIBCPP_HIDE_FROM_ABI _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
286 // __atomic_fetch_add accepts function pointers, guard against them.
287 static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed");
288 return std::__cxx_atomic_fetch_add(std::addressof(this->__a_), __op, __m);
289 }
290
291 _LIBCPP_HIDE_FROM_ABI _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
292 // __atomic_fetch_add accepts function pointers, guard against them.
293 static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed");
294 return std::__cxx_atomic_fetch_add(std::addressof(this->__a_), __op, __m);
295 }
296
297 _LIBCPP_HIDE_FROM_ABI _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
298 // __atomic_fetch_add accepts function pointers, guard against them.
299 static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed");
300 return std::__cxx_atomic_fetch_sub(std::addressof(this->__a_), __op, __m);
301 }
302
303 _LIBCPP_HIDE_FROM_ABI _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
304 // __atomic_fetch_add accepts function pointers, guard against them.
305 static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed");
306 return std::__cxx_atomic_fetch_sub(std::addressof(this->__a_), __op, __m);
307 }
308
309 _LIBCPP_HIDE_FROM_ABI _Tp* operator++(int) volatile _NOEXCEPT { return fetch_add(1); }
310 _LIBCPP_HIDE_FROM_ABI _Tp* operator++(int) _NOEXCEPT { return fetch_add(1); }
311 _LIBCPP_HIDE_FROM_ABI _Tp* operator--(int) volatile _NOEXCEPT { return fetch_sub(1); }
312 _LIBCPP_HIDE_FROM_ABI _Tp* operator--(int) _NOEXCEPT { return fetch_sub(1); }
313 _LIBCPP_HIDE_FROM_ABI _Tp* operator++() volatile _NOEXCEPT { return fetch_add(1) + 1; }
314 _LIBCPP_HIDE_FROM_ABI _Tp* operator++() _NOEXCEPT { return fetch_add(1) + 1; }
315 _LIBCPP_HIDE_FROM_ABI _Tp* operator--() volatile _NOEXCEPT { return fetch_sub(1) - 1; }
316 _LIBCPP_HIDE_FROM_ABI _Tp* operator--() _NOEXCEPT { return fetch_sub(1) - 1; }
317 _LIBCPP_HIDE_FROM_ABI _Tp* operator+=(ptrdiff_t __op) volatile _NOEXCEPT { return fetch_add(__op) + __op; }
318 _LIBCPP_HIDE_FROM_ABI _Tp* operator+=(ptrdiff_t __op) _NOEXCEPT { return fetch_add(__op) + __op; }
319 _LIBCPP_HIDE_FROM_ABI _Tp* operator-=(ptrdiff_t __op) volatile _NOEXCEPT { return fetch_sub(__op) - __op; }
320 _LIBCPP_HIDE_FROM_ABI _Tp* operator-=(ptrdiff_t __op) _NOEXCEPT { return fetch_sub(__op) - __op; }
321
322 atomic& operator=(const atomic&) = delete;
323 atomic& operator=(const atomic&) volatile = delete;
324};
325
326#if _LIBCPP_STD_VER >= 20
327template <class _Tp>
328struct __atomic_waitable_traits<atomic<_Tp> > : __atomic_waitable_traits<__atomic_base<_Tp> > {};
329
330template <class _Tp>
331 requires is_floating_point_v<_Tp>
332struct atomic<_Tp> : __atomic_base<_Tp> {
333private:
334 template <class _This, class _Operation, class _BuiltinOp>
335 _LIBCPP_HIDE_FROM_ABI static _Tp
336 __rmw_op(_This&& __self, _Tp __operand, memory_order __m, _Operation __operation, _BuiltinOp __builtin_op) {
337 if constexpr (std::__has_rmw_builtin<_Tp>()) {
338 return __builtin_op(std::addressof(std::forward<_This>(__self).__a_), __operand, __m);
339 } else {
340 _Tp __old = __self.load(memory_order_relaxed);
341 _Tp __new = __operation(__old, __operand);
342 while (!__self.compare_exchange_weak(__old, __new, __m, memory_order_relaxed)) {
343# ifdef _LIBCPP_COMPILER_CLANG_BASED
344 if constexpr (std::__is_fp80_long_double<_Tp>()) {
345 // https://llvm.org/PR47978
346 // clang bug: __old is not updated on failure for atomic<long double>::compare_exchange_weak
347 // Note __old = __self.load(memory_order_relaxed) will not work
348 std::__cxx_atomic_load_inplace(std::addressof(__self.__a_), std::addressof(__old), memory_order_relaxed);
349 }
350# endif
351 __new = __operation(__old, __operand);
352 }
353 return __old;
354 }
355 }
356
357 template <class _This>
358 _LIBCPP_HIDE_FROM_ABI static _Tp __fetch_add(_This&& __self, _Tp __operand, memory_order __m) {
359 auto __builtin_op = [](auto __a, auto __builtin_operand, auto __order) {
360 return std::__cxx_atomic_fetch_add(__a, __builtin_operand, __order);
361 };
362 auto __plus = [](auto __a, auto __b) { return __a + __b; };
363 return __rmw_op(std::forward<_This>(__self), __operand, __m, __plus, __builtin_op);
364 }
365
366 template <class _This>
367 _LIBCPP_HIDE_FROM_ABI static _Tp __fetch_sub(_This&& __self, _Tp __operand, memory_order __m) {
368 auto __builtin_op = [](auto __a, auto __builtin_operand, auto __order) {
369 return std::__cxx_atomic_fetch_sub(__a, __builtin_operand, __order);
370 };
371 auto __minus = [](auto __a, auto __b) { return __a - __b; };
372 return __rmw_op(std::forward<_This>(__self), __operand, __m, __minus, __builtin_op);
373 }
374
375public:
376 using __base _LIBCPP_NODEBUG = __atomic_base<_Tp>;
377 using value_type = _Tp;
378 using difference_type = value_type;
379
380 _LIBCPP_HIDE_FROM_ABI constexpr atomic() noexcept = default;
381 _LIBCPP_HIDE_FROM_ABI constexpr atomic(_Tp __d) noexcept : __base(__d) {}
382
383 atomic(const atomic&) = delete;
384 atomic& operator=(const atomic&) = delete;
385 atomic& operator=(const atomic&) volatile = delete;
386
387 _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __d) volatile noexcept
388 requires __base::is_always_lock_free
389 {
390 __base::store(__d);
391 return __d;
392 }
393 _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __d) noexcept {
394 __base::store(__d);
395 return __d;
396 }
397
398 _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) volatile noexcept
399 requires __base::is_always_lock_free
400 {
401 return __fetch_add(*this, __op, __m);
402 }
403
404 _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) noexcept {
405 return __fetch_add(*this, __op, __m);
406 }
407
408 _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) volatile noexcept
409 requires __base::is_always_lock_free
410 {
411 return __fetch_sub(*this, __op, __m);
412 }
413
414 _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) noexcept {
415 return __fetch_sub(*this, __op, __m);
416 }
417
418 _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __op) volatile noexcept
419 requires __base::is_always_lock_free
420 {
421 return fetch_add(__op) + __op;
422 }
423
424 _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __op) noexcept { return fetch_add(__op) + __op; }
425
426 _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __op) volatile noexcept
427 requires __base::is_always_lock_free
428 {
429 return fetch_sub(__op) - __op;
430 }
431
432 _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __op) noexcept { return fetch_sub(__op) - __op; }
433};
434
435#endif // _LIBCPP_STD_VER >= 20
436
437// atomic_is_lock_free
438
439template <class _Tp>
440[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool atomic_is_lock_free(const volatile atomic<_Tp>* __o) _NOEXCEPT {
441 return __o->is_lock_free();
442}
443
444template <class _Tp>
445[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool atomic_is_lock_free(const atomic<_Tp>* __o) _NOEXCEPT {
446 return __o->is_lock_free();
447}
448
449// atomic_init
450
451template <class _Tp>
452_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI void
453atomic_init(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
454 std::__cxx_atomic_init(std::addressof(__o->__a_), __d);
455}
456
457template <class _Tp>
458_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI void
459atomic_init(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
460 std::__cxx_atomic_init(std::addressof(__o->__a_), __d);
461}
462
463// atomic_store
464
465template <class _Tp>
466_LIBCPP_HIDE_FROM_ABI void atomic_store(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
467 __o->store(__d);
468}
469
470template <class _Tp>
471_LIBCPP_HIDE_FROM_ABI void atomic_store(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
472 __o->store(__d);
473}
474
475// atomic_store_explicit
476
477template <class _Tp>
478_LIBCPP_HIDE_FROM_ABI void
479atomic_store_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT
480 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) {
481 __o->store(__d, __m);
482}
483
484template <class _Tp>
485_LIBCPP_HIDE_FROM_ABI void
486atomic_store_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT
487 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) {
488 __o->store(__d, __m);
489}
490
491// atomic_load
492
493template <class _Tp>
494[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _Tp atomic_load(const volatile atomic<_Tp>* __o) _NOEXCEPT {
495 return __o->load();
496}
497
498template <class _Tp>
499[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _Tp atomic_load(const atomic<_Tp>* __o) _NOEXCEPT {
500 return __o->load();
501}
502
503// atomic_load_explicit
504
505template <class _Tp>
506[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _Tp
507atomic_load_explicit(const volatile atomic<_Tp>* __o, memory_order __m) _NOEXCEPT _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
508 return __o->load(__m);
509}
510
511template <class _Tp>
512[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _Tp atomic_load_explicit(const atomic<_Tp>* __o, memory_order __m) _NOEXCEPT
513 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
514 return __o->load(__m);
515}
516
517// atomic_exchange
518
519template <class _Tp>
520_LIBCPP_HIDE_FROM_ABI _Tp atomic_exchange(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
521 return __o->exchange(__d);
522}
523
524template <class _Tp>
525_LIBCPP_HIDE_FROM_ABI _Tp atomic_exchange(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
526 return __o->exchange(__d);
527}
528
529// atomic_exchange_explicit
530
531template <class _Tp>
532_LIBCPP_HIDE_FROM_ABI _Tp
533atomic_exchange_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT {
534 return __o->exchange(__d, __m);
535}
536
537template <class _Tp>
538_LIBCPP_HIDE_FROM_ABI _Tp
539atomic_exchange_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT {
540 return __o->exchange(__d, __m);
541}
542
543// atomic_compare_exchange_weak
544
545template <class _Tp>
546_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_weak(
547 volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
548 return __o->compare_exchange_weak(*__e, __d);
549}
550
551template <class _Tp>
552_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_weak(
553 atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
554 return __o->compare_exchange_weak(*__e, __d);
555}
556
557// atomic_compare_exchange_strong
558
559template <class _Tp>
560_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_strong(
561 volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
562 return __o->compare_exchange_strong(*__e, __d);
563}
564
565template <class _Tp>
566_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_strong(
567 atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
568 return __o->compare_exchange_strong(*__e, __d);
569}
570
571// atomic_compare_exchange_weak_explicit
572
573template <class _Tp>
574_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_weak_explicit(
575 volatile atomic<_Tp>* __o,
576 typename atomic<_Tp>::value_type* __e,
577 typename atomic<_Tp>::value_type __d,
578 memory_order __s,
579 memory_order __f) _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {
580 return __o->compare_exchange_weak(*__e, __d, __s, __f);
581}
582
583template <class _Tp>
584_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_weak_explicit(
585 atomic<_Tp>* __o,
586 typename atomic<_Tp>::value_type* __e,
587 typename atomic<_Tp>::value_type __d,
588 memory_order __s,
589 memory_order __f) _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {
590 return __o->compare_exchange_weak(*__e, __d, __s, __f);
591}
592
593// atomic_compare_exchange_strong_explicit
594
595template <class _Tp>
596_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_strong_explicit(
597 volatile atomic<_Tp>* __o,
598 typename atomic<_Tp>::value_type* __e,
599 typename atomic<_Tp>::value_type __d,
600 memory_order __s,
601 memory_order __f) _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {
602 return __o->compare_exchange_strong(*__e, __d, __s, __f);
603}
604
605template <class _Tp>
606_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_strong_explicit(
607 atomic<_Tp>* __o,
608 typename atomic<_Tp>::value_type* __e,
609 typename atomic<_Tp>::value_type __d,
610 memory_order __s,
611 memory_order __f) _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {
612 return __o->compare_exchange_strong(*__e, __d, __s, __f);
613}
614
615#if _LIBCPP_STD_VER >= 20
616
617// atomic_wait
618
619template <class _Tp>
620_LIBCPP_HIDE_FROM_ABI void
621atomic_wait(const volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __v) _NOEXCEPT {
622 return __o->wait(__v);
623}
624
625template <class _Tp>
626_LIBCPP_HIDE_FROM_ABI void atomic_wait(const atomic<_Tp>* __o, typename atomic<_Tp>::value_type __v) _NOEXCEPT {
627 return __o->wait(__v);
628}
629
630// atomic_wait_explicit
631
632template <class _Tp>
633_LIBCPP_HIDE_FROM_ABI void
634atomic_wait_explicit(const volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __v, memory_order __m) _NOEXCEPT
635 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
636 return __o->wait(__v, __m);
637}
638
639template <class _Tp>
640_LIBCPP_HIDE_FROM_ABI void
641atomic_wait_explicit(const atomic<_Tp>* __o, typename atomic<_Tp>::value_type __v, memory_order __m) _NOEXCEPT
642 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
643 return __o->wait(__v, __m);
644}
645
646// atomic_notify_one
647
648template <class _Tp>
649_LIBCPP_HIDE_FROM_ABI void atomic_notify_one(volatile atomic<_Tp>* __o) _NOEXCEPT {
650 __o->notify_one();
651}
652template <class _Tp>
653_LIBCPP_HIDE_FROM_ABI void atomic_notify_one(atomic<_Tp>* __o) _NOEXCEPT {
654 __o->notify_one();
655}
656
657// atomic_notify_all
658
659template <class _Tp>
660_LIBCPP_HIDE_FROM_ABI void atomic_notify_all(volatile atomic<_Tp>* __o) _NOEXCEPT {
661 __o->notify_all();
662}
663template <class _Tp>
664_LIBCPP_HIDE_FROM_ABI void atomic_notify_all(atomic<_Tp>* __o) _NOEXCEPT {
665 __o->notify_all();
666}
667
668#endif // _LIBCPP_STD_VER >= 20
669
670// atomic_fetch_add
671
672template <class _Tp>
673_LIBCPP_HIDE_FROM_ABI _Tp
674atomic_fetch_add(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT {
675 return __o->fetch_add(__op);
676}
677
678template <class _Tp>
679_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_add(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT {
680 return __o->fetch_add(__op);
681}
682
683// atomic_fetch_add_explicit
684
685template <class _Tp>
686_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_add_explicit(
687 volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT {
688 return __o->fetch_add(__op, __m);
689}
690
691template <class _Tp>
692_LIBCPP_HIDE_FROM_ABI _Tp
693atomic_fetch_add_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT {
694 return __o->fetch_add(__op, __m);
695}
696
697// atomic_fetch_sub
698
699template <class _Tp>
700_LIBCPP_HIDE_FROM_ABI _Tp
701atomic_fetch_sub(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT {
702 return __o->fetch_sub(__op);
703}
704
705template <class _Tp>
706_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_sub(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT {
707 return __o->fetch_sub(__op);
708}
709
710// atomic_fetch_sub_explicit
711
712template <class _Tp>
713_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_sub_explicit(
714 volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT {
715 return __o->fetch_sub(__op, __m);
716}
717
718template <class _Tp>
719_LIBCPP_HIDE_FROM_ABI _Tp
720atomic_fetch_sub_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT {
721 return __o->fetch_sub(__op, __m);
722}
723
724// atomic_fetch_and
725
726template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
727_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_and(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT {
728 return __o->fetch_and(__op);
729}
730
731template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
732_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_and(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT {
733 return __o->fetch_and(__op);
734}
735
736// atomic_fetch_and_explicit
737
738template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
739_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_and_explicit(
740 volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT {
741 return __o->fetch_and(__op, __m);
742}
743
744template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
745_LIBCPP_HIDE_FROM_ABI _Tp
746atomic_fetch_and_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT {
747 return __o->fetch_and(__op, __m);
748}
749
750// atomic_fetch_or
751
752template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
753_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_or(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT {
754 return __o->fetch_or(__op);
755}
756
757template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
758_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_or(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT {
759 return __o->fetch_or(__op);
760}
761
762// atomic_fetch_or_explicit
763
764template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
765_LIBCPP_HIDE_FROM_ABI _Tp
766atomic_fetch_or_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT {
767 return __o->fetch_or(__op, __m);
768}
769
770template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
771_LIBCPP_HIDE_FROM_ABI _Tp
772atomic_fetch_or_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT {
773 return __o->fetch_or(__op, __m);
774}
775
776// atomic_fetch_xor
777
778template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
779_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_xor(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT {
780 return __o->fetch_xor(__op);
781}
782
783template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
784_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_xor(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT {
785 return __o->fetch_xor(__op);
786}
787
788// atomic_fetch_xor_explicit
789
790template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
791_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_xor_explicit(
792 volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT {
793 return __o->fetch_xor(__op, __m);
794}
795
796template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
797_LIBCPP_HIDE_FROM_ABI _Tp
798atomic_fetch_xor_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT {
799 return __o->fetch_xor(__op, __m);
800}
801
802_LIBCPP_END_NAMESPACE_STD
803
804#endif // _LIBCPP___ATOMIC_ATOMIC_H
805