1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#ifndef _LIBCPP___ATOMIC_ATOMIC_H
10#define _LIBCPP___ATOMIC_ATOMIC_H
11
12#include <__atomic/atomic_sync.h>
13#include <__atomic/atomic_waitable_traits.h>
14#include <__atomic/check_memory_order.h>
15#include <__atomic/floating_point_helper.h>
16#include <__atomic/is_always_lock_free.h>
17#include <__atomic/memory_order.h>
18#include <__atomic/support.h>
19#include <__concepts/arithmetic.h>
20#include <__concepts/same_as.h>
21#include <__config>
22#include <__cstddef/ptrdiff_t.h>
23#include <__memory/addressof.h>
24#include <__type_traits/enable_if.h>
25#include <__type_traits/is_constructible.h>
26#include <__type_traits/is_floating_point.h>
27#include <__type_traits/is_function.h>
28#include <__type_traits/is_integral.h>
29#include <__type_traits/is_nothrow_constructible.h>
30#include <__type_traits/is_pointer.h>
31#include <__type_traits/is_same.h>
32#include <__type_traits/is_trivially_copyable.h>
33#include <__type_traits/remove_pointer.h>
34#include <__utility/forward.h>
35#include <cstring>
36
37#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
38# pragma GCC system_header
39#endif
40
41_LIBCPP_BEGIN_NAMESPACE_STD
42
43template <class _Tp, bool = is_integral<_Tp>::value && !is_same<_Tp, bool>::value>
44struct __atomic_base // false
45{
46 mutable __cxx_atomic_impl<_Tp> __a_;
47
48 using value_type = _Tp;
49
50#if _LIBCPP_STD_VER >= 17
51 static constexpr bool is_always_lock_free = __libcpp_is_always_lock_free<__cxx_atomic_impl<_Tp> >::__value;
52#endif
53
54 [[__nodiscard__]] bool is_lock_free() const volatile _NOEXCEPT {
55 return __cxx_atomic_is_lock_free(sizeof(__cxx_atomic_impl<_Tp>));
56 }
57 [[__nodiscard__]] bool is_lock_free() const _NOEXCEPT {
58 return static_cast<__atomic_base const volatile*>(this)->is_lock_free();
59 }
60 void store(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
61 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) {
62 std::__cxx_atomic_store(std::addressof(__a_), __d, __m);
63 }
64 void store(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) {
65 std::__cxx_atomic_store(std::addressof(__a_), __d, __m);
66 }
67 [[__nodiscard__]] _Tp load(memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT
68 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
69 return std::__cxx_atomic_load(std::addressof(__a_), __m);
70 }
71 [[__nodiscard__]] _Tp load(memory_order __m = memory_order_seq_cst) const _NOEXCEPT
72 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
73 return std::__cxx_atomic_load(std::addressof(__a_), __m);
74 }
75 operator _Tp() const volatile _NOEXCEPT { return load(); }
76 operator _Tp() const _NOEXCEPT { return load(); }
77 _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
78 return std::__cxx_atomic_exchange(std::addressof(__a_), __d, __m);
79 }
80 _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
81 return std::__cxx_atomic_exchange(std::addressof(__a_), __d, __m);
82 }
83 bool compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) volatile _NOEXCEPT
84 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {
85 return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __s, __f);
86 }
87 bool compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) _NOEXCEPT
88 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {
89 return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __s, __f);
90 }
91 bool compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) volatile _NOEXCEPT
92 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {
93 return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __s, __f);
94 }
95 bool compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) _NOEXCEPT
96 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {
97 return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __s, __f);
98 }
99 bool compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
100 return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __m, __m);
101 }
102 bool compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
103 return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __m, __m);
104 }
105 bool compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
106 return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __m, __m);
107 }
108 bool compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
109 return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __m, __m);
110 }
111
112#if _LIBCPP_STD_VER >= 20
113 void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const volatile noexcept {
114 std::__atomic_wait(*this, __v, __m);
115 }
116 void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const noexcept { std::__atomic_wait(*this, __v, __m); }
117 void notify_one() volatile noexcept { std::__atomic_notify_one(*this); }
118 void notify_one() noexcept { std::__atomic_notify_one(*this); }
119 void notify_all() volatile noexcept { std::__atomic_notify_all(*this); }
120 void notify_all() noexcept { std::__atomic_notify_all(*this); }
121#endif // _LIBCPP_STD_VER >= 20
122
123#if _LIBCPP_STD_VER >= 20
124 constexpr __atomic_base() noexcept(is_nothrow_default_constructible_v<_Tp>) : __a_(_Tp()) {}
125#else
126 __atomic_base() _NOEXCEPT = default;
127#endif
128
129 _LIBCPP_CONSTEXPR __atomic_base(_Tp __d) _NOEXCEPT : __a_(__d) {}
130
131 __atomic_base(const __atomic_base&) = delete;
132};
133
134// atomic<Integral>
135
136template <class _Tp>
137struct __atomic_base<_Tp, true> : public __atomic_base<_Tp, false> {
138 using __base _LIBCPP_NODEBUG = __atomic_base<_Tp, false>;
139
140 using difference_type = typename __base::value_type;
141
142 _LIBCPP_CONSTEXPR_SINCE_CXX20 __atomic_base() _NOEXCEPT = default;
143
144 _LIBCPP_CONSTEXPR __atomic_base(_Tp __d) _NOEXCEPT : __base(__d) {}
145
146 _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
147 return std::__cxx_atomic_fetch_add(std::addressof(this->__a_), __op, __m);
148 }
149 _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
150 return std::__cxx_atomic_fetch_add(std::addressof(this->__a_), __op, __m);
151 }
152 _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
153 return std::__cxx_atomic_fetch_sub(std::addressof(this->__a_), __op, __m);
154 }
155 _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
156 return std::__cxx_atomic_fetch_sub(std::addressof(this->__a_), __op, __m);
157 }
158 _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
159 return std::__cxx_atomic_fetch_and(std::addressof(this->__a_), __op, __m);
160 }
161 _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
162 return std::__cxx_atomic_fetch_and(std::addressof(this->__a_), __op, __m);
163 }
164 _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
165 return std::__cxx_atomic_fetch_or(std::addressof(this->__a_), __op, __m);
166 }
167 _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
168 return std::__cxx_atomic_fetch_or(std::addressof(this->__a_), __op, __m);
169 }
170 _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
171 return std::__cxx_atomic_fetch_xor(std::addressof(this->__a_), __op, __m);
172 }
173 _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
174 return std::__cxx_atomic_fetch_xor(std::addressof(this->__a_), __op, __m);
175 }
176#if _LIBCPP_STD_VER >= 26
177 _LIBCPP_HIDE_FROM_ABI _Tp fetch_max(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
178 return std::__cxx_atomic_fetch_max(std::addressof(this->__a_), __op, __m);
179 }
180 _LIBCPP_HIDE_FROM_ABI _Tp fetch_max(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
181 return std::__cxx_atomic_fetch_max(std::addressof(this->__a_), __op, __m);
182 }
183 _LIBCPP_HIDE_FROM_ABI _Tp fetch_min(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
184 return std::__cxx_atomic_fetch_min(std::addressof(this->__a_), __op, __m);
185 }
186 _LIBCPP_HIDE_FROM_ABI _Tp fetch_min(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
187 return std::__cxx_atomic_fetch_min(std::addressof(this->__a_), __op, __m);
188 }
189#endif
190
191 _Tp operator++(int) volatile _NOEXCEPT { return fetch_add(_Tp(1)); }
192 _Tp operator++(int) _NOEXCEPT { return fetch_add(_Tp(1)); }
193 _Tp operator--(int) volatile _NOEXCEPT { return fetch_sub(_Tp(1)); }
194 _Tp operator--(int) _NOEXCEPT { return fetch_sub(_Tp(1)); }
195 _Tp operator++() volatile _NOEXCEPT { return fetch_add(_Tp(1)) + _Tp(1); }
196 _Tp operator++() _NOEXCEPT { return fetch_add(_Tp(1)) + _Tp(1); }
197 _Tp operator--() volatile _NOEXCEPT { return fetch_sub(_Tp(1)) - _Tp(1); }
198 _Tp operator--() _NOEXCEPT { return fetch_sub(_Tp(1)) - _Tp(1); }
199 _Tp operator+=(_Tp __op) volatile _NOEXCEPT { return fetch_add(__op) + __op; }
200 _Tp operator+=(_Tp __op) _NOEXCEPT { return fetch_add(__op) + __op; }
201 _Tp operator-=(_Tp __op) volatile _NOEXCEPT { return fetch_sub(__op) - __op; }
202 _Tp operator-=(_Tp __op) _NOEXCEPT { return fetch_sub(__op) - __op; }
203 _Tp operator&=(_Tp __op) volatile _NOEXCEPT { return fetch_and(__op) & __op; }
204 _Tp operator&=(_Tp __op) _NOEXCEPT { return fetch_and(__op) & __op; }
205 _Tp operator|=(_Tp __op) volatile _NOEXCEPT { return fetch_or(__op) | __op; }
206 _Tp operator|=(_Tp __op) _NOEXCEPT { return fetch_or(__op) | __op; }
207 _Tp operator^=(_Tp __op) volatile _NOEXCEPT { return fetch_xor(__op) ^ __op; }
208 _Tp operator^=(_Tp __op) _NOEXCEPT { return fetch_xor(__op) ^ __op; }
209};
210
211#if _LIBCPP_STD_VER >= 17
212// Here we need _IsIntegral because the default template argument is not enough
213// e.g __atomic_base<int> is __atomic_base<int, true>, which inherits from
214// __atomic_base<int, false> and the caller of the wait function is
215// __atomic_base<int, false>. So specializing __atomic_base<_Tp> does not work
216template <class _Tp, bool _IsIntegral>
217struct __atomic_waitable_traits<__atomic_base<_Tp, _IsIntegral> > {
218 using __value_type _LIBCPP_NODEBUG = _Tp;
219
220 static _Tp __atomic_load(const __atomic_base<_Tp, _IsIntegral>& __a, memory_order __order) {
221 return __a.load(__order);
222 }
223
224 static _Tp __atomic_load(const volatile __atomic_base<_Tp, _IsIntegral>& __this, memory_order __order) {
225 return __this.load(__order);
226 }
227
228 static const __cxx_atomic_impl<_Tp>* __atomic_contention_address(const __atomic_base<_Tp, _IsIntegral>& __a) {
229 return std::addressof(__a.__a_);
230 }
231
232 static const volatile __cxx_atomic_impl<_Tp>*
233 __atomic_contention_address(const volatile __atomic_base<_Tp, _IsIntegral>& __this) {
234 return std::addressof(__this.__a_);
235 }
236};
237
238#endif // _LIBCPP_STD_VER >= 17
239
240template <typename _Tp>
241struct __check_atomic_mandates {
242 using type _LIBCPP_NODEBUG = _Tp;
243 static_assert(is_trivially_copyable<_Tp>::value, "std::atomic<T> requires that 'T' be a trivially copyable type");
244};
245
246template <class _Tp>
247struct atomic : public __atomic_base<typename __check_atomic_mandates<_Tp>::type> {
248 using __base _LIBCPP_NODEBUG = __atomic_base<_Tp>;
249
250#if _LIBCPP_STD_VER >= 20
251 atomic()
252 requires is_default_constructible_v<_Tp>
253 = default;
254#else
255 atomic() _NOEXCEPT = default;
256#endif
257
258 _LIBCPP_CONSTEXPR atomic(_Tp __d) _NOEXCEPT : __base(__d) {}
259
260 _Tp operator=(_Tp __d) volatile _NOEXCEPT {
261 __base::store(__d);
262 return __d;
263 }
264 _Tp operator=(_Tp __d) _NOEXCEPT {
265 __base::store(__d);
266 return __d;
267 }
268
269 atomic& operator=(const atomic&) = delete;
270 atomic& operator=(const atomic&) volatile = delete;
271};
272
273// atomic<T*>
274
275template <class _Tp>
276struct atomic<_Tp*> : public __atomic_base<_Tp*> {
277 using __base _LIBCPP_NODEBUG = __atomic_base<_Tp*>;
278
279 using difference_type = ptrdiff_t;
280
281 atomic() _NOEXCEPT = default;
282
283 _LIBCPP_CONSTEXPR atomic(_Tp* __d) _NOEXCEPT : __base(__d) {}
284
285 _Tp* operator=(_Tp* __d) volatile _NOEXCEPT {
286 __base::store(__d);
287 return __d;
288 }
289 _Tp* operator=(_Tp* __d) _NOEXCEPT {
290 __base::store(__d);
291 return __d;
292 }
293
294 _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
295 // __atomic_fetch_add accepts function pointers, guard against them.
296 static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed");
297 return std::__cxx_atomic_fetch_add(std::addressof(this->__a_), __op, __m);
298 }
299
300 _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
301 // __atomic_fetch_add accepts function pointers, guard against them.
302 static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed");
303 return std::__cxx_atomic_fetch_add(std::addressof(this->__a_), __op, __m);
304 }
305
306 _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
307 // __atomic_fetch_sub accepts function pointers, guard against them.
308 static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed");
309 return std::__cxx_atomic_fetch_sub(std::addressof(this->__a_), __op, __m);
310 }
311
312 _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
313 // __atomic_fetch_sub accepts function pointers, guard against them.
314 static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed");
315 return std::__cxx_atomic_fetch_sub(std::addressof(this->__a_), __op, __m);
316 }
317
318#if _LIBCPP_STD_VER >= 26
319 _Tp* fetch_max(_Tp* __op, memory_order __m = memory_order_seq_cst) volatile noexcept {
320 static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed");
321 return std::__cxx_atomic_fetch_max(std::addressof(this->__a_), __op, __m);
322 }
323
324 _Tp* fetch_max(_Tp* __op, memory_order __m = memory_order_seq_cst) noexcept {
325 static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed");
326 return std::__cxx_atomic_fetch_max(std::addressof(this->__a_), __op, __m);
327 }
328
329 _Tp* fetch_min(_Tp* __op, memory_order __m = memory_order_seq_cst) volatile noexcept {
330 static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed");
331 return std::__cxx_atomic_fetch_min(std::addressof(this->__a_), __op, __m);
332 }
333
334 _Tp* fetch_min(_Tp* __op, memory_order __m = memory_order_seq_cst) noexcept {
335 static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed");
336 return std::__cxx_atomic_fetch_min(std::addressof(this->__a_), __op, __m);
337 }
338#endif
339
340 _Tp* operator++(int) volatile _NOEXCEPT { return fetch_add(1); }
341 _Tp* operator++(int) _NOEXCEPT { return fetch_add(1); }
342 _Tp* operator--(int) volatile _NOEXCEPT { return fetch_sub(1); }
343 _Tp* operator--(int) _NOEXCEPT { return fetch_sub(1); }
344 _Tp* operator++() volatile _NOEXCEPT { return fetch_add(1) + 1; }
345 _Tp* operator++() _NOEXCEPT { return fetch_add(1) + 1; }
346 _Tp* operator--() volatile _NOEXCEPT { return fetch_sub(1) - 1; }
347 _Tp* operator--() _NOEXCEPT { return fetch_sub(1) - 1; }
348 _Tp* operator+=(ptrdiff_t __op) volatile _NOEXCEPT { return fetch_add(__op) + __op; }
349 _Tp* operator+=(ptrdiff_t __op) _NOEXCEPT { return fetch_add(__op) + __op; }
350 _Tp* operator-=(ptrdiff_t __op) volatile _NOEXCEPT { return fetch_sub(__op) - __op; }
351 _Tp* operator-=(ptrdiff_t __op) _NOEXCEPT { return fetch_sub(__op) - __op; }
352
353 atomic& operator=(const atomic&) = delete;
354 atomic& operator=(const atomic&) volatile = delete;
355};
356
357#if _LIBCPP_STD_VER >= 17
358template <class _Tp>
359struct __atomic_waitable_traits<atomic<_Tp> > : __atomic_waitable_traits<__atomic_base<_Tp> > {};
360#endif // _LIBCPP_STD_VER >= 17
361
362#if _LIBCPP_STD_VER >= 20
363template <class _Tp>
364 requires is_floating_point_v<_Tp>
365struct atomic<_Tp> : __atomic_base<_Tp> {
366private:
367 template <class _This, class _Operation, class _BuiltinOp>
368 static _Tp
369 __rmw_op(_This&& __self, _Tp __operand, memory_order __m, _Operation __operation, _BuiltinOp __builtin_op) {
370 if constexpr (std::__has_rmw_builtin<_Tp>()) {
371 return __builtin_op(std::addressof(std::forward<_This>(__self).__a_), __operand, __m);
372 } else {
373 _Tp __old = __self.load(memory_order_relaxed);
374 _Tp __new = __operation(__old, __operand);
375 while (!__self.compare_exchange_weak(__old, __new, __m, memory_order_relaxed)) {
376# ifdef _LIBCPP_COMPILER_CLANG_BASED
377 if constexpr (std::__is_fp80_long_double<_Tp>()) {
378 // https://llvm.org/PR47978
379 // clang bug: __old is not updated on failure for atomic<long double>::compare_exchange_weak
380 // Note __old = __self.load(memory_order_relaxed) will not work
381 std::__cxx_atomic_load_inplace(std::addressof(__self.__a_), std::addressof(__old), memory_order_relaxed);
382 }
383# endif
384 __new = __operation(__old, __operand);
385 }
386 return __old;
387 }
388 }
389
390 template <class _This>
391 static _Tp __fetch_add(_This&& __self, _Tp __operand, memory_order __m) {
392 auto __builtin_op = [](auto __a, auto __builtin_operand, auto __order) {
393 return std::__cxx_atomic_fetch_add(__a, __builtin_operand, __order);
394 };
395 auto __plus = [](auto __a, auto __b) { return __a + __b; };
396 return __rmw_op(std::forward<_This>(__self), __operand, __m, __plus, __builtin_op);
397 }
398
399 template <class _This>
400 static _Tp __fetch_sub(_This&& __self, _Tp __operand, memory_order __m) {
401 auto __builtin_op = [](auto __a, auto __builtin_operand, auto __order) {
402 return std::__cxx_atomic_fetch_sub(__a, __builtin_operand, __order);
403 };
404 auto __minus = [](auto __a, auto __b) { return __a - __b; };
405 return __rmw_op(std::forward<_This>(__self), __operand, __m, __minus, __builtin_op);
406 }
407
408public:
409 using __base _LIBCPP_NODEBUG = __atomic_base<_Tp>;
410 using value_type = _Tp;
411 using difference_type = value_type;
412
413 constexpr atomic() noexcept = default;
414 constexpr atomic(_Tp __d) noexcept : __base(__d) {}
415
416 atomic(const atomic&) = delete;
417 atomic& operator=(const atomic&) = delete;
418 atomic& operator=(const atomic&) volatile = delete;
419
420 _Tp operator=(_Tp __d) volatile noexcept
421 requires __base::is_always_lock_free
422 {
423 __base::store(__d);
424 return __d;
425 }
426 _Tp operator=(_Tp __d) noexcept {
427 __base::store(__d);
428 return __d;
429 }
430
431 _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) volatile noexcept
432 requires __base::is_always_lock_free
433 {
434 return __fetch_add(*this, __op, __m);
435 }
436
437 _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) noexcept { return __fetch_add(*this, __op, __m); }
438
439 _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) volatile noexcept
440 requires __base::is_always_lock_free
441 {
442 return __fetch_sub(*this, __op, __m);
443 }
444
445 _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) noexcept { return __fetch_sub(*this, __op, __m); }
446
447 _Tp operator+=(_Tp __op) volatile noexcept
448 requires __base::is_always_lock_free
449 {
450 return fetch_add(__op) + __op;
451 }
452
453 _Tp operator+=(_Tp __op) noexcept { return fetch_add(__op) + __op; }
454
455 _Tp operator-=(_Tp __op) volatile noexcept
456 requires __base::is_always_lock_free
457 {
458 return fetch_sub(__op) - __op;
459 }
460
461 _Tp operator-=(_Tp __op) noexcept { return fetch_sub(__op) - __op; }
462};
463
464#endif // _LIBCPP_STD_VER >= 20
465
466// atomic_is_lock_free
467
468template <class _Tp>
469[[__nodiscard__]] bool atomic_is_lock_free(const volatile atomic<_Tp>* __o) _NOEXCEPT {
470 return __o->is_lock_free();
471}
472
473template <class _Tp>
474[[__nodiscard__]] bool atomic_is_lock_free(const atomic<_Tp>* __o) _NOEXCEPT {
475 return __o->is_lock_free();
476}
477
478// atomic_init
479
480template <class _Tp>
481_LIBCPP_DEPRECATED_IN_CXX20 void
482atomic_init(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
483 std::__cxx_atomic_init(std::addressof(__o->__a_), __d);
484}
485
486template <class _Tp>
487_LIBCPP_DEPRECATED_IN_CXX20 void atomic_init(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
488 std::__cxx_atomic_init(std::addressof(__o->__a_), __d);
489}
490
491// atomic_store
492
493template <class _Tp>
494void atomic_store(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
495 __o->store(__d);
496}
497
498template <class _Tp>
499void atomic_store(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
500 __o->store(__d);
501}
502
503// atomic_store_explicit
504
505template <class _Tp>
506void atomic_store_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT
507 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) {
508 __o->store(__d, __m);
509}
510
511template <class _Tp>
512void atomic_store_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT
513 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) {
514 __o->store(__d, __m);
515}
516
517// atomic_load
518
519template <class _Tp>
520[[__nodiscard__]] _Tp atomic_load(const volatile atomic<_Tp>* __o) _NOEXCEPT {
521 return __o->load();
522}
523
524template <class _Tp>
525[[__nodiscard__]] _Tp atomic_load(const atomic<_Tp>* __o) _NOEXCEPT {
526 return __o->load();
527}
528
529// atomic_load_explicit
530
531template <class _Tp>
532[[__nodiscard__]] _Tp atomic_load_explicit(const volatile atomic<_Tp>* __o, memory_order __m) _NOEXCEPT
533 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
534 return __o->load(__m);
535}
536
537template <class _Tp>
538[[__nodiscard__]] _Tp atomic_load_explicit(const atomic<_Tp>* __o, memory_order __m) _NOEXCEPT
539 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
540 return __o->load(__m);
541}
542
543// atomic_exchange
544
545template <class _Tp>
546_Tp atomic_exchange(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
547 return __o->exchange(__d);
548}
549
550template <class _Tp>
551_Tp atomic_exchange(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
552 return __o->exchange(__d);
553}
554
555// atomic_exchange_explicit
556
557template <class _Tp>
558_Tp atomic_exchange_explicit(
559 volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT {
560 return __o->exchange(__d, __m);
561}
562
563template <class _Tp>
564_Tp atomic_exchange_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT {
565 return __o->exchange(__d, __m);
566}
567
568// atomic_compare_exchange_weak
569
570template <class _Tp>
571bool atomic_compare_exchange_weak(
572 volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
573 return __o->compare_exchange_weak(*__e, __d);
574}
575
576template <class _Tp>
577bool atomic_compare_exchange_weak(
578 atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
579 return __o->compare_exchange_weak(*__e, __d);
580}
581
582// atomic_compare_exchange_strong
583
584template <class _Tp>
585bool atomic_compare_exchange_strong(
586 volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
587 return __o->compare_exchange_strong(*__e, __d);
588}
589
590template <class _Tp>
591bool atomic_compare_exchange_strong(
592 atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
593 return __o->compare_exchange_strong(*__e, __d);
594}
595
596// atomic_compare_exchange_weak_explicit
597
598template <class _Tp>
599bool atomic_compare_exchange_weak_explicit(
600 volatile atomic<_Tp>* __o,
601 typename atomic<_Tp>::value_type* __e,
602 typename atomic<_Tp>::value_type __d,
603 memory_order __s,
604 memory_order __f) _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {
605 return __o->compare_exchange_weak(*__e, __d, __s, __f);
606}
607
608template <class _Tp>
609bool atomic_compare_exchange_weak_explicit(
610 atomic<_Tp>* __o,
611 typename atomic<_Tp>::value_type* __e,
612 typename atomic<_Tp>::value_type __d,
613 memory_order __s,
614 memory_order __f) _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {
615 return __o->compare_exchange_weak(*__e, __d, __s, __f);
616}
617
618// atomic_compare_exchange_strong_explicit
619
620template <class _Tp>
621bool atomic_compare_exchange_strong_explicit(
622 volatile atomic<_Tp>* __o,
623 typename atomic<_Tp>::value_type* __e,
624 typename atomic<_Tp>::value_type __d,
625 memory_order __s,
626 memory_order __f) _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {
627 return __o->compare_exchange_strong(*__e, __d, __s, __f);
628}
629
630template <class _Tp>
631bool atomic_compare_exchange_strong_explicit(
632 atomic<_Tp>* __o,
633 typename atomic<_Tp>::value_type* __e,
634 typename atomic<_Tp>::value_type __d,
635 memory_order __s,
636 memory_order __f) _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {
637 return __o->compare_exchange_strong(*__e, __d, __s, __f);
638}
639
640#if _LIBCPP_STD_VER >= 20
641
642// atomic_wait
643
644template <class _Tp>
645void atomic_wait(const volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __v) noexcept {
646 return __o->wait(__v);
647}
648
649template <class _Tp>
650void atomic_wait(const atomic<_Tp>* __o, typename atomic<_Tp>::value_type __v) noexcept {
651 return __o->wait(__v);
652}
653
654// atomic_wait_explicit
655
656template <class _Tp>
657void atomic_wait_explicit(
658 const volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __v, memory_order __m) noexcept
659 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
660 return __o->wait(__v, __m);
661}
662
663template <class _Tp>
664void atomic_wait_explicit(const atomic<_Tp>* __o, typename atomic<_Tp>::value_type __v, memory_order __m) noexcept
665 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
666 return __o->wait(__v, __m);
667}
668
669// atomic_notify_one
670
671template <class _Tp>
672void atomic_notify_one(volatile atomic<_Tp>* __o) noexcept {
673 __o->notify_one();
674}
675template <class _Tp>
676void atomic_notify_one(atomic<_Tp>* __o) noexcept {
677 __o->notify_one();
678}
679
680// atomic_notify_all
681
682template <class _Tp>
683void atomic_notify_all(volatile atomic<_Tp>* __o) noexcept {
684 __o->notify_all();
685}
686template <class _Tp>
687void atomic_notify_all(atomic<_Tp>* __o) noexcept {
688 __o->notify_all();
689}
690
691#endif // _LIBCPP_STD_VER >= 20
692
693// atomic_fetch_add
694
695template <class _Tp>
696_Tp atomic_fetch_add(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT {
697 return __o->fetch_add(__op);
698}
699
700template <class _Tp>
701_Tp atomic_fetch_add(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT {
702 return __o->fetch_add(__op);
703}
704
705// atomic_fetch_add_explicit
706
707template <class _Tp>
708_Tp atomic_fetch_add_explicit(
709 volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT {
710 return __o->fetch_add(__op, __m);
711}
712
713template <class _Tp>
714_Tp atomic_fetch_add_explicit(
715 atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT {
716 return __o->fetch_add(__op, __m);
717}
718
719// atomic_fetch_sub
720
721template <class _Tp>
722_Tp atomic_fetch_sub(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT {
723 return __o->fetch_sub(__op);
724}
725
726template <class _Tp>
727_Tp atomic_fetch_sub(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT {
728 return __o->fetch_sub(__op);
729}
730
731// atomic_fetch_sub_explicit
732
733template <class _Tp>
734_Tp atomic_fetch_sub_explicit(
735 volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT {
736 return __o->fetch_sub(__op, __m);
737}
738
739template <class _Tp>
740_Tp atomic_fetch_sub_explicit(
741 atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT {
742 return __o->fetch_sub(__op, __m);
743}
744
745// atomic_fetch_and
746
747template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
748_Tp atomic_fetch_and(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT {
749 return __o->fetch_and(__op);
750}
751
752template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
753_Tp atomic_fetch_and(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT {
754 return __o->fetch_and(__op);
755}
756
757// atomic_fetch_and_explicit
758
759template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
760_Tp atomic_fetch_and_explicit(
761 volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT {
762 return __o->fetch_and(__op, __m);
763}
764
765template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
766_Tp atomic_fetch_and_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT {
767 return __o->fetch_and(__op, __m);
768}
769
770// atomic_fetch_or
771
772template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
773_Tp atomic_fetch_or(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT {
774 return __o->fetch_or(__op);
775}
776
777template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
778_Tp atomic_fetch_or(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT {
779 return __o->fetch_or(__op);
780}
781
782// atomic_fetch_or_explicit
783
784template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
785_Tp atomic_fetch_or_explicit(
786 volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT {
787 return __o->fetch_or(__op, __m);
788}
789
790template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
791_Tp atomic_fetch_or_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT {
792 return __o->fetch_or(__op, __m);
793}
794
795// atomic_fetch_xor
796
797template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
798_Tp atomic_fetch_xor(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT {
799 return __o->fetch_xor(__op);
800}
801
802template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
803_Tp atomic_fetch_xor(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT {
804 return __o->fetch_xor(__op);
805}
806
807// atomic_fetch_xor_explicit
808
809template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
810_Tp atomic_fetch_xor_explicit(
811 volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT {
812 return __o->fetch_xor(__op, __m);
813}
814
815template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
816_Tp atomic_fetch_xor_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT {
817 return __o->fetch_xor(__op, __m);
818}
819
820#if _LIBCPP_STD_VER >= 26
821// atomic_fetch_max
822
823template <class _Tp>
824 requires((integral<_Tp> && !same_as<_Tp, bool>) || is_pointer_v<_Tp>)
825_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_max(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) noexcept {
826 return __o->fetch_max(__op);
827}
828
829template <class _Tp>
830 requires((integral<_Tp> && !same_as<_Tp, bool>) || is_pointer_v<_Tp>)
831_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_max(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) noexcept {
832 return __o->fetch_max(__op);
833}
834
835// atomic_fetch_max_explicit
836
837template <class _Tp>
838 requires((integral<_Tp> && !same_as<_Tp, bool>) || is_pointer_v<_Tp>)
839_LIBCPP_HIDE_FROM_ABI _Tp
840atomic_fetch_max_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) noexcept {
841 return __o->fetch_max(__op, __m);
842}
843
844template <class _Tp>
845 requires((integral<_Tp> && !same_as<_Tp, bool>) || is_pointer_v<_Tp>)
846_LIBCPP_HIDE_FROM_ABI _Tp
847atomic_fetch_max_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) noexcept {
848 return __o->fetch_max(__op, __m);
849}
850
851// atomic_fetch_min
852
853template <class _Tp>
854 requires((integral<_Tp> && !same_as<_Tp, bool>) || is_pointer_v<_Tp>)
855_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_min(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) noexcept {
856 return __o->fetch_min(__op);
857}
858
859template <class _Tp>
860 requires((integral<_Tp> && !same_as<_Tp, bool>) || is_pointer_v<_Tp>)
861_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_min(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) noexcept {
862 return __o->fetch_min(__op);
863}
864
865// atomic_fetch_min_explicit
866
867template <class _Tp>
868 requires((integral<_Tp> && !same_as<_Tp, bool>) || is_pointer_v<_Tp>)
869_LIBCPP_HIDE_FROM_ABI _Tp
870atomic_fetch_min_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) noexcept {
871 return __o->fetch_min(__op, __m);
872}
873
874template <class _Tp>
875 requires((integral<_Tp> && !same_as<_Tp, bool>) || is_pointer_v<_Tp>)
876_LIBCPP_HIDE_FROM_ABI _Tp
877atomic_fetch_min_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) noexcept {
878 return __o->fetch_min(__op, __m);
879}
880#endif
881
882_LIBCPP_END_NAMESPACE_STD
883
884#endif // _LIBCPP___ATOMIC_ATOMIC_H
885