1// -*- C++ -*-
2//===----------------------------------------------------------------------===//
3//
4// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5// See https://llvm.org/LICENSE.txt for license information.
6// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7//
8// Kokkos v. 4.0
9// Copyright (2022) National Technology & Engineering
10// Solutions of Sandia, LLC (NTESS).
11//
12// Under the terms of Contract DE-NA0003525 with NTESS,
13// the U.S. Government retains certain rights in this software.
14//
15//===---------------------------------------------------------------------===//
16
17#ifndef _LIBCPP___ATOMIC_ATOMIC_REF_H
18#define _LIBCPP___ATOMIC_ATOMIC_REF_H
19
20#include <__assert>
21#include <__atomic/atomic_sync.h>
22#include <__atomic/atomic_waitable_traits.h>
23#include <__atomic/check_memory_order.h>
24#include <__atomic/floating_point_helper.h>
25#include <__atomic/memory_order.h>
26#include <__atomic/to_gcc_order.h>
27#include <__concepts/arithmetic.h>
28#include <__concepts/same_as.h>
29#include <__config>
30#include <__cstddef/byte.h>
31#include <__cstddef/ptrdiff_t.h>
32#include <__memory/addressof.h>
33#include <__memory/is_sufficiently_aligned.h>
34#include <__type_traits/copy_cv.h>
35#include <__type_traits/has_unique_object_representation.h>
36#include <__type_traits/is_trivially_copyable.h>
37#include <cstring>
38
39#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
40# pragma GCC system_header
41#endif
42
43_LIBCPP_PUSH_MACROS
44#include <__undef_macros>
45
46_LIBCPP_BEGIN_NAMESPACE_STD
47
48#if _LIBCPP_STD_VER >= 20
49
50// These types are required to make __atomic_is_always_lock_free work across GCC and Clang.
51// The purpose of this trick is to make sure that we provide an object with the correct alignment
52// to __atomic_is_always_lock_free, since that answer depends on the alignment.
53template <size_t _Alignment>
54struct __alignment_checker_type {
55 alignas(_Alignment) char __data;
56};
57
58template <size_t _Alignment>
59struct __get_aligner_instance {
60 static constexpr __alignment_checker_type<_Alignment> __instance{};
61};
62
63template <class _Tp>
64struct __atomic_ref_base {
65private:
66 static _Tp* __clear_padding(_Tp& __val) noexcept {
67 _Tp* __ptr = std::addressof(__val);
68# if __has_builtin(__builtin_clear_padding)
69 __builtin_clear_padding(__ptr);
70# endif
71 return __ptr;
72 }
73
74 static bool __compare_exchange(
75 _Tp* __ptr, _Tp* __expected, _Tp* __desired, bool __is_weak, int __success, int __failure) noexcept {
76 if constexpr (
77# if __has_builtin(__builtin_clear_padding)
78 has_unique_object_representations_v<_Tp> || floating_point<_Tp>
79# else
80 true // NOLINT(readability-simplify-boolean-expr)
81# endif
82 ) {
83 return __atomic_compare_exchange(__ptr, __expected, __desired, __is_weak, __success, __failure);
84 } else { // _Tp has padding bits and __builtin_clear_padding is available
85 __clear_padding(val&: *__desired);
86 _Tp __copy = *__expected;
87 __clear_padding(val&: __copy);
88 // The algorithm we use here is basically to perform `__atomic_compare_exchange` on the
89 // values until it has either succeeded, or failed because the value representation of the
90 // objects involved was different. This is why we loop around __atomic_compare_exchange:
91 // we basically loop until its failure is caused by the value representation of the objects
92 // being different, not only their object representation.
93 while (true) {
94 _Tp __prev = __copy;
95 if (__atomic_compare_exchange(__ptr, std::addressof(__copy), __desired, __is_weak, __success, __failure)) {
96 return true;
97 }
98 _Tp __curr = __copy;
99 if (std::memcmp(s1: __clear_padding(val&: __prev), s2: __clear_padding(val&: __curr), n: sizeof(_Tp)) != 0) {
100 // Value representation without padding bits do not compare equal ->
101 // write the current content of *ptr into *expected
102 std::memcpy(dest: __expected, src: std::addressof(__copy), n: sizeof(_Tp));
103 return false;
104 }
105 }
106 }
107 }
108
109 friend struct __atomic_waitable_traits<__atomic_ref_base<_Tp>>;
110
111 // require types that are 1, 2, 4, 8, or 16 bytes in length to be aligned to at least their size to be potentially
112 // used lock-free
113 static constexpr size_t __min_alignment = (sizeof(_Tp) & (sizeof(_Tp) - 1)) || (sizeof(_Tp) > 16) ? 0 : sizeof(_Tp);
114
115public:
116 using value_type = _Tp;
117
118 static constexpr size_t required_alignment = alignof(_Tp) > __min_alignment ? alignof(_Tp) : __min_alignment;
119
120 // The __atomic_always_lock_free builtin takes into account the alignment of the pointer if provided,
121 // so we create a fake pointer with a suitable alignment when querying it. Note that we are guaranteed
122 // that the pointer is going to be aligned properly at runtime because that is a (checked) precondition
123 // of atomic_ref's constructor.
124 static constexpr bool is_always_lock_free =
125 __atomic_always_lock_free(sizeof(_Tp), std::addressof(__get_aligner_instance<required_alignment>::__instance));
126
127 [[nodiscard]] bool is_lock_free() const noexcept { return __atomic_is_lock_free(sizeof(_Tp), __ptr_); }
128
129 void store(_Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept
130 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__order) {
131 _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
132 __order == memory_order::relaxed || __order == memory_order::release || __order == memory_order::seq_cst,
133 "atomic_ref: memory order argument to atomic store operation is invalid");
134 __atomic_store(__ptr_, __clear_padding(val&: __desired), std::__to_gcc_order(__order));
135 }
136
137 _Tp operator=(_Tp __desired) const noexcept {
138 store(__desired);
139 return __desired;
140 }
141
142 [[nodiscard]] _Tp load(memory_order __order = memory_order::seq_cst) const noexcept
143 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__order) {
144 _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
145 __order == memory_order::relaxed || __order == memory_order::consume || __order == memory_order::acquire ||
146 __order == memory_order::seq_cst,
147 "atomic_ref: memory order argument to atomic load operation is invalid");
148 alignas(_Tp) byte __mem[sizeof(_Tp)];
149 auto* __ret = reinterpret_cast<_Tp*>(__mem);
150 __atomic_load(__ptr_, __ret, std::__to_gcc_order(__order));
151 return *__ret;
152 }
153
154 operator _Tp() const noexcept { return load(); }
155
156 _Tp exchange(_Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept {
157 alignas(_Tp) byte __mem[sizeof(_Tp)];
158 auto* __ret = reinterpret_cast<_Tp*>(__mem);
159 __atomic_exchange(__ptr_, __clear_padding(val&: __desired), __ret, std::__to_gcc_order(__order));
160 return *__ret;
161 }
162
163 bool
164 compare_exchange_weak(_Tp& __expected, _Tp __desired, memory_order __success, memory_order __failure) const noexcept
165 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__success, __failure) {
166 _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
167 __failure == memory_order::relaxed || __failure == memory_order::consume ||
168 __failure == memory_order::acquire || __failure == memory_order::seq_cst,
169 "atomic_ref: failure memory order argument to weak atomic compare-and-exchange operation is invalid");
170 return __compare_exchange(
171 ptr: __ptr_,
172 expected: std::addressof(__expected),
173 desired: std::addressof(__desired),
174 is_weak: true,
175 success: std::__to_gcc_order(order: __success),
176 failure: std::__to_gcc_order(order: __failure));
177 }
178 bool
179 compare_exchange_strong(_Tp& __expected, _Tp __desired, memory_order __success, memory_order __failure) const noexcept
180 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__success, __failure) {
181 _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
182 __failure == memory_order::relaxed || __failure == memory_order::consume ||
183 __failure == memory_order::acquire || __failure == memory_order::seq_cst,
184 "atomic_ref: failure memory order argument to strong atomic compare-and-exchange operation is invalid");
185 return __compare_exchange(
186 ptr: __ptr_,
187 expected: std::addressof(__expected),
188 desired: std::addressof(__desired),
189 is_weak: false,
190 success: std::__to_gcc_order(order: __success),
191 failure: std::__to_gcc_order(order: __failure));
192 }
193
194 bool
195 compare_exchange_weak(_Tp& __expected, _Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept {
196 return __compare_exchange(
197 ptr: __ptr_,
198 expected: std::addressof(__expected),
199 desired: std::addressof(__desired),
200 is_weak: true,
201 success: std::__to_gcc_order(__order),
202 failure: std::__to_gcc_failure_order(__order));
203 }
204 bool
205 compare_exchange_strong(_Tp& __expected, _Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept {
206 return __compare_exchange(
207 ptr: __ptr_,
208 expected: std::addressof(__expected),
209 desired: std::addressof(__desired),
210 is_weak: false,
211 success: std::__to_gcc_order(__order),
212 failure: std::__to_gcc_failure_order(__order));
213 }
214
215 void wait(_Tp __old, memory_order __order = memory_order::seq_cst) const noexcept
216 _LIBCPP_CHECK_WAIT_MEMORY_ORDER(__order) {
217 _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
218 __order == memory_order::relaxed || __order == memory_order::consume || __order == memory_order::acquire ||
219 __order == memory_order::seq_cst,
220 "atomic_ref: memory order argument to atomic wait operation is invalid");
221 std::__atomic_wait(*this, __old, __order);
222 }
223 void notify_one() const noexcept { std::__atomic_notify_one(*this); }
224 void notify_all() const noexcept { std::__atomic_notify_all(*this); }
225# if _LIBCPP_STD_VER >= 26
226 [[nodiscard]] constexpr __copy_cv_t<_Tp, void>* address() const noexcept { return __ptr_; }
227# endif
228
229protected:
230 using _Aligned_Tp [[__gnu__::__aligned__(required_alignment), __gnu__::__nodebug__]] = _Tp;
231 _Aligned_Tp* __ptr_;
232
233 __atomic_ref_base(_Tp& __obj) : __ptr_(std::addressof(__obj)) {}
234};
235
236template <class _Tp>
237struct __atomic_waitable_traits<__atomic_ref_base<_Tp>> {
238 using __value_type _LIBCPP_NODEBUG = _Tp;
239
240 static _Tp __atomic_load(const __atomic_ref_base<_Tp>& __a, memory_order __order) { return __a.load(__order); }
241 static const _Tp* __atomic_contention_address(const __atomic_ref_base<_Tp>& __a) { return __a.__ptr_; }
242};
243
244template <class _Tp>
245struct atomic_ref : public __atomic_ref_base<_Tp> {
246 static_assert(is_trivially_copyable_v<_Tp>, "std::atomic_ref<T> requires that 'T' be a trivially copyable type");
247
248 using __base _LIBCPP_NODEBUG = __atomic_ref_base<_Tp>;
249
250 explicit atomic_ref(_Tp& __obj) : __base(__obj) {
251 _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
252 std::__is_sufficiently_aligned<__base::required_alignment>(std::addressof(__obj)),
253 "atomic_ref ctor: referenced object must be aligned to required_alignment");
254 }
255
256 explicit atomic_ref(_Tp&&) = delete;
257
258 atomic_ref(const atomic_ref&) noexcept = default;
259
260 _Tp operator=(_Tp __desired) const noexcept { return __base::operator=(__desired); }
261
262 atomic_ref& operator=(const atomic_ref&) = delete;
263};
264
265template <class _Tp>
266 requires(std::integral<_Tp> && !std::same_as<bool, _Tp>)
267struct atomic_ref<_Tp> : public __atomic_ref_base<_Tp> {
268 using __base _LIBCPP_NODEBUG = __atomic_ref_base<_Tp>;
269
270 using difference_type = __base::value_type;
271
272 explicit atomic_ref(_Tp& __obj) : __base(__obj) {
273 _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
274 std::__is_sufficiently_aligned<__base::required_alignment>(std::addressof(__obj)),
275 "atomic_ref ctor: referenced object must be aligned to required_alignment");
276 }
277
278 explicit atomic_ref(_Tp&&) = delete;
279
280 atomic_ref(const atomic_ref&) noexcept = default;
281
282 _Tp operator=(_Tp __desired) const noexcept { return __base::operator=(__desired); }
283
284 atomic_ref& operator=(const atomic_ref&) = delete;
285
286 _Tp fetch_add(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
287 return __atomic_fetch_add(this->__ptr_, __arg, std::__to_gcc_order(__order));
288 }
289 _Tp fetch_sub(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
290 return __atomic_fetch_sub(this->__ptr_, __arg, std::__to_gcc_order(__order));
291 }
292 _Tp fetch_and(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
293 return __atomic_fetch_and(this->__ptr_, __arg, std::__to_gcc_order(__order));
294 }
295 _Tp fetch_or(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
296 return __atomic_fetch_or(this->__ptr_, __arg, std::__to_gcc_order(__order));
297 }
298 _Tp fetch_xor(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
299 return __atomic_fetch_xor(this->__ptr_, __arg, std::__to_gcc_order(__order));
300 }
301# if _LIBCPP_STD_VER >= 26
302 _Tp fetch_max(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
303# if __has_builtin(__atomic_fetch_max)
304 return __atomic_fetch_max(this->__ptr_, __arg, std::__to_gcc_order(__order));
305# else
306 _Tp __old = this->load(memory_order_relaxed);
307 _Tp __new;
308 do {
309 __new = __old > __arg ? __old : __arg;
310 } while (!this->compare_exchange_weak(__old, __new, __order, memory_order_relaxed));
311 return __old;
312# endif
313 }
314 _Tp fetch_min(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
315# if __has_builtin(__atomic_fetch_min)
316 return __atomic_fetch_min(this->__ptr_, __arg, std::__to_gcc_order(__order));
317# else
318 _Tp __old = this->load(memory_order_relaxed);
319 _Tp __new;
320 do {
321 __new = __old < __arg ? __old : __arg;
322 } while (!this->compare_exchange_weak(__old, __new, __order, memory_order_relaxed));
323 return __old;
324# endif
325 }
326# endif
327
328 _Tp operator++(int) const noexcept { return fetch_add(arg: _Tp(1)); }
329 _Tp operator--(int) const noexcept { return fetch_sub(arg: _Tp(1)); }
330 _Tp operator++() const noexcept { return fetch_add(arg: _Tp(1)) + _Tp(1); }
331 _Tp operator--() const noexcept { return fetch_sub(arg: _Tp(1)) - _Tp(1); }
332 _Tp operator+=(_Tp __arg) const noexcept { return fetch_add(__arg) + __arg; }
333 _Tp operator-=(_Tp __arg) const noexcept { return fetch_sub(__arg) - __arg; }
334 _Tp operator&=(_Tp __arg) const noexcept { return fetch_and(__arg) & __arg; }
335 _Tp operator|=(_Tp __arg) const noexcept { return fetch_or(__arg) | __arg; }
336 _Tp operator^=(_Tp __arg) const noexcept { return fetch_xor(__arg) ^ __arg; }
337};
338
339template <class _Tp>
340 requires std::floating_point<_Tp>
341struct atomic_ref<_Tp> : public __atomic_ref_base<_Tp> {
342 using __base _LIBCPP_NODEBUG = __atomic_ref_base<_Tp>;
343
344 using difference_type = __base::value_type;
345
346 explicit atomic_ref(_Tp& __obj) : __base(__obj) {
347 _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
348 std::__is_sufficiently_aligned<__base::required_alignment>(std::addressof(__obj)),
349 "atomic_ref ctor: referenced object must be aligned to required_alignment");
350 }
351
352 explicit atomic_ref(_Tp&&) = delete;
353
354 atomic_ref(const atomic_ref&) noexcept = default;
355
356 _Tp operator=(_Tp __desired) const noexcept { return __base::operator=(__desired); }
357
358 atomic_ref& operator=(const atomic_ref&) = delete;
359
360 _Tp fetch_add(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
361 if constexpr (std::__has_rmw_builtin<_Tp>()) {
362 return __atomic_fetch_add(this->__ptr_, __arg, std::__to_gcc_order(__order));
363 } else {
364 _Tp __old = this->load(memory_order_relaxed);
365 _Tp __new = __old + __arg;
366 while (!this->compare_exchange_weak(__old, __new, __order, memory_order_relaxed)) {
367 __new = __old + __arg;
368 }
369 return __old;
370 }
371 }
372 _Tp fetch_sub(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
373 if constexpr (std::__has_rmw_builtin<_Tp>()) {
374 return __atomic_fetch_sub(this->__ptr_, __arg, std::__to_gcc_order(__order));
375 } else {
376 _Tp __old = this->load(memory_order_relaxed);
377 _Tp __new = __old - __arg;
378 while (!this->compare_exchange_weak(__old, __new, __order, memory_order_relaxed)) {
379 __new = __old - __arg;
380 }
381 return __old;
382 }
383 }
384
385 _Tp operator+=(_Tp __arg) const noexcept { return fetch_add(__arg) + __arg; }
386 _Tp operator-=(_Tp __arg) const noexcept { return fetch_sub(__arg) - __arg; }
387};
388
389template <class _Tp>
390struct atomic_ref<_Tp*> : public __atomic_ref_base<_Tp*> {
391 using __base _LIBCPP_NODEBUG = __atomic_ref_base<_Tp*>;
392
393 using difference_type = ptrdiff_t;
394
395 explicit atomic_ref(_Tp*& __ptr) : __base(__ptr) {}
396
397 explicit atomic_ref(_Tp*&&) = delete;
398
399 _Tp* operator=(_Tp* __desired) const noexcept { return __base::operator=(__desired); }
400
401 atomic_ref& operator=(const atomic_ref&) = delete;
402
403 _Tp* fetch_add(ptrdiff_t __arg, memory_order __order = memory_order_seq_cst) const noexcept {
404 return __atomic_fetch_add(this->__ptr_, __arg * sizeof(_Tp), std::__to_gcc_order(__order));
405 }
406 _Tp* fetch_sub(ptrdiff_t __arg, memory_order __order = memory_order_seq_cst) const noexcept {
407 return __atomic_fetch_sub(this->__ptr_, __arg * sizeof(_Tp), std::__to_gcc_order(__order));
408 }
409# if _LIBCPP_STD_VER >= 26
410 _Tp* fetch_max(_Tp* __arg, memory_order __order = memory_order_seq_cst) const noexcept {
411 // TODO: Use __atomic_fetch_max once it accepts pointer arguments.
412 _Tp* __old = this->load(memory_order_relaxed);
413 _Tp* __new;
414 do {
415 __new = __old > __arg ? __old : __arg;
416 } while (!this->compare_exchange_weak(__old, __new, __order, memory_order_relaxed));
417 return __old;
418 }
419 _Tp* fetch_min(_Tp* __arg, memory_order __order = memory_order_seq_cst) const noexcept {
420 // TODO: Use __atomic_fetch_min once it accepts pointer arguments.
421 _Tp* __old = this->load(memory_order_relaxed);
422 _Tp* __new;
423 do {
424 __new = __old < __arg ? __old : __arg;
425 } while (!this->compare_exchange_weak(__old, __new, __order, memory_order_relaxed));
426 return __old;
427 }
428# endif
429
430 _Tp* operator++(int) const noexcept { return fetch_add(arg: 1); }
431 _Tp* operator--(int) const noexcept { return fetch_sub(arg: 1); }
432 _Tp* operator++() const noexcept { return fetch_add(arg: 1) + 1; }
433 _Tp* operator--() const noexcept { return fetch_sub(arg: 1) - 1; }
434 _Tp* operator+=(ptrdiff_t __arg) const noexcept { return fetch_add(__arg) + __arg; }
435 _Tp* operator-=(ptrdiff_t __arg) const noexcept { return fetch_sub(__arg) - __arg; }
436};
437
438_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(atomic_ref);
439
440#endif // _LIBCPP_STD_VER >= 20
441
442_LIBCPP_END_NAMESPACE_STD
443
444_LIBCPP_POP_MACROS
445
446#endif // _LIBCPP__ATOMIC_ATOMIC_REF_H
447