1// -*- C++ -*-
2//===----------------------------------------------------------------------===//
3//
4// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5// See https://llvm.org/LICENSE.txt for license information.
6// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7//
8// Kokkos v. 4.0
9// Copyright (2022) National Technology & Engineering
10// Solutions of Sandia, LLC (NTESS).
11//
12// Under the terms of Contract DE-NA0003525 with NTESS,
13// the U.S. Government retains certain rights in this software.
14//
15//===---------------------------------------------------------------------===//
16
17#ifndef _LIBCPP___ATOMIC_ATOMIC_REF_H
18#define _LIBCPP___ATOMIC_ATOMIC_REF_H
19
20#include <__assert>
21#include <__atomic/atomic_sync.h>
22#include <__atomic/atomic_waitable_traits.h>
23#include <__atomic/check_memory_order.h>
24#include <__atomic/floating_point_helper.h>
25#include <__atomic/memory_order.h>
26#include <__atomic/to_gcc_order.h>
27#include <__concepts/arithmetic.h>
28#include <__concepts/same_as.h>
29#include <__config>
30#include <__cstddef/byte.h>
31#include <__cstddef/ptrdiff_t.h>
32#include <__memory/addressof.h>
33#include <__memory/is_sufficiently_aligned.h>
34#include <__type_traits/copy_cv.h>
35#include <__type_traits/has_unique_object_representation.h>
36#include <__type_traits/is_trivially_copyable.h>
37#include <cstring>
38
39#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
40# pragma GCC system_header
41#endif
42
43_LIBCPP_PUSH_MACROS
44#include <__undef_macros>
45
46_LIBCPP_BEGIN_NAMESPACE_STD
47
48#if _LIBCPP_STD_VER >= 20
49
50// These types are required to make __atomic_is_always_lock_free work across GCC and Clang.
51// The purpose of this trick is to make sure that we provide an object with the correct alignment
52// to __atomic_is_always_lock_free, since that answer depends on the alignment.
53template <size_t _Alignment>
54struct __alignment_checker_type {
55 alignas(_Alignment) char __data;
56};
57
58template <size_t _Alignment>
59struct __get_aligner_instance {
60 static constexpr __alignment_checker_type<_Alignment> __instance{};
61};
62
63template <class _Tp>
64struct __atomic_ref_base {
65private:
66 _LIBCPP_HIDE_FROM_ABI static _Tp* __clear_padding(_Tp& __val) noexcept {
67 _Tp* __ptr = std::addressof(__val);
68# if __has_builtin(__builtin_clear_padding)
69 __builtin_clear_padding(__ptr);
70# endif
71 return __ptr;
72 }
73
74 _LIBCPP_HIDE_FROM_ABI static bool __compare_exchange(
75 _Tp* __ptr, _Tp* __expected, _Tp* __desired, bool __is_weak, int __success, int __failure) noexcept {
76 if constexpr (
77# if __has_builtin(__builtin_clear_padding)
78 has_unique_object_representations_v<_Tp> || floating_point<_Tp>
79# else
80 true // NOLINT(readability-simplify-boolean-expr)
81# endif
82 ) {
83 return __atomic_compare_exchange(__ptr, __expected, __desired, __is_weak, __success, __failure);
84 } else { // _Tp has padding bits and __builtin_clear_padding is available
85 __clear_padding(val&: *__desired);
86 _Tp __copy = *__expected;
87 __clear_padding(val&: __copy);
88 // The algorithm we use here is basically to perform `__atomic_compare_exchange` on the
89 // values until it has either succeeded, or failed because the value representation of the
90 // objects involved was different. This is why we loop around __atomic_compare_exchange:
91 // we basically loop until its failure is caused by the value representation of the objects
92 // being different, not only their object representation.
93 while (true) {
94 _Tp __prev = __copy;
95 if (__atomic_compare_exchange(__ptr, std::addressof(__copy), __desired, __is_weak, __success, __failure)) {
96 return true;
97 }
98 _Tp __curr = __copy;
99 if (std::memcmp(s1: __clear_padding(val&: __prev), s2: __clear_padding(val&: __curr), n: sizeof(_Tp)) != 0) {
100 // Value representation without padding bits do not compare equal ->
101 // write the current content of *ptr into *expected
102 std::memcpy(dest: __expected, src: std::addressof(__copy), n: sizeof(_Tp));
103 return false;
104 }
105 }
106 }
107 }
108
109 friend struct __atomic_waitable_traits<__atomic_ref_base<_Tp>>;
110
111 // require types that are 1, 2, 4, 8, or 16 bytes in length to be aligned to at least their size to be potentially
112 // used lock-free
113 static constexpr size_t __min_alignment = (sizeof(_Tp) & (sizeof(_Tp) - 1)) || (sizeof(_Tp) > 16) ? 0 : sizeof(_Tp);
114
115public:
116 using value_type = _Tp;
117
118 static constexpr size_t required_alignment = alignof(_Tp) > __min_alignment ? alignof(_Tp) : __min_alignment;
119
120 // The __atomic_always_lock_free builtin takes into account the alignment of the pointer if provided,
121 // so we create a fake pointer with a suitable alignment when querying it. Note that we are guaranteed
122 // that the pointer is going to be aligned properly at runtime because that is a (checked) precondition
123 // of atomic_ref's constructor.
124 static constexpr bool is_always_lock_free =
125 __atomic_always_lock_free(sizeof(_Tp), std::addressof(__get_aligner_instance<required_alignment>::__instance));
126
127 [[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool is_lock_free() const noexcept {
128 return __atomic_is_lock_free(sizeof(_Tp), __ptr_);
129 }
130
131 _LIBCPP_HIDE_FROM_ABI void store(_Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept
132 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__order) {
133 _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
134 __order == memory_order::relaxed || __order == memory_order::release || __order == memory_order::seq_cst,
135 "atomic_ref: memory order argument to atomic store operation is invalid");
136 __atomic_store(__ptr_, __clear_padding(val&: __desired), std::__to_gcc_order(__order));
137 }
138
139 _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __desired) const noexcept {
140 store(__desired);
141 return __desired;
142 }
143
144 [[nodiscard]] _LIBCPP_HIDE_FROM_ABI _Tp load(memory_order __order = memory_order::seq_cst) const noexcept
145 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__order) {
146 _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
147 __order == memory_order::relaxed || __order == memory_order::consume || __order == memory_order::acquire ||
148 __order == memory_order::seq_cst,
149 "atomic_ref: memory order argument to atomic load operation is invalid");
150 alignas(_Tp) byte __mem[sizeof(_Tp)];
151 auto* __ret = reinterpret_cast<_Tp*>(__mem);
152 __atomic_load(__ptr_, __ret, std::__to_gcc_order(__order));
153 return *__ret;
154 }
155
156 _LIBCPP_HIDE_FROM_ABI operator _Tp() const noexcept { return load(); }
157
158 _LIBCPP_HIDE_FROM_ABI _Tp exchange(_Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept {
159 alignas(_Tp) byte __mem[sizeof(_Tp)];
160 auto* __ret = reinterpret_cast<_Tp*>(__mem);
161 __atomic_exchange(__ptr_, __clear_padding(val&: __desired), __ret, std::__to_gcc_order(__order));
162 return *__ret;
163 }
164
165 _LIBCPP_HIDE_FROM_ABI bool
166 compare_exchange_weak(_Tp& __expected, _Tp __desired, memory_order __success, memory_order __failure) const noexcept
167 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__success, __failure) {
168 _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
169 __failure == memory_order::relaxed || __failure == memory_order::consume ||
170 __failure == memory_order::acquire || __failure == memory_order::seq_cst,
171 "atomic_ref: failure memory order argument to weak atomic compare-and-exchange operation is invalid");
172 return __compare_exchange(
173 ptr: __ptr_,
174 expected: std::addressof(__expected),
175 desired: std::addressof(__desired),
176 is_weak: true,
177 success: std::__to_gcc_order(order: __success),
178 failure: std::__to_gcc_order(order: __failure));
179 }
180 _LIBCPP_HIDE_FROM_ABI bool
181 compare_exchange_strong(_Tp& __expected, _Tp __desired, memory_order __success, memory_order __failure) const noexcept
182 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__success, __failure) {
183 _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
184 __failure == memory_order::relaxed || __failure == memory_order::consume ||
185 __failure == memory_order::acquire || __failure == memory_order::seq_cst,
186 "atomic_ref: failure memory order argument to strong atomic compare-and-exchange operation is invalid");
187 return __compare_exchange(
188 ptr: __ptr_,
189 expected: std::addressof(__expected),
190 desired: std::addressof(__desired),
191 is_weak: false,
192 success: std::__to_gcc_order(order: __success),
193 failure: std::__to_gcc_order(order: __failure));
194 }
195
196 _LIBCPP_HIDE_FROM_ABI bool
197 compare_exchange_weak(_Tp& __expected, _Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept {
198 return __compare_exchange(
199 ptr: __ptr_,
200 expected: std::addressof(__expected),
201 desired: std::addressof(__desired),
202 is_weak: true,
203 success: std::__to_gcc_order(__order),
204 failure: std::__to_gcc_failure_order(__order));
205 }
206 _LIBCPP_HIDE_FROM_ABI bool
207 compare_exchange_strong(_Tp& __expected, _Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept {
208 return __compare_exchange(
209 ptr: __ptr_,
210 expected: std::addressof(__expected),
211 desired: std::addressof(__desired),
212 is_weak: false,
213 success: std::__to_gcc_order(__order),
214 failure: std::__to_gcc_failure_order(__order));
215 }
216
217 _LIBCPP_HIDE_FROM_ABI void wait(_Tp __old, memory_order __order = memory_order::seq_cst) const noexcept
218 _LIBCPP_CHECK_WAIT_MEMORY_ORDER(__order) {
219 _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
220 __order == memory_order::relaxed || __order == memory_order::consume || __order == memory_order::acquire ||
221 __order == memory_order::seq_cst,
222 "atomic_ref: memory order argument to atomic wait operation is invalid");
223 std::__atomic_wait(*this, __old, __order);
224 }
225 _LIBCPP_HIDE_FROM_ABI void notify_one() const noexcept { std::__atomic_notify_one(*this); }
226 _LIBCPP_HIDE_FROM_ABI void notify_all() const noexcept { std::__atomic_notify_all(*this); }
227# if _LIBCPP_STD_VER >= 26
228 [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __copy_cv_t<_Tp, void>* address() const noexcept { return __ptr_; }
229# endif
230
231protected:
232 using _Aligned_Tp [[__gnu__::__aligned__(required_alignment), __gnu__::__nodebug__]] = _Tp;
233 _Aligned_Tp* __ptr_;
234
235 _LIBCPP_HIDE_FROM_ABI __atomic_ref_base(_Tp& __obj) : __ptr_(std::addressof(__obj)) {}
236};
237
238template <class _Tp>
239struct __atomic_waitable_traits<__atomic_ref_base<_Tp>> {
240 using __value_type _LIBCPP_NODEBUG = _Tp;
241
242 static _LIBCPP_HIDE_FROM_ABI _Tp __atomic_load(const __atomic_ref_base<_Tp>& __a, memory_order __order) {
243 return __a.load(__order);
244 }
245 static _LIBCPP_HIDE_FROM_ABI const _Tp* __atomic_contention_address(const __atomic_ref_base<_Tp>& __a) {
246 return __a.__ptr_;
247 }
248};
249
250template <class _Tp>
251struct atomic_ref : public __atomic_ref_base<_Tp> {
252 static_assert(is_trivially_copyable_v<_Tp>, "std::atomic_ref<T> requires that 'T' be a trivially copyable type");
253
254 using __base _LIBCPP_NODEBUG = __atomic_ref_base<_Tp>;
255
256 _LIBCPP_HIDE_FROM_ABI explicit atomic_ref(_Tp& __obj) : __base(__obj) {
257 _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
258 std::__is_sufficiently_aligned<__base::required_alignment>(std::addressof(__obj)),
259 "atomic_ref ctor: referenced object must be aligned to required_alignment");
260 }
261
262 _LIBCPP_HIDE_FROM_ABI atomic_ref(const atomic_ref&) noexcept = default;
263
264 _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __desired) const noexcept { return __base::operator=(__desired); }
265
266 atomic_ref& operator=(const atomic_ref&) = delete;
267};
268
269template <class _Tp>
270 requires(std::integral<_Tp> && !std::same_as<bool, _Tp>)
271struct atomic_ref<_Tp> : public __atomic_ref_base<_Tp> {
272 using __base _LIBCPP_NODEBUG = __atomic_ref_base<_Tp>;
273
274 using difference_type = __base::value_type;
275
276 _LIBCPP_HIDE_FROM_ABI explicit atomic_ref(_Tp& __obj) : __base(__obj) {
277 _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
278 std::__is_sufficiently_aligned<__base::required_alignment>(std::addressof(__obj)),
279 "atomic_ref ctor: referenced object must be aligned to required_alignment");
280 }
281
282 _LIBCPP_HIDE_FROM_ABI atomic_ref(const atomic_ref&) noexcept = default;
283
284 _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __desired) const noexcept { return __base::operator=(__desired); }
285
286 atomic_ref& operator=(const atomic_ref&) = delete;
287
288 _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
289 return __atomic_fetch_add(this->__ptr_, __arg, std::__to_gcc_order(__order));
290 }
291 _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
292 return __atomic_fetch_sub(this->__ptr_, __arg, std::__to_gcc_order(__order));
293 }
294 _LIBCPP_HIDE_FROM_ABI _Tp fetch_and(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
295 return __atomic_fetch_and(this->__ptr_, __arg, std::__to_gcc_order(__order));
296 }
297 _LIBCPP_HIDE_FROM_ABI _Tp fetch_or(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
298 return __atomic_fetch_or(this->__ptr_, __arg, std::__to_gcc_order(__order));
299 }
300 _LIBCPP_HIDE_FROM_ABI _Tp fetch_xor(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
301 return __atomic_fetch_xor(this->__ptr_, __arg, std::__to_gcc_order(__order));
302 }
303
304 _LIBCPP_HIDE_FROM_ABI _Tp operator++(int) const noexcept { return fetch_add(arg: _Tp(1)); }
305 _LIBCPP_HIDE_FROM_ABI _Tp operator--(int) const noexcept { return fetch_sub(arg: _Tp(1)); }
306 _LIBCPP_HIDE_FROM_ABI _Tp operator++() const noexcept { return fetch_add(arg: _Tp(1)) + _Tp(1); }
307 _LIBCPP_HIDE_FROM_ABI _Tp operator--() const noexcept { return fetch_sub(arg: _Tp(1)) - _Tp(1); }
308 _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __arg) const noexcept { return fetch_add(__arg) + __arg; }
309 _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __arg) const noexcept { return fetch_sub(__arg) - __arg; }
310 _LIBCPP_HIDE_FROM_ABI _Tp operator&=(_Tp __arg) const noexcept { return fetch_and(__arg) & __arg; }
311 _LIBCPP_HIDE_FROM_ABI _Tp operator|=(_Tp __arg) const noexcept { return fetch_or(__arg) | __arg; }
312 _LIBCPP_HIDE_FROM_ABI _Tp operator^=(_Tp __arg) const noexcept { return fetch_xor(__arg) ^ __arg; }
313};
314
315template <class _Tp>
316 requires std::floating_point<_Tp>
317struct atomic_ref<_Tp> : public __atomic_ref_base<_Tp> {
318 using __base _LIBCPP_NODEBUG = __atomic_ref_base<_Tp>;
319
320 using difference_type = __base::value_type;
321
322 _LIBCPP_HIDE_FROM_ABI explicit atomic_ref(_Tp& __obj) : __base(__obj) {
323 _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
324 std::__is_sufficiently_aligned<__base::required_alignment>(std::addressof(__obj)),
325 "atomic_ref ctor: referenced object must be aligned to required_alignment");
326 }
327
328 _LIBCPP_HIDE_FROM_ABI atomic_ref(const atomic_ref&) noexcept = default;
329
330 _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __desired) const noexcept { return __base::operator=(__desired); }
331
332 atomic_ref& operator=(const atomic_ref&) = delete;
333
334 _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
335 if constexpr (std::__has_rmw_builtin<_Tp>()) {
336 return __atomic_fetch_add(this->__ptr_, __arg, std::__to_gcc_order(__order));
337 } else {
338 _Tp __old = this->load(memory_order_relaxed);
339 _Tp __new = __old + __arg;
340 while (!this->compare_exchange_weak(__old, __new, __order, memory_order_relaxed)) {
341 __new = __old + __arg;
342 }
343 return __old;
344 }
345 }
346 _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
347 if constexpr (std::__has_rmw_builtin<_Tp>()) {
348 return __atomic_fetch_sub(this->__ptr_, __arg, std::__to_gcc_order(__order));
349 } else {
350 _Tp __old = this->load(memory_order_relaxed);
351 _Tp __new = __old - __arg;
352 while (!this->compare_exchange_weak(__old, __new, __order, memory_order_relaxed)) {
353 __new = __old - __arg;
354 }
355 return __old;
356 }
357 }
358
359 _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __arg) const noexcept { return fetch_add(__arg) + __arg; }
360 _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __arg) const noexcept { return fetch_sub(__arg) - __arg; }
361};
362
363template <class _Tp>
364struct atomic_ref<_Tp*> : public __atomic_ref_base<_Tp*> {
365 using __base _LIBCPP_NODEBUG = __atomic_ref_base<_Tp*>;
366
367 using difference_type = ptrdiff_t;
368
369 _LIBCPP_HIDE_FROM_ABI explicit atomic_ref(_Tp*& __ptr) : __base(__ptr) {}
370
371 _LIBCPP_HIDE_FROM_ABI _Tp* operator=(_Tp* __desired) const noexcept { return __base::operator=(__desired); }
372
373 atomic_ref& operator=(const atomic_ref&) = delete;
374
375 _LIBCPP_HIDE_FROM_ABI _Tp* fetch_add(ptrdiff_t __arg, memory_order __order = memory_order_seq_cst) const noexcept {
376 return __atomic_fetch_add(this->__ptr_, __arg * sizeof(_Tp), std::__to_gcc_order(__order));
377 }
378 _LIBCPP_HIDE_FROM_ABI _Tp* fetch_sub(ptrdiff_t __arg, memory_order __order = memory_order_seq_cst) const noexcept {
379 return __atomic_fetch_sub(this->__ptr_, __arg * sizeof(_Tp), std::__to_gcc_order(__order));
380 }
381
382 _LIBCPP_HIDE_FROM_ABI _Tp* operator++(int) const noexcept { return fetch_add(arg: 1); }
383 _LIBCPP_HIDE_FROM_ABI _Tp* operator--(int) const noexcept { return fetch_sub(arg: 1); }
384 _LIBCPP_HIDE_FROM_ABI _Tp* operator++() const noexcept { return fetch_add(arg: 1) + 1; }
385 _LIBCPP_HIDE_FROM_ABI _Tp* operator--() const noexcept { return fetch_sub(arg: 1) - 1; }
386 _LIBCPP_HIDE_FROM_ABI _Tp* operator+=(ptrdiff_t __arg) const noexcept { return fetch_add(__arg) + __arg; }
387 _LIBCPP_HIDE_FROM_ABI _Tp* operator-=(ptrdiff_t __arg) const noexcept { return fetch_sub(__arg) - __arg; }
388};
389
390_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(atomic_ref);
391
392#endif // _LIBCPP_STD_VER >= 20
393
394_LIBCPP_END_NAMESPACE_STD
395
396_LIBCPP_POP_MACROS
397
398#endif // _LIBCPP__ATOMIC_ATOMIC_REF_H
399