1// -*- C++ -*-
2//===----------------------------------------------------------------------===//
3//
4// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5// See https://llvm.org/LICENSE.txt for license information.
6// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7//
8// Kokkos v. 4.0
9// Copyright (2022) National Technology & Engineering
10// Solutions of Sandia, LLC (NTESS).
11//
12// Under the terms of Contract DE-NA0003525 with NTESS,
13// the U.S. Government retains certain rights in this software.
14//
15//===---------------------------------------------------------------------===//
16
17#ifndef _LIBCPP___ATOMIC_ATOMIC_REF_H
18#define _LIBCPP___ATOMIC_ATOMIC_REF_H
19
20#include <__assert>
21#include <__atomic/atomic_sync.h>
22#include <__atomic/check_memory_order.h>
23#include <__atomic/memory_order.h>
24#include <__atomic/to_gcc_order.h>
25#include <__concepts/arithmetic.h>
26#include <__concepts/same_as.h>
27#include <__config>
28#include <__cstddef/byte.h>
29#include <__cstddef/ptrdiff_t.h>
30#include <__memory/addressof.h>
31#include <__type_traits/has_unique_object_representation.h>
32#include <__type_traits/is_trivially_copyable.h>
33#include <cstdint>
34#include <cstring>
35
36#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
37# pragma GCC system_header
38#endif
39
40_LIBCPP_PUSH_MACROS
41#include <__undef_macros>
42
43_LIBCPP_BEGIN_NAMESPACE_STD
44
45#if _LIBCPP_STD_VER >= 20
46
47// These types are required to make __atomic_is_always_lock_free work across GCC and Clang.
48// The purpose of this trick is to make sure that we provide an object with the correct alignment
49// to __atomic_is_always_lock_free, since that answer depends on the alignment.
50template <size_t _Alignment>
51struct __alignment_checker_type {
52 alignas(_Alignment) char __data;
53};
54
55template <size_t _Alignment>
56struct __get_aligner_instance {
57 static constexpr __alignment_checker_type<_Alignment> __instance{};
58};
59
60template <class _Tp>
61struct __atomic_ref_base {
62private:
63 _LIBCPP_HIDE_FROM_ABI static _Tp* __clear_padding(_Tp& __val) noexcept {
64 _Tp* __ptr = std::addressof(__val);
65# if __has_builtin(__builtin_clear_padding)
66 __builtin_clear_padding(__ptr);
67# endif
68 return __ptr;
69 }
70
71 _LIBCPP_HIDE_FROM_ABI static bool __compare_exchange(
72 _Tp* __ptr, _Tp* __expected, _Tp* __desired, bool __is_weak, int __success, int __failure) noexcept {
73 if constexpr (
74# if __has_builtin(__builtin_clear_padding)
75 has_unique_object_representations_v<_Tp> || floating_point<_Tp>
76# else
77 true // NOLINT(readability-simplify-boolean-expr)
78# endif
79 ) {
80 return __atomic_compare_exchange(__ptr, __expected, __desired, __is_weak, __success, __failure);
81 } else { // _Tp has padding bits and __builtin_clear_padding is available
82 __clear_padding(val&: *__desired);
83 _Tp __copy = *__expected;
84 __clear_padding(val&: __copy);
85 // The algorithm we use here is basically to perform `__atomic_compare_exchange` on the
86 // values until it has either succeeded, or failed because the value representation of the
87 // objects involved was different. This is why we loop around __atomic_compare_exchange:
88 // we basically loop until its failure is caused by the value representation of the objects
89 // being different, not only their object representation.
90 while (true) {
91 _Tp __prev = __copy;
92 if (__atomic_compare_exchange(__ptr, std::addressof(__copy), __desired, __is_weak, __success, __failure)) {
93 return true;
94 }
95 _Tp __curr = __copy;
96 if (std::memcmp(s1: __clear_padding(val&: __prev), s2: __clear_padding(val&: __curr), n: sizeof(_Tp)) != 0) {
97 // Value representation without padding bits do not compare equal ->
98 // write the current content of *ptr into *expected
99 std::memcpy(dest: __expected, src: std::addressof(__copy), n: sizeof(_Tp));
100 return false;
101 }
102 }
103 }
104 }
105
106 friend struct __atomic_waitable_traits<__atomic_ref_base<_Tp>>;
107
108 // require types that are 1, 2, 4, 8, or 16 bytes in length to be aligned to at least their size to be potentially
109 // used lock-free
110 static constexpr size_t __min_alignment = (sizeof(_Tp) & (sizeof(_Tp) - 1)) || (sizeof(_Tp) > 16) ? 0 : sizeof(_Tp);
111
112public:
113 using value_type = _Tp;
114
115 static constexpr size_t required_alignment = alignof(_Tp) > __min_alignment ? alignof(_Tp) : __min_alignment;
116
117 // The __atomic_always_lock_free builtin takes into account the alignment of the pointer if provided,
118 // so we create a fake pointer with a suitable alignment when querying it. Note that we are guaranteed
119 // that the pointer is going to be aligned properly at runtime because that is a (checked) precondition
120 // of atomic_ref's constructor.
121 static constexpr bool is_always_lock_free =
122 __atomic_always_lock_free(sizeof(_Tp), std::addressof(__get_aligner_instance<required_alignment>::__instance));
123
124 _LIBCPP_HIDE_FROM_ABI bool is_lock_free() const noexcept { return __atomic_is_lock_free(sizeof(_Tp), __ptr_); }
125
126 _LIBCPP_HIDE_FROM_ABI void store(_Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept
127 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__order) {
128 _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
129 __order == memory_order::relaxed || __order == memory_order::release || __order == memory_order::seq_cst,
130 "atomic_ref: memory order argument to atomic store operation is invalid");
131 __atomic_store(__ptr_, __clear_padding(val&: __desired), std::__to_gcc_order(__order));
132 }
133
134 _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __desired) const noexcept {
135 store(__desired);
136 return __desired;
137 }
138
139 _LIBCPP_HIDE_FROM_ABI _Tp load(memory_order __order = memory_order::seq_cst) const noexcept
140 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__order) {
141 _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
142 __order == memory_order::relaxed || __order == memory_order::consume || __order == memory_order::acquire ||
143 __order == memory_order::seq_cst,
144 "atomic_ref: memory order argument to atomic load operation is invalid");
145 alignas(_Tp) byte __mem[sizeof(_Tp)];
146 auto* __ret = reinterpret_cast<_Tp*>(__mem);
147 __atomic_load(__ptr_, __ret, std::__to_gcc_order(__order));
148 return *__ret;
149 }
150
151 _LIBCPP_HIDE_FROM_ABI operator _Tp() const noexcept { return load(); }
152
153 _LIBCPP_HIDE_FROM_ABI _Tp exchange(_Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept {
154 alignas(_Tp) byte __mem[sizeof(_Tp)];
155 auto* __ret = reinterpret_cast<_Tp*>(__mem);
156 __atomic_exchange(__ptr_, __clear_padding(val&: __desired), __ret, std::__to_gcc_order(__order));
157 return *__ret;
158 }
159
160 _LIBCPP_HIDE_FROM_ABI bool
161 compare_exchange_weak(_Tp& __expected, _Tp __desired, memory_order __success, memory_order __failure) const noexcept
162 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__success, __failure) {
163 _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
164 __failure == memory_order::relaxed || __failure == memory_order::consume ||
165 __failure == memory_order::acquire || __failure == memory_order::seq_cst,
166 "atomic_ref: failure memory order argument to weak atomic compare-and-exchange operation is invalid");
167 return __compare_exchange(
168 ptr: __ptr_,
169 expected: std::addressof(__expected),
170 desired: std::addressof(__desired),
171 is_weak: true,
172 success: std::__to_gcc_order(order: __success),
173 failure: std::__to_gcc_order(order: __failure));
174 }
175 _LIBCPP_HIDE_FROM_ABI bool
176 compare_exchange_strong(_Tp& __expected, _Tp __desired, memory_order __success, memory_order __failure) const noexcept
177 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__success, __failure) {
178 _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
179 __failure == memory_order::relaxed || __failure == memory_order::consume ||
180 __failure == memory_order::acquire || __failure == memory_order::seq_cst,
181 "atomic_ref: failure memory order argument to strong atomic compare-and-exchange operation is invalid");
182 return __compare_exchange(
183 ptr: __ptr_,
184 expected: std::addressof(__expected),
185 desired: std::addressof(__desired),
186 is_weak: false,
187 success: std::__to_gcc_order(order: __success),
188 failure: std::__to_gcc_order(order: __failure));
189 }
190
191 _LIBCPP_HIDE_FROM_ABI bool
192 compare_exchange_weak(_Tp& __expected, _Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept {
193 return __compare_exchange(
194 ptr: __ptr_,
195 expected: std::addressof(__expected),
196 desired: std::addressof(__desired),
197 is_weak: true,
198 success: std::__to_gcc_order(__order),
199 failure: std::__to_gcc_failure_order(__order));
200 }
201 _LIBCPP_HIDE_FROM_ABI bool
202 compare_exchange_strong(_Tp& __expected, _Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept {
203 return __compare_exchange(
204 ptr: __ptr_,
205 expected: std::addressof(__expected),
206 desired: std::addressof(__desired),
207 is_weak: false,
208 success: std::__to_gcc_order(__order),
209 failure: std::__to_gcc_failure_order(__order));
210 }
211
212 _LIBCPP_HIDE_FROM_ABI void wait(_Tp __old, memory_order __order = memory_order::seq_cst) const noexcept
213 _LIBCPP_CHECK_WAIT_MEMORY_ORDER(__order) {
214 _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
215 __order == memory_order::relaxed || __order == memory_order::consume || __order == memory_order::acquire ||
216 __order == memory_order::seq_cst,
217 "atomic_ref: memory order argument to atomic wait operation is invalid");
218 std::__atomic_wait(*this, __old, __order);
219 }
220 _LIBCPP_HIDE_FROM_ABI void notify_one() const noexcept { std::__atomic_notify_one(*this); }
221 _LIBCPP_HIDE_FROM_ABI void notify_all() const noexcept { std::__atomic_notify_all(*this); }
222
223protected:
224 using _Aligned_Tp [[__gnu__::__aligned__(required_alignment), __gnu__::__nodebug__]] = _Tp;
225 _Aligned_Tp* __ptr_;
226
227 _LIBCPP_HIDE_FROM_ABI __atomic_ref_base(_Tp& __obj) : __ptr_(std::addressof(__obj)) {}
228};
229
230template <class _Tp>
231struct __atomic_waitable_traits<__atomic_ref_base<_Tp>> {
232 static _LIBCPP_HIDE_FROM_ABI _Tp __atomic_load(const __atomic_ref_base<_Tp>& __a, memory_order __order) {
233 return __a.load(__order);
234 }
235 static _LIBCPP_HIDE_FROM_ABI const _Tp* __atomic_contention_address(const __atomic_ref_base<_Tp>& __a) {
236 return __a.__ptr_;
237 }
238};
239
240template <class _Tp>
241struct atomic_ref : public __atomic_ref_base<_Tp> {
242 static_assert(is_trivially_copyable_v<_Tp>, "std::atomic_ref<T> requires that 'T' be a trivially copyable type");
243
244 using __base _LIBCPP_NODEBUG = __atomic_ref_base<_Tp>;
245
246 _LIBCPP_HIDE_FROM_ABI explicit atomic_ref(_Tp& __obj) : __base(__obj) {
247 _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
248 reinterpret_cast<uintptr_t>(std::addressof(__obj)) % __base::required_alignment == 0,
249 "atomic_ref ctor: referenced object must be aligned to required_alignment");
250 }
251
252 _LIBCPP_HIDE_FROM_ABI atomic_ref(const atomic_ref&) noexcept = default;
253
254 _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __desired) const noexcept { return __base::operator=(__desired); }
255
256 atomic_ref& operator=(const atomic_ref&) = delete;
257};
258
259template <class _Tp>
260 requires(std::integral<_Tp> && !std::same_as<bool, _Tp>)
261struct atomic_ref<_Tp> : public __atomic_ref_base<_Tp> {
262 using __base _LIBCPP_NODEBUG = __atomic_ref_base<_Tp>;
263
264 using difference_type = __base::value_type;
265
266 _LIBCPP_HIDE_FROM_ABI explicit atomic_ref(_Tp& __obj) : __base(__obj) {
267 _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
268 reinterpret_cast<uintptr_t>(std::addressof(__obj)) % __base::required_alignment == 0,
269 "atomic_ref ctor: referenced object must be aligned to required_alignment");
270 }
271
272 _LIBCPP_HIDE_FROM_ABI atomic_ref(const atomic_ref&) noexcept = default;
273
274 _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __desired) const noexcept { return __base::operator=(__desired); }
275
276 atomic_ref& operator=(const atomic_ref&) = delete;
277
278 _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
279 return __atomic_fetch_add(this->__ptr_, __arg, std::__to_gcc_order(__order));
280 }
281 _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
282 return __atomic_fetch_sub(this->__ptr_, __arg, std::__to_gcc_order(__order));
283 }
284 _LIBCPP_HIDE_FROM_ABI _Tp fetch_and(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
285 return __atomic_fetch_and(this->__ptr_, __arg, std::__to_gcc_order(__order));
286 }
287 _LIBCPP_HIDE_FROM_ABI _Tp fetch_or(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
288 return __atomic_fetch_or(this->__ptr_, __arg, std::__to_gcc_order(__order));
289 }
290 _LIBCPP_HIDE_FROM_ABI _Tp fetch_xor(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
291 return __atomic_fetch_xor(this->__ptr_, __arg, std::__to_gcc_order(__order));
292 }
293
294 _LIBCPP_HIDE_FROM_ABI _Tp operator++(int) const noexcept { return fetch_add(arg: _Tp(1)); }
295 _LIBCPP_HIDE_FROM_ABI _Tp operator--(int) const noexcept { return fetch_sub(arg: _Tp(1)); }
296 _LIBCPP_HIDE_FROM_ABI _Tp operator++() const noexcept { return fetch_add(arg: _Tp(1)) + _Tp(1); }
297 _LIBCPP_HIDE_FROM_ABI _Tp operator--() const noexcept { return fetch_sub(arg: _Tp(1)) - _Tp(1); }
298 _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __arg) const noexcept { return fetch_add(__arg) + __arg; }
299 _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __arg) const noexcept { return fetch_sub(__arg) - __arg; }
300 _LIBCPP_HIDE_FROM_ABI _Tp operator&=(_Tp __arg) const noexcept { return fetch_and(__arg) & __arg; }
301 _LIBCPP_HIDE_FROM_ABI _Tp operator|=(_Tp __arg) const noexcept { return fetch_or(__arg) | __arg; }
302 _LIBCPP_HIDE_FROM_ABI _Tp operator^=(_Tp __arg) const noexcept { return fetch_xor(__arg) ^ __arg; }
303};
304
305template <class _Tp>
306 requires std::floating_point<_Tp>
307struct atomic_ref<_Tp> : public __atomic_ref_base<_Tp> {
308 using __base _LIBCPP_NODEBUG = __atomic_ref_base<_Tp>;
309
310 using difference_type = __base::value_type;
311
312 _LIBCPP_HIDE_FROM_ABI explicit atomic_ref(_Tp& __obj) : __base(__obj) {
313 _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
314 reinterpret_cast<uintptr_t>(std::addressof(__obj)) % __base::required_alignment == 0,
315 "atomic_ref ctor: referenced object must be aligned to required_alignment");
316 }
317
318 _LIBCPP_HIDE_FROM_ABI atomic_ref(const atomic_ref&) noexcept = default;
319
320 _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __desired) const noexcept { return __base::operator=(__desired); }
321
322 atomic_ref& operator=(const atomic_ref&) = delete;
323
324 _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
325 _Tp __old = this->load(memory_order_relaxed);
326 _Tp __new = __old + __arg;
327 while (!this->compare_exchange_weak(__old, __new, __order, memory_order_relaxed)) {
328 __new = __old + __arg;
329 }
330 return __old;
331 }
332 _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
333 _Tp __old = this->load(memory_order_relaxed);
334 _Tp __new = __old - __arg;
335 while (!this->compare_exchange_weak(__old, __new, __order, memory_order_relaxed)) {
336 __new = __old - __arg;
337 }
338 return __old;
339 }
340
341 _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __arg) const noexcept { return fetch_add(__arg) + __arg; }
342 _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __arg) const noexcept { return fetch_sub(__arg) - __arg; }
343};
344
345template <class _Tp>
346struct atomic_ref<_Tp*> : public __atomic_ref_base<_Tp*> {
347 using __base _LIBCPP_NODEBUG = __atomic_ref_base<_Tp*>;
348
349 using difference_type = ptrdiff_t;
350
351 _LIBCPP_HIDE_FROM_ABI explicit atomic_ref(_Tp*& __ptr) : __base(__ptr) {}
352
353 _LIBCPP_HIDE_FROM_ABI _Tp* operator=(_Tp* __desired) const noexcept { return __base::operator=(__desired); }
354
355 atomic_ref& operator=(const atomic_ref&) = delete;
356
357 _LIBCPP_HIDE_FROM_ABI _Tp* fetch_add(ptrdiff_t __arg, memory_order __order = memory_order_seq_cst) const noexcept {
358 return __atomic_fetch_add(this->__ptr_, __arg * sizeof(_Tp), std::__to_gcc_order(__order));
359 }
360 _LIBCPP_HIDE_FROM_ABI _Tp* fetch_sub(ptrdiff_t __arg, memory_order __order = memory_order_seq_cst) const noexcept {
361 return __atomic_fetch_sub(this->__ptr_, __arg * sizeof(_Tp), std::__to_gcc_order(__order));
362 }
363
364 _LIBCPP_HIDE_FROM_ABI _Tp* operator++(int) const noexcept { return fetch_add(arg: 1); }
365 _LIBCPP_HIDE_FROM_ABI _Tp* operator--(int) const noexcept { return fetch_sub(arg: 1); }
366 _LIBCPP_HIDE_FROM_ABI _Tp* operator++() const noexcept { return fetch_add(arg: 1) + 1; }
367 _LIBCPP_HIDE_FROM_ABI _Tp* operator--() const noexcept { return fetch_sub(arg: 1) - 1; }
368 _LIBCPP_HIDE_FROM_ABI _Tp* operator+=(ptrdiff_t __arg) const noexcept { return fetch_add(__arg) + __arg; }
369 _LIBCPP_HIDE_FROM_ABI _Tp* operator-=(ptrdiff_t __arg) const noexcept { return fetch_sub(__arg) - __arg; }
370};
371
372_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(atomic_ref);
373
374#endif // _LIBCPP_STD_VER >= 20
375
376_LIBCPP_END_NAMESPACE_STD
377
378_LIBCPP_POP_MACROS
379
380#endif // _LIBCPP__ATOMIC_ATOMIC_REF_H
381