1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#ifndef _LIBCPP___ATOMIC_ATOMIC_SYNC_TIMED_H
10#define _LIBCPP___ATOMIC_ATOMIC_SYNC_TIMED_H
11
12#include <__atomic/atomic_waitable_traits.h>
13#include <__atomic/contention_t.h>
14#include <__atomic/memory_order.h>
15#include <__chrono/duration.h>
16#include <__config>
17#include <__memory/addressof.h>
18#include <__thread/poll_with_backoff.h>
19#include <__thread/timed_backoff_policy.h>
20#include <__type_traits/decay.h>
21#include <cstdint>
22#include <cstring>
23
24#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
25# pragma GCC system_header
26#endif
27
28#if _LIBCPP_STD_VER >= 20 && _LIBCPP_HAS_THREADS
29
30_LIBCPP_BEGIN_NAMESPACE_STD
31
32# if _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
33
34_LIBCPP_BEGIN_EXPLICIT_ABI_ANNOTATIONS
35_LIBCPP_AVAILABILITY_NEW_SYNC
36_LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __atomic_monitor_global(void const* __address) _NOEXCEPT;
37
38// wait on the global contention state to be changed from the given value for the address
39_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void __atomic_wait_global_table_with_timeout(
40 void const* __address, __cxx_contention_t __monitor_value, uint64_t __timeout_ns) _NOEXCEPT;
41
42// wait on the address directly with the native platform wait
43template <std::size_t _Size>
44_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void
45__atomic_wait_native_with_timeout(void const* __address, void const* __old_value, uint64_t __timeout_ns) _NOEXCEPT;
46_LIBCPP_END_EXPLICIT_ABI_ANNOTATIONS
47
48template <class _AtomicWaitable, class _Poll, class _Rep, class _Period>
49struct __atomic_wait_timed_backoff_impl {
50 const _AtomicWaitable& __a_;
51 _Poll __poll_;
52 memory_order __order_;
53 chrono::duration<_Rep, _Period> __rel_time_;
54
55 using __waitable_traits _LIBCPP_NODEBUG = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >;
56 using __value_type _LIBCPP_NODEBUG = typename __waitable_traits::__value_type;
57
58 _LIBCPP_HIDE_FROM_ABI __backoff_results operator()(chrono::nanoseconds __elapsed) const {
59 if (__elapsed > chrono::microseconds(4)) {
60 auto __contention_address = const_cast<const void*>(
61 static_cast<const volatile void*>(__waitable_traits::__atomic_contention_address(__a_)));
62
63 uint64_t __timeout_ns =
64 static_cast<uint64_t>((chrono::duration_cast<chrono::nanoseconds>(__rel_time_) - __elapsed).count());
65
66 if constexpr (__has_native_atomic_wait<__value_type>) {
67 auto __atomic_value = __waitable_traits::__atomic_load(__a_, __order_);
68 if (__poll_(__atomic_value))
69 return __backoff_results::__poll_success;
70 std::__atomic_wait_native_with_timeout<sizeof(__value_type)>(
71 __contention_address, std::addressof(__atomic_value), __timeout_ns);
72 } else {
73 __cxx_contention_t __monitor_val = std::__atomic_monitor_global(address: __contention_address);
74 auto __atomic_value = __waitable_traits::__atomic_load(__a_, __order_);
75 if (__poll_(__atomic_value))
76 return __backoff_results::__poll_success;
77 std::__atomic_wait_global_table_with_timeout(address: __contention_address, monitor_value: __monitor_val, __timeout_ns);
78 }
79 } else {
80 } // poll
81 return __backoff_results::__continue_poll;
82 }
83};
84
85// The semantics of this function are similar to `atomic`'s
86// `.wait(T old, std::memory_order order)` with a timeout, but instead of having a hardcoded
87// predicate (is the loaded value unequal to `old`?), the predicate function is
88// specified as an argument. The loaded value is given as an in-out argument to
89// the predicate. If the predicate function returns `true`,
90// `__atomic_wait_unless_with_timeout` will return. If the predicate function returns
91// `false`, it must set the argument to its current understanding of the atomic
92// value. The predicate function must not return `false` spuriously.
93template <class _AtomicWaitable, class _Poll, class _Rep, class _Period>
94_LIBCPP_HIDE_FROM_ABI bool __atomic_wait_unless_with_timeout(
95 const _AtomicWaitable& __a,
96 memory_order __order,
97 _Poll&& __poll,
98 chrono::duration<_Rep, _Period> const& __rel_time) {
99 static_assert(__atomic_waitable<_AtomicWaitable>, "");
100 __atomic_wait_timed_backoff_impl<_AtomicWaitable, __decay_t<_Poll>, _Rep, _Period> __backoff_fn = {
101 __a, __poll, __order, __rel_time};
102 auto __poll_result = std::__libcpp_thread_poll_with_backoff(
103 /* poll */
104 [&]() {
105 auto __current_val = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__atomic_load(__a, __order);
106 return __poll(__current_val);
107 },
108 /* backoff */ __backoff_fn,
109 __rel_time);
110
111 return __poll_result == __poll_with_backoff_results::__poll_success;
112}
113
114# else // _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
115
116template <class _AtomicWaitable, class _Poll, class _Rep, class _Period>
117_LIBCPP_HIDE_FROM_ABI bool __atomic_wait_unless_with_timeout(
118 const _AtomicWaitable& __a,
119 memory_order __order,
120 _Poll&& __poll,
121 chrono::duration<_Rep, _Period> const& __rel_time) {
122 auto __res = std::__libcpp_thread_poll_with_backoff(
123 /* poll */
124 [&]() {
125 auto __current_val = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__atomic_load(__a, __order);
126 return __poll(__current_val);
127 },
128 /* backoff */ __libcpp_timed_backoff_policy(),
129 __rel_time);
130 return __res == __poll_with_backoff_results::__poll_success;
131}
132
133# endif // _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
134
135_LIBCPP_END_NAMESPACE_STD
136
137#endif // _LIBCPP_STD_VER >= 20 && _LIBCPP_HAS_THREADS
138
139#endif // _LIBCPP___ATOMIC_ATOMIC_SYNC_TIMED_H
140