1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include <__thread/timed_backoff_policy.h>
10#include <atomic>
11#include <climits>
12#include <cstddef>
13#include <cstdint>
14#include <cstring>
15#include <functional>
16#include <new>
17#include <thread>
18#include <type_traits>
19
20// Determine whether to use the public os_sync API on Apple platforms based on the deployment target.
21// Otherwise we fall back to the private __ulock API.
22#if defined(__APPLE__)
23# if (defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && \
24 __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 140400) || \
25 (defined(__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__) && \
26 __ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__ >= 170400) || \
27 (defined(__ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__ >= 170400) || \
28 (defined(__ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__) && \
29 __ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__ >= 100400)
30# define _LIBCPP_USE_OS_SYNC
31# endif
32#endif
33
34#ifdef __linux__
35
36# include <linux/futex.h>
37# include <sys/syscall.h>
38# include <unistd.h>
39
40// libc++ uses SYS_futex as a universal syscall name. However, on 32 bit architectures
41// with a 64 bit time_t, we need to specify SYS_futex_time64.
42# if !defined(SYS_futex) && defined(SYS_futex_time64)
43# define SYS_futex SYS_futex_time64
44# endif
45# define _LIBCPP_FUTEX(...) syscall(SYS_futex, __VA_ARGS__)
46
47#elif defined(__FreeBSD__)
48
49# include <sys/types.h>
50# include <sys/umtx.h>
51
52# define _LIBCPP_FUTEX(...) syscall(SYS_futex, __VA_ARGS__)
53
54#elif defined(__OpenBSD__)
55
56# include <sys/futex.h>
57
58// OpenBSD has no indirect syscalls
59# define _LIBCPP_FUTEX(...) futex(__VA_ARGS__)
60
61#elif defined(_WIN32)
62
63# include <memory>
64# include <windows.h>
65
66#elif defined(__APPLE__)
67
68# if defined(_LIBCPP_USE_OS_SYNC)
69# include <os/os_sync_wait_on_address.h>
70# endif
71
72#elif defined(__Fuchsia__)
73
74# include <zircon/syscalls.h>
75
76#else // <- Add other operating systems here
77
78// Baseline needs no new headers
79
80# define _LIBCPP_FUTEX(...) syscall(SYS_futex, __VA_ARGS__)
81
82#endif
83
84_LIBCPP_PUSH_MACROS
85#include <__undef_macros>
86
87_LIBCPP_BEGIN_NAMESPACE_STD
88_LIBCPP_BEGIN_EXPLICIT_ABI_ANNOTATIONS
89
90struct NoTimeout {};
91
92#ifdef __linux__
93
94template <std::size_t _Size, class MaybeTimeout>
95static void __platform_wait_on_address(void const* __ptr, void const* __val, MaybeTimeout maybe_timeout_ns) {
96 static_assert(_Size == 4, "Can only wait on 4 bytes value");
97 alignas(__cxx_contention_t) char buffer[_Size];
98 std::memcpy(dest: &buffer, src: const_cast<const void*>(__val), n: _Size);
99 static constexpr timespec __default_timeout = {.tv_sec: 2, .tv_nsec: 0};
100 timespec __timeout;
101 if constexpr (is_same_v<MaybeTimeout, NoTimeout>) {
102 __timeout = __default_timeout;
103 } else {
104 __timeout.tv_sec = maybe_timeout_ns / 1'000'000'000;
105 __timeout.tv_nsec = maybe_timeout_ns % 1'000'000'000;
106 }
107 _LIBCPP_FUTEX(__ptr, FUTEX_WAIT_PRIVATE, *reinterpret_cast<__cxx_contention_t const*>(&buffer), &__timeout, 0, 0);
108}
109
110template <std::size_t _Size>
111static void __platform_wake_by_address(void const* __ptr, bool __notify_one) {
112 static_assert(_Size == 4, "Can only wake up on 4 bytes value");
113 _LIBCPP_FUTEX(__ptr, FUTEX_WAKE_PRIVATE, __notify_one ? 1 : INT_MAX, 0, 0, 0);
114}
115
116#elif defined(__APPLE__)
117
118# if defined(_LIBCPP_USE_OS_SYNC)
119
120template <std::size_t _Size, class MaybeTimeout>
121static void __platform_wait_on_address(void const* __ptr, void const* __val, MaybeTimeout maybe_timeout_ns) {
122 std::uint64_t __value = [__val]() -> std::uint64_t {
123 if constexpr (_Size == 4) {
124 std::uint32_t __result;
125 std::memcpy(&__result, __val, _Size);
126 return __result;
127 } else if constexpr (_Size == 8) {
128 std::uint64_t __result;
129 std::memcpy(&__result, __val, _Size);
130 return __result;
131 } else {
132 static_assert(false, "Can only wait on 8 bytes or 4 bytes value");
133 }
134 }();
135
136 if constexpr (is_same_v<MaybeTimeout, NoTimeout>) {
137 os_sync_wait_on_address(const_cast<void*>(__ptr), __value, _Size, OS_SYNC_WAIT_ON_ADDRESS_NONE);
138 } else {
139 // os_sync_wait_on_address_with_timeout returns EINVAL on a 0-ns timeout, so clamp to 1ns
140 // when an already-expired deadline produced a zero-duration timeout.
141 uint64_t __timeout_ns = std::max<uint64_t>(maybe_timeout_ns, 1);
142 os_sync_wait_on_address_with_timeout(
143 const_cast<void*>(__ptr),
144 __value,
145 _Size,
146 OS_SYNC_WAIT_ON_ADDRESS_NONE,
147 OS_CLOCK_MACH_ABSOLUTE_TIME,
148 __timeout_ns);
149 }
150}
151
152template <std::size_t _Size>
153static void __platform_wake_by_address(void const* __ptr, bool __notify_one) {
154 static_assert(_Size == 8 || _Size == 4, "Can only wake up on 8 bytes or 4 bytes value");
155 if (__notify_one)
156 os_sync_wake_by_address_any(const_cast<void*>(__ptr), _Size, OS_SYNC_WAKE_BY_ADDRESS_NONE);
157 else
158 os_sync_wake_by_address_all(const_cast<void*>(__ptr), _Size, OS_SYNC_WAKE_BY_ADDRESS_NONE);
159}
160
161# else // !_LIBCPP_USE_OS_SYNC -- fall back to the private __ulock API
162
163extern "C" int __ulock_wait(
164 uint32_t operation, void* addr, uint64_t value, uint32_t timeout); /* timeout is specified in microseconds */
165extern "C" int __ulock_wake(uint32_t operation, void* addr, uint64_t wake_value);
166
167// https://github.com/apple/darwin-xnu/blob/2ff845c2e033bd0ff64b5b6aa6063a1f8f65aa32/bsd/sys/ulock.h#L82
168# define UL_COMPARE_AND_WAIT 1
169# define UL_COMPARE_AND_WAIT64 5
170# define ULF_WAKE_ALL 0x00000100
171
172template <std::size_t _Size, class MaybeTimeout>
173static void __platform_wait_on_address(void const* __ptr, void const* __val, MaybeTimeout maybe_timeout_ns) {
174 auto __timeout_us = [&] {
175 if constexpr (is_same_v<MaybeTimeout, NoTimeout>) {
176 return uint32_t(0);
177 } else {
178 return std::max(static_cast<uint32_t>(maybe_timeout_ns / 1000), uint32_t(1));
179 }
180 }();
181
182 std::uint64_t __value = [__val]() -> std::uint64_t {
183 if constexpr (_Size == 4) {
184 std::uint32_t __result;
185 std::memcpy(&__result, __val, _Size);
186 return __result;
187 } else if constexpr (_Size == 8) {
188 std::uint64_t __result;
189 std::memcpy(&__result, __val, _Size);
190 return __result;
191 } else {
192 static_assert(false, "Can only wait on 8 bytes or 4 bytes value");
193 }
194 }();
195
196 __ulock_wait(
197 _Size == 4 ? UL_COMPARE_AND_WAIT : UL_COMPARE_AND_WAIT64, const_cast<void*>(__ptr), __value, __timeout_us);
198}
199
200template <std::size_t _Size>
201static void __platform_wake_by_address(void const* __ptr, bool __notify_one) {
202 static_assert(_Size == 8 || _Size == 4, "Can only wake up on 8 bytes or 4 bytes value");
203
204 if constexpr (_Size == 4)
205 __ulock_wake(UL_COMPARE_AND_WAIT | (__notify_one ? 0 : ULF_WAKE_ALL), const_cast<void*>(__ptr), 0);
206 else
207 __ulock_wake(UL_COMPARE_AND_WAIT64 | (__notify_one ? 0 : ULF_WAKE_ALL), const_cast<void*>(__ptr), 0);
208}
209
210# endif // _LIBCPP_USE_OS_SYNC
211
212#elif defined(__FreeBSD__) && __SIZEOF_LONG__ == 8
213/*
214 * Since __cxx_contention_t is int64_t even on 32bit FreeBSD
215 * platforms, we have to use umtx ops that work on the long type, and
216 * limit its use to architectures where long and int64_t are synonyms.
217 */
218
219template <std::size_t _Size, class MaybeTimeout>
220static void __platform_wait_on_address(void const* __ptr, void const* __val, MaybeTimeout maybe_timeout_ns) {
221 static_assert(_Size == 8, "Can only wait on 8 bytes value");
222 // At the moment, FreeBSD is stuck on stable ABI, which only supports platform wait with __cxx_contention_t
223 // It is safe to reinterpret_cast the val as it is ever going to be passed a __cxx_contention_t under this ABI
224 // If in the future FreeBSD decides to experiment unstable ABI to support more types, this cast will no longer be
225 // safe.
226 __cxx_contention_t value = *reinterpret_cast<const __cxx_contention_t*>(__val);
227 if constexpr (is_same_v<MaybeTimeout, NoTimeout>) {
228 _umtx_op(const_cast<void*>(__ptr), UMTX_OP_WAIT, value, nullptr, nullptr);
229 } else {
230 timespec timeout{};
231 timeout.tv_sec = maybe_timeout_ns / 1'000'000'000;
232 timeout.tv_nsec = maybe_timeout_ns % 1'000'000'000;
233
234 _umtx_op(const_cast<void*>(__ptr),
235 UMTX_OP_WAIT,
236 value,
237 reinterpret_cast<void*>(static_cast<uintptr_t>(sizeof(timeout))),
238 &timeout);
239 }
240}
241
242template <std::size_t _Size>
243static void __platform_wake_by_address(void const* __ptr, bool __notify_one) {
244 static_assert(_Size == 8, "Can only wake up on 8 bytes value");
245 _umtx_op(const_cast<void*>(__ptr), UMTX_OP_WAKE, __notify_one ? 1 : INT_MAX, nullptr, nullptr);
246}
247
248#elif defined(_WIN32)
249
250static void* win32_get_synch_api_function(const char* function_name) {
251 // Attempt to load the API set. Note that as per the Microsoft STL implementation, we assume this API is already
252 // loaded and accessible. While this isn't explicitly guaranteed by publicly available Win32 API documentation, it is
253 // true in practice, and may be guaranteed by internal documentation not released publicly. In any case the fact that
254 // the Microsoft STL made this assumption is reasonable basis to say that we can too. The alternative to this would be
255 // to use LoadLibrary, but then leak the module handle. We can't call FreeLibrary, as this would have to be triggered
256 // by a global static destructor, which would hang off DllMain, and calling FreeLibrary from DllMain is explicitly
257 // mentioned as not being allowed:
258 // https://learn.microsoft.com/en-us/windows/win32/dlls/dllmain
259 // Given the range of bad options here, we have chosen to mirror what Microsoft did, as it seems fair to assume that
260 // Microsoft will guarantee compatibility for us, as we are exposed to the same conditions as all existing Windows
261 // apps using the Microsoft STL VS2015/2017/2019/2022 runtimes, where Windows 7 support has not been excluded at
262 // compile time.
263 static auto module_handle = GetModuleHandleW(L"api-ms-win-core-synch-l1-2-0.dll");
264 if (module_handle == nullptr) {
265 return nullptr;
266 }
267
268 // Attempt to locate the function in the API and return the result to the caller. Note that the NULL return from this
269 // method is documented as being interchangeable with nullptr.
270 // https://devblogs.microsoft.com/oldnewthing/20180307-00/?p=98175
271 return reinterpret_cast<void*>(GetProcAddress(module_handle, function_name));
272}
273
274template <std::size_t _Size, class MaybeTimeout>
275static void __platform_wait_on_address(void const* __ptr, void const* __val, MaybeTimeout maybe_timeout_ns) {
276 static_assert(_Size == 8, "Can only wait on 8 bytes value");
277 // WaitOnAddress was added in Windows 8 (build 9200)
278 static auto wait_on_address =
279 reinterpret_cast<BOOL(WINAPI*)(void*, PVOID, SIZE_T, DWORD)>(win32_get_synch_api_function("WaitOnAddress"));
280 if (wait_on_address != nullptr) {
281 auto timeout_ms = [&]() -> DWORD {
282 if constexpr (is_same_v<MaybeTimeout, NoTimeout>) {
283 return INFINITE;
284 } else {
285 uint64_t ms = maybe_timeout_ns / 1'000'000;
286 if (ms == 0 && maybe_timeout_ns > 100'000)
287 // Round up to 1ms if requested between 100us - 1ms
288 return 1;
289
290 return static_cast<DWORD>(std::min(static_cast<uint64_t>(INFINITE), ms));
291 }
292 }();
293 wait_on_address(const_cast<void*>(__ptr), const_cast<void*>(__val), _Size, timeout_ms);
294 } else {
295 std::chrono::nanoseconds timeout = std::chrono::nanoseconds(0);
296 if constexpr (!is_same_v<MaybeTimeout, NoTimeout>) {
297 timeout = std::chrono::nanoseconds(maybe_timeout_ns);
298 }
299 __libcpp_thread_poll_with_backoff(
300 [=]() -> bool { return std::memcmp(const_cast<const void*>(__ptr), __val, _Size) != 0; },
301 __libcpp_timed_backoff_policy(),
302 timeout);
303 }
304}
305
306template <std::size_t _Size>
307static void __platform_wake_by_address(void const* __ptr, bool __notify_one) {
308 static_assert(_Size == 8, "Can only wake up on 8 bytes value");
309 if (__notify_one) {
310 // WakeByAddressSingle was added in Windows 8 (build 9200)
311 static auto wake_by_address_single =
312 reinterpret_cast<void(WINAPI*)(PVOID)>(win32_get_synch_api_function("WakeByAddressSingle"));
313 if (wake_by_address_single != nullptr) {
314 wake_by_address_single(const_cast<void*>(__ptr));
315 } else {
316 // The fallback implementation of waking does nothing, as the fallback wait implementation just does polling, so
317 // there's nothing to do here.
318 }
319 } else {
320 // WakeByAddressAll was added in Windows 8 (build 9200)
321 static auto wake_by_address_all =
322 reinterpret_cast<void(WINAPI*)(PVOID)>(win32_get_synch_api_function("WakeByAddressAll"));
323 if (wake_by_address_all != nullptr) {
324 wake_by_address_all(const_cast<void*>(__ptr));
325 } else {
326 // The fallback implementation of waking does nothing, as the fallback wait implementation just does polling, so
327 // there's nothing to do here.
328 }
329 }
330}
331
332#elif defined(__Fuchsia__)
333
334template <std::size_t _Size>
335static inline zx_futex_t const* __get_zx_futex(void const* __ptr) {
336 static_assert(_Size == sizeof(zx_futex_t), "Can only wait/wake on zx_futex_t-compatible value");
337
338 // Implicitly link against the vDSO system call ABI without requiring the
339 // final link to specify -lzircon explicitly when statically linking libc++.
340# pragma comment(lib, "zircon")
341
342 return reinterpret_cast<zx_futex_t const*>(__ptr);
343}
344
345template <std::size_t _Size, class MaybeTimeout>
346static void __platform_wait_on_address(void const* __ptr, void const* __val, MaybeTimeout maybe_timeout_ns) {
347 zx_futex_t val;
348 std::memcpy(&val, __val, _Size);
349 zx_instant_mono_t deadline;
350 if constexpr (is_same_v<MaybeTimeout, NoTimeout>) {
351 deadline = ZX_TIME_INFINITE;
352 } else {
353 deadline = _zx_deadline_after(maybe_timeout_ns);
354 }
355 _zx_futex_wait(__get_zx_futex<_Size>(__ptr), val, ZX_HANDLE_INVALID, deadline);
356}
357
358template <std::size_t _Size>
359static void __platform_wake_by_address(void const* __ptr, bool __notify_one) {
360 _zx_futex_wake(__get_zx_futex<_Size>(__ptr), __notify_one ? 1 : UINT32_MAX);
361}
362
363#else // <- Add other operating systems here
364
365// Baseline is just a timed backoff
366
367template <std::size_t _Size, class MaybeTimeout>
368static void __platform_wait_on_address(void const* __ptr, void const* __val, MaybeTimeout maybe_timeout_ns) {
369 std::chrono::nanoseconds timeout = std::chrono::nanoseconds(0);
370 if constexpr (!is_same_v<MaybeTimeout, NoTimeout>) {
371 timeout = std::chrono::nanoseconds(maybe_timeout_ns);
372 }
373 __libcpp_thread_poll_with_backoff(
374 [=]() -> bool { return std::memcmp(const_cast<const void*>(__ptr), __val, _Size) != 0; },
375 __libcpp_timed_backoff_policy(),
376 timeout);
377}
378
379template <std::size_t _Size>
380static void __platform_wake_by_address(void const*, bool) {}
381
382#endif // __linux__
383
384// =============================
385// Local hidden helper functions
386// =============================
387
388/* Given an atomic to track contention and an atomic to actually wait on, which may be
389 the same atomic, we try to detect contention to avoid spuriously calling the platform. */
390
391template <std::size_t _Size>
392static void
393__contention_notify(__cxx_atomic_contention_t* __waiter_count, void const* __address_to_notify, bool __notify_one) {
394 if (0 != __cxx_atomic_load(a: __waiter_count, order: memory_order_seq_cst))
395 // We only call 'wake' if we consumed a contention bit here.
396 __platform_wake_by_address<_Size>(__address_to_notify, __notify_one);
397}
398
399template <std::size_t _Size, class MaybeTimeout>
400static void __contention_wait(__cxx_atomic_contention_t* __waiter_count,
401 void const* __address_to_wait,
402 void const* __old_value,
403 MaybeTimeout maybe_timeout_ns) {
404 __cxx_atomic_fetch_add(a: __waiter_count, delta: __cxx_contention_t(1), order: memory_order_relaxed);
405 // https://llvm.org/PR109290
406 // There are no platform guarantees of a memory barrier in the platform wait implementation
407 __cxx_atomic_thread_fence(order: memory_order_seq_cst);
408 // We sleep as long as the monitored value hasn't changed.
409 __platform_wait_on_address<_Size>(__address_to_wait, __old_value, maybe_timeout_ns);
410 __cxx_atomic_fetch_sub(a: __waiter_count, delta: __cxx_contention_t(1), order: memory_order_release);
411}
412
413static constexpr size_t __contention_table_size = (1 << 8); /* < there's no magic in this number */
414
415static constexpr hash<void const*> __contention_hasher;
416
417// Waiter count table for all atomics with the correct size that use itself as the wait/notify address.
418
419struct alignas(
420 std::hardware_constructive_interference_size) /* aim to avoid false sharing */ __contention_state_native {
421 __cxx_atomic_contention_t __waiter_count;
422 constexpr __contention_state_native() : __waiter_count(0) {}
423};
424
425static __contention_state_native __contention_table_native[__contention_table_size];
426
427static __cxx_atomic_contention_t* __get_native_waiter_count(void const* p) {
428 return &__contention_table_native[__contention_hasher(p) & (__contention_table_size - 1)].__waiter_count;
429}
430
431// Global contention table for all atomics with the wrong size that use the global table's atomic as wait/notify
432// address.
433
434struct alignas(
435 std::hardware_constructive_interference_size) /* aim to avoid false sharing */ __contention_state_global {
436 __cxx_atomic_contention_t __waiter_count;
437 __cxx_atomic_contention_t __platform_state;
438 constexpr __contention_state_global() : __waiter_count(0), __platform_state(0) {}
439};
440
441static __contention_state_global __contention_table_global[__contention_table_size];
442
443static __contention_state_global* __get_global_contention_state(void const* p) {
444 return &__contention_table_global[__contention_hasher(p) & (__contention_table_size - 1)];
445}
446
447/* When the incoming atomic is the wrong size for the platform wait size, need to
448 launder the value sequence through an atomic from our table. */
449
450static void __atomic_notify_global_table(void const* __location) {
451 auto const __entry = __get_global_contention_state(p: __location);
452 // The value sequence laundering happens on the next line below.
453 __cxx_atomic_fetch_add(a: &__entry->__platform_state, delta: __cxx_contention_t(1), order: memory_order_seq_cst);
454 __contention_notify<sizeof(__cxx_atomic_contention_t)>(
455 waiter_count: &__entry->__waiter_count, address_to_notify: &__entry->__platform_state, notify_one: false /* when laundering, we can't handle notify_one */);
456}
457
458// =============================
459// New dylib exported symbols
460// =============================
461
462// global
463_LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __atomic_monitor_global(void const* __location) noexcept {
464 auto const __entry = __get_global_contention_state(p: __location);
465 return __cxx_atomic_load(a: &__entry->__platform_state, order: memory_order_acquire);
466}
467
468_LIBCPP_EXPORTED_FROM_ABI void
469__atomic_wait_global_table(void const* __location, __cxx_contention_t __old_value) noexcept {
470 auto const __entry = __get_global_contention_state(p: __location);
471 __contention_wait<sizeof(__cxx_atomic_contention_t)>(
472 waiter_count: &__entry->__waiter_count, address_to_wait: &__entry->__platform_state, old_value: &__old_value, maybe_timeout_ns: NoTimeout{});
473}
474
475_LIBCPP_EXPORTED_FROM_ABI void __atomic_wait_global_table_with_timeout(
476 void const* __location, __cxx_contention_t __old_value, uint64_t __timeout_ns) _NOEXCEPT {
477 auto const __entry = __get_global_contention_state(p: __location);
478 __contention_wait<sizeof(__cxx_atomic_contention_t)>(
479 waiter_count: &__entry->__waiter_count, address_to_wait: &__entry->__platform_state, old_value: &__old_value, maybe_timeout_ns: __timeout_ns);
480}
481
482_LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_one_global_table(void const* __location) noexcept {
483 __atomic_notify_global_table(__location);
484}
485
486_LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_all_global_table(void const* __location) noexcept {
487 __atomic_notify_global_table(__location);
488}
489
490// native
491
492template <std::size_t _Size>
493_LIBCPP_EXPORTED_FROM_ABI void __atomic_wait_native(void const* __address, void const* __old_value) noexcept {
494 __contention_wait<_Size>(__get_native_waiter_count(p: __address), __address, __old_value, NoTimeout{});
495}
496
497template <std::size_t _Size>
498_LIBCPP_EXPORTED_FROM_ABI void
499__atomic_wait_native_with_timeout(void const* __address, void const* __old_value, uint64_t __timeout_ns) noexcept {
500 __contention_wait<_Size>(__get_native_waiter_count(p: __address), __address, __old_value, __timeout_ns);
501}
502
503template <std::size_t _Size>
504_LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_one_native(void const* __location) noexcept {
505 __contention_notify<_Size>(__get_native_waiter_count(p: __location), __location, true);
506}
507
508template <std::size_t _Size>
509_LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_all_native(void const* __location) noexcept {
510 __contention_notify<_Size>(__get_native_waiter_count(p: __location), __location, false);
511}
512
513// ==================================================
514// Instantiation of the templates with supported size
515// ==================================================
516
517#if defined(_LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE)
518
519# define _INSTANTIATE(_SIZE) \
520 template _LIBCPP_EXPORTED_FROM_ABI void __atomic_wait_native<_SIZE>(void const*, void const*) noexcept; \
521 template _LIBCPP_EXPORTED_FROM_ABI void __atomic_wait_native_with_timeout<_SIZE>( \
522 void const*, void const*, uint64_t) noexcept; \
523 template _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_one_native<_SIZE>(void const*) noexcept; \
524 template _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_all_native<_SIZE>(void const*) noexcept;
525
526_LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_INSTANTIATE)
527
528# undef _INSTANTIATE
529
530#else // _LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE
531
532template _LIBCPP_EXPORTED_FROM_ABI void
533__atomic_wait_native<sizeof(__cxx_contention_t)>(void const* __address, void const* __old_value) noexcept;
534
535template _LIBCPP_EXPORTED_FROM_ABI void __atomic_wait_native_with_timeout<sizeof(__cxx_contention_t)>(
536 void const* __address, void const* __old_value, uint64_t) noexcept;
537
538template _LIBCPP_EXPORTED_FROM_ABI void
539__atomic_notify_one_native<sizeof(__cxx_contention_t)>(void const* __location) noexcept;
540
541template _LIBCPP_EXPORTED_FROM_ABI void
542__atomic_notify_all_native<sizeof(__cxx_contention_t)>(void const* __location) noexcept;
543
544#endif // _LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE
545
546// =============================================================
547// Old dylib exported symbols, for backwards compatibility
548// =============================================================
549_LIBCPP_DIAGNOSTIC_PUSH
550_LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wmissing-prototypes")
551
552_LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(void const volatile* __location) noexcept {
553 __atomic_notify_global_table(location: const_cast<void const*>(__location));
554}
555
556_LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(void const volatile* __location) noexcept {
557 __atomic_notify_global_table(location: const_cast<void const*>(__location));
558}
559
560_LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __libcpp_atomic_monitor(void const volatile* __location) noexcept {
561 auto const __entry = __get_global_contention_state(p: const_cast<void const*>(__location));
562 return __cxx_atomic_load(a: &__entry->__platform_state, order: memory_order_acquire);
563}
564
565_LIBCPP_EXPORTED_FROM_ABI void
566__libcpp_atomic_wait(void const volatile* __location, __cxx_contention_t __old_value) noexcept {
567 auto const __entry = __get_global_contention_state(p: const_cast<void const*>(__location));
568 __contention_wait<sizeof(__cxx_atomic_contention_t)>(
569 waiter_count: &__entry->__waiter_count, address_to_wait: &__entry->__platform_state, old_value: &__old_value, maybe_timeout_ns: NoTimeout{});
570}
571
572_LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(__cxx_atomic_contention_t const volatile* __location) noexcept {
573 auto __location_cast = const_cast<const void*>(static_cast<const volatile void*>(__location));
574 __contention_notify<sizeof(__cxx_atomic_contention_t)>(
575 waiter_count: __get_native_waiter_count(p: __location_cast), address_to_notify: __location_cast, notify_one: true);
576}
577
578_LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(__cxx_atomic_contention_t const volatile* __location) noexcept {
579 auto __location_cast = const_cast<const void*>(static_cast<const volatile void*>(__location));
580 __contention_notify<sizeof(__cxx_atomic_contention_t)>(
581 waiter_count: __get_native_waiter_count(p: __location_cast), address_to_notify: __location_cast, notify_one: false);
582}
583
584_LIBCPP_EXPORTED_FROM_ABI void
585__libcpp_atomic_wait(__cxx_atomic_contention_t const volatile* __location, __cxx_contention_t __old_value) noexcept {
586 auto __location_cast = const_cast<const void*>(static_cast<const volatile void*>(__location));
587 __contention_wait<sizeof(__cxx_atomic_contention_t)>(
588 waiter_count: __get_native_waiter_count(p: __location_cast), address_to_wait: __location_cast, old_value: &__old_value, maybe_timeout_ns: NoTimeout{});
589}
590
591// this function is even unused in the old ABI
592_LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t
593__libcpp_atomic_monitor(__cxx_atomic_contention_t const volatile* __location) noexcept {
594 return __cxx_atomic_load(a: __location, order: memory_order_acquire);
595}
596
597_LIBCPP_DIAGNOSTIC_POP
598
599_LIBCPP_END_EXPLICIT_ABI_ANNOTATIONS
600_LIBCPP_END_NAMESPACE_STD
601
602_LIBCPP_POP_MACROS
603