1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include <__thread/timed_backoff_policy.h>
10#include <atomic>
11#include <climits>
12#include <cstddef>
13#include <cstdint>
14#include <cstring>
15#include <functional>
16#include <new>
17#include <thread>
18#include <type_traits>
19
20// Determine whether to use the public os_sync API on Apple platforms based on the deployment target.
21// Otherwise we fall back to the private __ulock API.
22#if defined(__APPLE__)
23# if (defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && \
24 __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 140400) || \
25 (defined(__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__) && \
26 __ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__ >= 170400) || \
27 (defined(__ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__ >= 170400) || \
28 (defined(__ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__) && \
29 __ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__ >= 100400)
30# define _LIBCPP_USE_OS_SYNC
31# endif
32#endif
33
34#ifdef __linux__
35
36# include <linux/futex.h>
37# include <sys/syscall.h>
38# include <unistd.h>
39
40// libc++ uses SYS_futex as a universal syscall name. However, on 32 bit architectures
41// with a 64 bit time_t, we need to specify SYS_futex_time64.
42# if !defined(SYS_futex) && defined(SYS_futex_time64)
43# define SYS_futex SYS_futex_time64
44# endif
45# define _LIBCPP_FUTEX(...) syscall(SYS_futex, __VA_ARGS__)
46
47#elif defined(__FreeBSD__)
48
49# include <sys/types.h>
50# include <sys/umtx.h>
51
52# define _LIBCPP_FUTEX(...) syscall(SYS_futex, __VA_ARGS__)
53
54#elif defined(__OpenBSD__)
55
56# include <sys/futex.h>
57
58// OpenBSD has no indirect syscalls
59# define _LIBCPP_FUTEX(...) futex(__VA_ARGS__)
60
61#elif defined(_WIN32)
62
63# include <memory>
64# include <windows.h>
65
66#elif defined(__APPLE__)
67
68# if defined(_LIBCPP_USE_OS_SYNC)
69# include <os/os_sync_wait_on_address.h>
70# endif
71
72#elif defined(__Fuchsia__)
73
74# include <zircon/syscalls.h>
75
76#else // <- Add other operating systems here
77
78// Baseline needs no new headers
79
80# define _LIBCPP_FUTEX(...) syscall(SYS_futex, __VA_ARGS__)
81
82#endif
83
84_LIBCPP_PUSH_MACROS
85#include <__undef_macros>
86
87_LIBCPP_BEGIN_NAMESPACE_STD
88_LIBCPP_BEGIN_EXPLICIT_ABI_ANNOTATIONS
89
90struct NoTimeout {};
91
92#ifdef __linux__
93
94template <std::size_t _Size, class MaybeTimeout>
95static void __platform_wait_on_address(void const* __ptr, void const* __val, MaybeTimeout maybe_timeout_ns) {
96 static_assert(_Size == 4, "Can only wait on 4 bytes value");
97 alignas(__cxx_contention_t) char buffer[_Size];
98 std::memcpy(dest: &buffer, src: const_cast<const void*>(__val), n: _Size);
99 static constexpr timespec __default_timeout = {.tv_sec: 2, .tv_nsec: 0};
100 timespec __timeout;
101 if constexpr (is_same_v<MaybeTimeout, NoTimeout>) {
102 __timeout = __default_timeout;
103 } else {
104 __timeout.tv_sec = maybe_timeout_ns / 1'000'000'000;
105 __timeout.tv_nsec = maybe_timeout_ns % 1'000'000'000;
106 }
107 _LIBCPP_FUTEX(__ptr, FUTEX_WAIT_PRIVATE, *reinterpret_cast<__cxx_contention_t const*>(&buffer), &__timeout, 0, 0);
108}
109
110template <std::size_t _Size>
111static void __platform_wake_by_address(void const* __ptr, bool __notify_one) {
112 static_assert(_Size == 4, "Can only wake up on 4 bytes value");
113 _LIBCPP_FUTEX(__ptr, FUTEX_WAKE_PRIVATE, __notify_one ? 1 : INT_MAX, 0, 0, 0);
114}
115
116#elif defined(__APPLE__)
117
118# if defined(_LIBCPP_USE_OS_SYNC)
119
120template <std::size_t _Size, class MaybeTimeout>
121static void __platform_wait_on_address(void const* __ptr, void const* __val, MaybeTimeout maybe_timeout_ns) {
122 std::uint64_t __value = [__val]() -> std::uint64_t {
123 if constexpr (_Size == 4) {
124 std::uint32_t __result;
125 std::memcpy(&__result, __val, _Size);
126 return __result;
127 } else if constexpr (_Size == 8) {
128 std::uint64_t __result;
129 std::memcpy(&__result, __val, _Size);
130 return __result;
131 } else {
132 static_assert(false, "Can only wait on 8 bytes or 4 bytes value");
133 }
134 }();
135
136 if constexpr (is_same_v<MaybeTimeout, NoTimeout>) {
137 os_sync_wait_on_address(const_cast<void*>(__ptr), __value, _Size, OS_SYNC_WAIT_ON_ADDRESS_NONE);
138 } else {
139 // os_sync_wait_on_address_with_timeout returns EINVAL on a 0-ns timeout, so clamp to 1ns
140 // when an already-expired deadline produced a zero-duration timeout.
141 uint64_t __timeout_ns = std::max<uint64_t>(maybe_timeout_ns, 1);
142 os_sync_wait_on_address_with_timeout(
143 const_cast<void*>(__ptr),
144 __value,
145 _Size,
146 OS_SYNC_WAIT_ON_ADDRESS_NONE,
147 OS_CLOCK_MACH_ABSOLUTE_TIME,
148 __timeout_ns);
149 }
150}
151
152template <std::size_t _Size>
153static void __platform_wake_by_address(void const* __ptr, bool __notify_one) {
154 static_assert(_Size == 8 || _Size == 4, "Can only wake up on 8 bytes or 4 bytes value");
155 if (__notify_one)
156 os_sync_wake_by_address_any(const_cast<void*>(__ptr), _Size, OS_SYNC_WAKE_BY_ADDRESS_NONE);
157 else
158 os_sync_wake_by_address_all(const_cast<void*>(__ptr), _Size, OS_SYNC_WAKE_BY_ADDRESS_NONE);
159}
160
161# else // !_LIBCPP_USE_OS_SYNC -- fall back to the private __ulock API
162
163extern "C" int __ulock_wait(
164 uint32_t operation, void* addr, uint64_t value, uint32_t timeout); /* timeout is specified in microseconds */
165extern "C" int __ulock_wake(uint32_t operation, void* addr, uint64_t wake_value);
166
167// https://github.com/apple/darwin-xnu/blob/2ff845c2e033bd0ff64b5b6aa6063a1f8f65aa32/bsd/sys/ulock.h#L82
168# define UL_COMPARE_AND_WAIT 1
169# define UL_COMPARE_AND_WAIT64 5
170# define ULF_WAKE_ALL 0x00000100
171
172template <std::size_t _Size, class MaybeTimeout>
173static void __platform_wait_on_address(void const* __ptr, void const* __val, MaybeTimeout maybe_timeout_ns) {
174 auto __timeout_us = [&] {
175 if constexpr (is_same_v<MaybeTimeout, NoTimeout>) {
176 return uint32_t(0);
177 } else {
178 return std::max(static_cast<uint32_t>(maybe_timeout_ns / 1000), uint32_t(1));
179 }
180 }();
181
182 std::uint64_t __value = [__val]() -> std::uint64_t {
183 if constexpr (_Size == 4) {
184 std::uint32_t __result;
185 std::memcpy(&__result, __val, _Size);
186 return __result;
187 } else if constexpr (_Size == 8) {
188 std::uint64_t __result;
189 std::memcpy(&__result, __val, _Size);
190 return __result;
191 } else {
192 static_assert(false, "Can only wait on 8 bytes or 4 bytes value");
193 }
194 }();
195
196 __ulock_wait(
197 _Size == 4 ? UL_COMPARE_AND_WAIT : UL_COMPARE_AND_WAIT64, const_cast<void*>(__ptr), __value, __timeout_us);
198}
199
200template <std::size_t _Size>
201static void __platform_wake_by_address(void const* __ptr, bool __notify_one) {
202 static_assert(_Size == 8 || _Size == 4, "Can only wake up on 8 bytes or 4 bytes value");
203
204 if constexpr (_Size == 4)
205 __ulock_wake(UL_COMPARE_AND_WAIT | (__notify_one ? 0 : ULF_WAKE_ALL), const_cast<void*>(__ptr), 0);
206 else
207 __ulock_wake(UL_COMPARE_AND_WAIT64 | (__notify_one ? 0 : ULF_WAKE_ALL), const_cast<void*>(__ptr), 0);
208}
209
210# endif // _LIBCPP_USE_OS_SYNC
211
212#elif defined(__FreeBSD__) && __SIZEOF_LONG__ == 8
213/*
214 * Since __cxx_contention_t is int64_t even on 32bit FreeBSD
215 * platforms, we have to use umtx ops that work on the long type, and
216 * limit its use to architectures where long and int64_t are synonyms.
217 */
218
219template <std::size_t _Size, class MaybeTimeout>
220static void __platform_wait_on_address(void const* __ptr, void const* __val, MaybeTimeout maybe_timeout_ns) {
221 static_assert(_Size == 8, "Can only wait on 8 bytes value");
222 // At the moment, FreeBSD is stuck on stable ABI, which only supports platform wait with __cxx_contention_t
223 // It is safe to reinterpret_cast the val as it is ever going to be passed a __cxx_contention_t under this ABI
224 // If in the future FreeBSD decides to experiment unstable ABI to support more types, this cast will no longer be
225 // safe.
226 __cxx_contention_t value = *reinterpret_cast<const __cxx_contention_t*>(__val);
227 if constexpr (is_same_v<MaybeTimeout, NoTimeout>) {
228 _umtx_op(const_cast<void*>(__ptr), UMTX_OP_WAIT, value, nullptr, nullptr);
229 } else {
230 timespec timeout{};
231 timeout.tv_sec = maybe_timeout_ns / 1'000'000'000;
232 timeout.tv_nsec = maybe_timeout_ns % 1'000'000'000;
233
234 _umtx_op(const_cast<void*>(__ptr),
235 UMTX_OP_WAIT,
236 value,
237 reinterpret_cast<void*>(static_cast<uintptr_t>(sizeof(timeout))),
238 &timeout);
239 }
240}
241
242template <std::size_t _Size>
243static void __platform_wake_by_address(void const* __ptr, bool __notify_one) {
244 static_assert(_Size == 8, "Can only wake up on 8 bytes value");
245 _umtx_op(const_cast<void*>(__ptr), UMTX_OP_WAKE, __notify_one ? 1 : INT_MAX, nullptr, nullptr);
246}
247
248#elif defined(_WIN32)
249
250static void* win32_get_synch_api_function(const char* function_name) {
251 // Attempt to load the API set. Note that as per the Microsoft STL implementation, we assume this API is already
252 // loaded and accessible. While this isn't explicitly guaranteed by publicly available Win32 API documentation, it is
253 // true in practice, and may be guaranteed by internal documentation not released publicly. In any case the fact that
254 // the Microsoft STL made this assumption is reasonable basis to say that we can too. The alternative to this would be
255 // to use LoadLibrary, but then leak the module handle. We can't call FreeLibrary, as this would have to be triggered
256 // by a global static destructor, which would hang off DllMain, and calling FreeLibrary from DllMain is explicitly
257 // mentioned as not being allowed:
258 // https://learn.microsoft.com/en-us/windows/win32/dlls/dllmain
259 // Given the range of bad options here, we have chosen to mirror what Microsoft did, as it seems fair to assume that
260 // Microsoft will guarantee compatibility for us, as we are exposed to the same conditions as all existing Windows
261 // apps using the Microsoft STL VS2015/2017/2019/2022 runtimes, where Windows 7 support has not been excluded at
262 // compile time.
263 static auto module_handle = GetModuleHandleW(L"api-ms-win-core-synch-l1-2-0.dll");
264 if (module_handle == nullptr) {
265 return nullptr;
266 }
267
268 // Attempt to locate the function in the API and return the result to the caller. Note that the NULL return from this
269 // method is documented as being interchangeable with nullptr.
270 // https://devblogs.microsoft.com/oldnewthing/20180307-00/?p=98175
271 return reinterpret_cast<void*>(GetProcAddress(module_handle, function_name));
272}
273
274template <std::size_t _Size, class MaybeTimeout>
275static void __platform_wait_on_address(void const* __ptr, void const* __val, MaybeTimeout maybe_timeout_ns) {
276 static_assert(_Size == 8, "Can only wait on 8 bytes value");
277 // WaitOnAddress was added in Windows 8 (build 9200)
278 static auto wait_on_address =
279 reinterpret_cast<BOOL(WINAPI*)(void*, PVOID, SIZE_T, DWORD)>(win32_get_synch_api_function("WaitOnAddress"));
280 if (wait_on_address != nullptr) {
281 auto timeout_ms = [&]() -> DWORD {
282 if constexpr (is_same_v<MaybeTimeout, NoTimeout>) {
283 return INFINITE;
284 } else {
285 uint64_t ms = maybe_timeout_ns / 1'000'000;
286 if (ms == 0 && maybe_timeout_ns > 100'000)
287 // Round up to 1ms if requested between 100us - 1ms
288 return 1;
289
290 return static_cast<DWORD>(std::min(static_cast<uint64_t>(INFINITE), ms));
291 }
292 }();
293 wait_on_address(const_cast<void*>(__ptr), const_cast<void*>(__val), _Size, timeout_ms);
294 } else {
295 std::chrono::nanoseconds timeout = std::chrono::nanoseconds(0);
296 if constexpr (!is_same_v<MaybeTimeout, NoTimeout>) {
297 timeout = std::chrono::nanoseconds(maybe_timeout_ns);
298 }
299 __libcpp_thread_poll_with_backoff(
300 [=]() -> bool { return std::memcmp(const_cast<const void*>(__ptr), __val, _Size) != 0; },
301 __libcpp_timed_backoff_policy(),
302 timeout);
303 }
304}
305
306template <std::size_t _Size>
307static void __platform_wake_by_address(void const* __ptr, bool __notify_one) {
308 static_assert(_Size == 8, "Can only wake up on 8 bytes value");
309 if (__notify_one) {
310 // WakeByAddressSingle was added in Windows 8 (build 9200)
311 static auto wake_by_address_single =
312 reinterpret_cast<void(WINAPI*)(PVOID)>(win32_get_synch_api_function("WakeByAddressSingle"));
313 if (wake_by_address_single != nullptr) {
314 wake_by_address_single(const_cast<void*>(__ptr));
315 } else {
316 // The fallback implementation of waking does nothing, as the fallback wait implementation just does polling, so
317 // there's nothing to do here.
318 }
319 } else {
320 // WakeByAddressAll was added in Windows 8 (build 9200)
321 static auto wake_by_address_all =
322 reinterpret_cast<void(WINAPI*)(PVOID)>(win32_get_synch_api_function("WakeByAddressAll"));
323 if (wake_by_address_all != nullptr) {
324 wake_by_address_all(const_cast<void*>(__ptr));
325 } else {
326 // The fallback implementation of waking does nothing, as the fallback wait implementation just does polling, so
327 // there's nothing to do here.
328 }
329 }
330}
331
332#elif defined(__Fuchsia__)
333
334template <std::size_t _Size>
335static inline zx_futex_t const* __get_zx_futex(void const* __ptr) {
336 static_assert(_Size == sizeof(zx_futex_t), "Can only wait/wake on zx_futex_t-compatible value");
337
338 // Implicitly link against the vDSO system call ABI without requiring the
339 // final link to specify -lzircon explicitly when statically linking libc++.
340# pragma comment(lib, "zircon")
341
342 return reinterpret_cast<zx_futex_t const*>(__ptr);
343}
344
345template <std::size_t _Size, class MaybeTimeout>
346static void __platform_wait_on_address(void const* __ptr, void const* __val, MaybeTimeout maybe_timeout_ns) {
347 zx_futex_t val;
348 std::memcpy(&val, __val, _Size);
349 zx_instant_mono_t deadline;
350 if constexpr (is_same_v<MaybeTimeout, NoTimeout>) {
351 deadline = ZX_TIME_INFINITE;
352 } else {
353 deadline = _zx_deadline_after(maybe_timeout_ns);
354 }
355 _zx_futex_wait(__get_zx_futex<_Size>(__ptr), val, ZX_HANDLE_INVALID, deadline);
356}
357
358template <std::size_t _Size>
359static void __platform_wake_by_address(void const* __ptr, bool __notify_one) {
360 _zx_futex_wake(__get_zx_futex<_Size>(__ptr), __notify_one ? 1 : UINT32_MAX);
361}
362
363#else // <- Add other operating systems here
364
365// Baseline is just a timed backoff
366
367template <std::size_t _Size, class MaybeTimeout>
368static void __platform_wait_on_address(void const* __ptr, void const* __val, MaybeTimeout maybe_timeout_ns) {
369 std::chrono::nanoseconds timeout = std::chrono::nanoseconds(0);
370 if constexpr (!is_same_v<MaybeTimeout, NoTimeout>) {
371 timeout = std::chrono::nanoseconds(maybe_timeout_ns);
372 }
373 __libcpp_thread_poll_with_backoff(
374 [=]() -> bool { return std::memcmp(const_cast<const void*>(__ptr), __val, _Size) != 0; },
375 __libcpp_timed_backoff_policy(),
376 timeout);
377}
378
379template <std::size_t _Size>
380static void __platform_wake_by_address(void const*, bool) {}
381
382#endif // __linux__
383
384// =============================
385// Local hidden helper functions
386// =============================
387
388/* Given an atomic to track contention and an atomic to actually wait on, which may be
389 the same atomic, we try to detect contention to avoid spuriously calling the platform. */
390
391template <std::size_t _Size>
392static void
393__contention_notify(__cxx_atomic_contention_t* __waiter_count, void const* __address_to_notify, bool __notify_one) {
394 if (0 != __cxx_atomic_load(a: __waiter_count, order: memory_order_seq_cst))
395 // We only call 'wake' if we consumed a contention bit here.
396 __platform_wake_by_address<_Size>(__address_to_notify, __notify_one);
397}
398
399template <std::size_t _Size, class MaybeTimeout>
400static void __contention_wait(__cxx_atomic_contention_t* __waiter_count,
401 void const* __address_to_wait,
402 void const* __old_value,
403 MaybeTimeout maybe_timeout_ns) {
404 __cxx_atomic_fetch_add(a: __waiter_count, delta: __cxx_contention_t(1), order: memory_order_relaxed);
405 // https://llvm.org/PR109290
406 // There are no platform guarantees of a memory barrier in the platform wait implementation
407 __cxx_atomic_thread_fence(order: memory_order_seq_cst);
408 // We sleep as long as the monitored value hasn't changed.
409 __platform_wait_on_address<_Size>(__address_to_wait, __old_value, maybe_timeout_ns);
410 __cxx_atomic_fetch_sub(a: __waiter_count, delta: __cxx_contention_t(1), order: memory_order_release);
411}
412
413static constexpr size_t __contention_table_size = (1 << 8); /* < there's no magic in this number */
414
415static constexpr hash<void const*> __contention_hasher;
416
417// Unified contention table shared by both the native path (atomics whose size matches the platform
418// wait size and are waited on directly) and the global path (atomics with the wrong size, which use
419// the global table's atomic as the wait/notify address). The native path only reads/writes the native
420// waiter count, and the global path uses the global waiter count and the platform state.
421//
422// We use a single table for both the native and non-native states to minimize the size impact, since
423// we would have plenty of unused padding otherwise. This technically means that if both a native and
424// a non-native atomic fall into the same bucket and are reading/writing waiter counts on the same
425// cache line, we would end up in a "false-sharing" situation. However, in practice, it would require
426// both native and non-native atomics to be used concurrently, and to fall in the same bucket, which
427// is expected to be unlikely.
428struct alignas(std::hardware_destructive_interference_size) /* aim to avoid false sharing */ __contention_state {
429 __cxx_atomic_contention_t __waiter_count_native;
430 __cxx_atomic_contention_t __waiter_count_global;
431 __cxx_atomic_contention_t __platform_state;
432 constexpr __contention_state() : __waiter_count_native(0), __waiter_count_global(0), __platform_state(0) {}
433};
434
435static constinit __contention_state __contention_table[__contention_table_size];
436
437static __contention_state* __get_contention_state(void const* p) {
438 return &__contention_table[__contention_hasher(p) & (__contention_table_size - 1)];
439}
440
441/* When the incoming atomic is the wrong size for the platform wait size, need to
442 launder the value sequence through an atomic from our table. */
443
444static void __atomic_notify_global_table(void const* __location) {
445 auto const __entry = __get_contention_state(p: __location);
446 // The value sequence laundering happens on the next line below.
447 __cxx_atomic_fetch_add(a: &__entry->__platform_state, delta: __cxx_contention_t(1), order: memory_order_seq_cst);
448 __contention_notify<sizeof(__cxx_atomic_contention_t)>(
449 waiter_count: &__entry->__waiter_count_global,
450 address_to_notify: &__entry->__platform_state,
451 notify_one: false /* when laundering, we can't handle notify_one */);
452}
453
454// =============================
455// New dylib exported symbols
456// =============================
457
458// global
459_LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __atomic_monitor_global(void const* __location) noexcept {
460 auto const __entry = __get_contention_state(p: __location);
461 return __cxx_atomic_load(a: &__entry->__platform_state, order: memory_order_acquire);
462}
463
464_LIBCPP_EXPORTED_FROM_ABI void
465__atomic_wait_global_table(void const* __location, __cxx_contention_t __old_value) noexcept {
466 auto const __entry = __get_contention_state(p: __location);
467 __contention_wait<sizeof(__cxx_atomic_contention_t)>(
468 waiter_count: &__entry->__waiter_count_global, address_to_wait: &__entry->__platform_state, old_value: &__old_value, maybe_timeout_ns: NoTimeout{});
469}
470
471_LIBCPP_EXPORTED_FROM_ABI void __atomic_wait_global_table_with_timeout(
472 void const* __location, __cxx_contention_t __old_value, uint64_t __timeout_ns) _NOEXCEPT {
473 auto const __entry = __get_contention_state(p: __location);
474 __contention_wait<sizeof(__cxx_atomic_contention_t)>(
475 waiter_count: &__entry->__waiter_count_global, address_to_wait: &__entry->__platform_state, old_value: &__old_value, maybe_timeout_ns: __timeout_ns);
476}
477
478_LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_one_global_table(void const* __location) noexcept {
479 __atomic_notify_global_table(__location);
480}
481
482_LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_all_global_table(void const* __location) noexcept {
483 __atomic_notify_global_table(__location);
484}
485
486// native
487
488template <std::size_t _Size>
489_LIBCPP_EXPORTED_FROM_ABI void __atomic_wait_native(void const* __address, void const* __old_value) noexcept {
490 __contention_wait<_Size>(
491 &__get_contention_state(p: __address)->__waiter_count_native, __address, __old_value, NoTimeout{});
492}
493
494template <std::size_t _Size>
495_LIBCPP_EXPORTED_FROM_ABI void
496__atomic_wait_native_with_timeout(void const* __address, void const* __old_value, uint64_t __timeout_ns) noexcept {
497 __contention_wait<_Size>(
498 &__get_contention_state(p: __address)->__waiter_count_native, __address, __old_value, __timeout_ns);
499}
500
501template <std::size_t _Size>
502_LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_one_native(void const* __location) noexcept {
503 __contention_notify<_Size>(&__get_contention_state(p: __location)->__waiter_count_native, __location, true);
504}
505
506template <std::size_t _Size>
507_LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_all_native(void const* __location) noexcept {
508 __contention_notify<_Size>(&__get_contention_state(p: __location)->__waiter_count_native, __location, false);
509}
510
511// ==================================================
512// Instantiation of the templates with supported size
513// ==================================================
514
515#if defined(_LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE)
516
517# define _INSTANTIATE(_SIZE) \
518 template _LIBCPP_EXPORTED_FROM_ABI void __atomic_wait_native<_SIZE>(void const*, void const*) noexcept; \
519 template _LIBCPP_EXPORTED_FROM_ABI void __atomic_wait_native_with_timeout<_SIZE>( \
520 void const*, void const*, uint64_t) noexcept; \
521 template _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_one_native<_SIZE>(void const*) noexcept; \
522 template _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_all_native<_SIZE>(void const*) noexcept;
523
524_LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_INSTANTIATE)
525
526# undef _INSTANTIATE
527
528#else // _LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE
529
530template _LIBCPP_EXPORTED_FROM_ABI void
531__atomic_wait_native<sizeof(__cxx_contention_t)>(void const* __address, void const* __old_value) noexcept;
532
533template _LIBCPP_EXPORTED_FROM_ABI void __atomic_wait_native_with_timeout<sizeof(__cxx_contention_t)>(
534 void const* __address, void const* __old_value, uint64_t) noexcept;
535
536template _LIBCPP_EXPORTED_FROM_ABI void
537__atomic_notify_one_native<sizeof(__cxx_contention_t)>(void const* __location) noexcept;
538
539template _LIBCPP_EXPORTED_FROM_ABI void
540__atomic_notify_all_native<sizeof(__cxx_contention_t)>(void const* __location) noexcept;
541
542#endif // _LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE
543
544// =============================================================
545// Old dylib exported symbols, for backwards compatibility
546// =============================================================
547_LIBCPP_DIAGNOSTIC_PUSH
548_LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wmissing-prototypes")
549
550_LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(void const volatile* __location) noexcept {
551 __atomic_notify_global_table(location: const_cast<void const*>(__location));
552}
553
554_LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(void const volatile* __location) noexcept {
555 __atomic_notify_global_table(location: const_cast<void const*>(__location));
556}
557
558_LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __libcpp_atomic_monitor(void const volatile* __location) noexcept {
559 auto const __entry = __get_contention_state(p: const_cast<void const*>(__location));
560 return __cxx_atomic_load(a: &__entry->__platform_state, order: memory_order_acquire);
561}
562
563_LIBCPP_EXPORTED_FROM_ABI void
564__libcpp_atomic_wait(void const volatile* __location, __cxx_contention_t __old_value) noexcept {
565 auto const __entry = __get_contention_state(p: const_cast<void const*>(__location));
566 __contention_wait<sizeof(__cxx_atomic_contention_t)>(
567 waiter_count: &__entry->__waiter_count_global, address_to_wait: &__entry->__platform_state, old_value: &__old_value, maybe_timeout_ns: NoTimeout{});
568}
569
570_LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(__cxx_atomic_contention_t const volatile* __location) noexcept {
571 auto __location_cast = const_cast<const void*>(static_cast<const volatile void*>(__location));
572 __contention_notify<sizeof(__cxx_atomic_contention_t)>(
573 waiter_count: &__get_contention_state(p: __location_cast)->__waiter_count_native, address_to_notify: __location_cast, notify_one: true);
574}
575
576_LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(__cxx_atomic_contention_t const volatile* __location) noexcept {
577 auto __location_cast = const_cast<const void*>(static_cast<const volatile void*>(__location));
578 __contention_notify<sizeof(__cxx_atomic_contention_t)>(
579 waiter_count: &__get_contention_state(p: __location_cast)->__waiter_count_native, address_to_notify: __location_cast, notify_one: false);
580}
581
582_LIBCPP_EXPORTED_FROM_ABI void
583__libcpp_atomic_wait(__cxx_atomic_contention_t const volatile* __location, __cxx_contention_t __old_value) noexcept {
584 auto __location_cast = const_cast<const void*>(static_cast<const volatile void*>(__location));
585 __contention_wait<sizeof(__cxx_atomic_contention_t)>(
586 waiter_count: &__get_contention_state(p: __location_cast)->__waiter_count_native, address_to_wait: __location_cast, old_value: &__old_value, maybe_timeout_ns: NoTimeout{});
587}
588
589// this function is even unused in the old ABI
590_LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t
591__libcpp_atomic_monitor(__cxx_atomic_contention_t const volatile* __location) noexcept {
592 return __cxx_atomic_load(a: __location, order: memory_order_acquire);
593}
594
595_LIBCPP_DIAGNOSTIC_POP
596
597_LIBCPP_END_EXPLICIT_ABI_ANNOTATIONS
598_LIBCPP_END_NAMESPACE_STD
599
600_LIBCPP_POP_MACROS
601