| 1 | //===----------------------------------------------------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include <__thread/timed_backoff_policy.h> |
| 10 | #include <atomic> |
| 11 | #include <climits> |
| 12 | #include <cstddef> |
| 13 | #include <cstdint> |
| 14 | #include <cstring> |
| 15 | #include <functional> |
| 16 | #include <new> |
| 17 | #include <thread> |
| 18 | #include <type_traits> |
| 19 | |
| 20 | #include "include/apple_availability.h" |
| 21 | |
| 22 | #ifdef __linux__ |
| 23 | |
| 24 | # include <linux/futex.h> |
| 25 | # include <sys/syscall.h> |
| 26 | # include <unistd.h> |
| 27 | |
| 28 | // libc++ uses SYS_futex as a universal syscall name. However, on 32 bit architectures |
| 29 | // with a 64 bit time_t, we need to specify SYS_futex_time64. |
| 30 | # if !defined(SYS_futex) && defined(SYS_futex_time64) |
| 31 | # define SYS_futex SYS_futex_time64 |
| 32 | # endif |
| 33 | # define _LIBCPP_FUTEX(...) syscall(SYS_futex, __VA_ARGS__) |
| 34 | |
| 35 | #elif defined(__FreeBSD__) |
| 36 | |
| 37 | # include <sys/types.h> |
| 38 | # include <sys/umtx.h> |
| 39 | |
| 40 | # define _LIBCPP_FUTEX(...) syscall(SYS_futex, __VA_ARGS__) |
| 41 | |
| 42 | #elif defined(__OpenBSD__) |
| 43 | |
| 44 | # include <sys/futex.h> |
| 45 | |
| 46 | // OpenBSD has no indirect syscalls |
| 47 | # define _LIBCPP_FUTEX(...) futex(__VA_ARGS__) |
| 48 | |
| 49 | #elif defined(_WIN32) |
| 50 | |
| 51 | # include <memory> |
| 52 | # include <windows.h> |
| 53 | |
| 54 | #else // <- Add other operating systems here |
| 55 | |
| 56 | // Baseline needs no new headers |
| 57 | |
| 58 | # define _LIBCPP_FUTEX(...) syscall(SYS_futex, __VA_ARGS__) |
| 59 | |
| 60 | #endif |
| 61 | |
| 62 | _LIBCPP_PUSH_MACROS |
| 63 | #include <__undef_macros> |
| 64 | |
| 65 | _LIBCPP_BEGIN_NAMESPACE_STD |
| 66 | |
| 67 | struct NoTimeout {}; |
| 68 | |
| 69 | #ifdef __linux__ |
| 70 | |
| 71 | template <std::size_t _Size, class MaybeTimeout> |
| 72 | static void __platform_wait_on_address(void const* __ptr, void const* __val, MaybeTimeout maybe_timeout_ns) { |
| 73 | static_assert(_Size == 4, "Can only wait on 4 bytes value" ); |
| 74 | alignas(__cxx_contention_t) char buffer[_Size]; |
| 75 | std::memcpy(dest: &buffer, src: const_cast<const void*>(__val), n: _Size); |
| 76 | static constexpr timespec __default_timeout = {.tv_sec: 2, .tv_nsec: 0}; |
| 77 | timespec __timeout; |
| 78 | if constexpr (is_same_v<MaybeTimeout, NoTimeout>) { |
| 79 | __timeout = __default_timeout; |
| 80 | } else { |
| 81 | __timeout.tv_sec = maybe_timeout_ns / 1'000'000'000; |
| 82 | __timeout.tv_nsec = maybe_timeout_ns % 1'000'000'000; |
| 83 | } |
| 84 | _LIBCPP_FUTEX(__ptr, FUTEX_WAIT_PRIVATE, *reinterpret_cast<__cxx_contention_t const*>(&buffer), &__timeout, 0, 0); |
| 85 | } |
| 86 | |
| 87 | template <std::size_t _Size> |
| 88 | static void __platform_wake_by_address(void const* __ptr, bool __notify_one) { |
| 89 | static_assert(_Size == 4, "Can only wake up on 4 bytes value" ); |
| 90 | _LIBCPP_FUTEX(__ptr, FUTEX_WAKE_PRIVATE, __notify_one ? 1 : INT_MAX, 0, 0, 0); |
| 91 | } |
| 92 | |
| 93 | #elif defined(__APPLE__) && defined(_LIBCPP_USE_ULOCK) |
| 94 | |
| 95 | extern "C" int __ulock_wait( |
| 96 | uint32_t operation, void* addr, uint64_t value, uint32_t timeout); /* timeout is specified in microseconds */ |
| 97 | extern "C" int __ulock_wake(uint32_t operation, void* addr, uint64_t wake_value); |
| 98 | |
| 99 | // https://github.com/apple/darwin-xnu/blob/2ff845c2e033bd0ff64b5b6aa6063a1f8f65aa32/bsd/sys/ulock.h#L82 |
| 100 | # define UL_COMPARE_AND_WAIT 1 |
| 101 | # define UL_COMPARE_AND_WAIT64 5 |
| 102 | # define ULF_WAKE_ALL 0x00000100 |
| 103 | |
| 104 | template <std::size_t _Size, class MaybeTimeout> |
| 105 | static void __platform_wait_on_address(void const* __ptr, void const* __val, MaybeTimeout maybe_timeout_ns) { |
| 106 | static_assert(_Size == 8 || _Size == 4, "Can only wait on 8 bytes or 4 bytes value" ); |
| 107 | auto __timeout_us = [&] { |
| 108 | if constexpr (is_same_v<MaybeTimeout, NoTimeout>) { |
| 109 | return uint32_t(0); |
| 110 | } else { |
| 111 | return std::max(static_cast<uint32_t>(maybe_timeout_ns / 1000), uint32_t(1)); |
| 112 | } |
| 113 | }(); |
| 114 | if constexpr (_Size == 4) { |
| 115 | alignas(uint32_t) char buffer[_Size]; |
| 116 | std::memcpy(&buffer, const_cast<const void*>(__val), _Size); |
| 117 | __ulock_wait( |
| 118 | UL_COMPARE_AND_WAIT, const_cast<void*>(__ptr), *reinterpret_cast<uint32_t const*>(&buffer), __timeout_us); |
| 119 | } else { |
| 120 | alignas(uint64_t) char buffer[_Size]; |
| 121 | std::memcpy(&buffer, const_cast<const void*>(__val), _Size); |
| 122 | __ulock_wait( |
| 123 | UL_COMPARE_AND_WAIT64, const_cast<void*>(__ptr), *reinterpret_cast<uint64_t const*>(&buffer), __timeout_us); |
| 124 | } |
| 125 | } |
| 126 | |
| 127 | template <std::size_t _Size> |
| 128 | static void __platform_wake_by_address(void const* __ptr, bool __notify_one) { |
| 129 | static_assert(_Size == 8 || _Size == 4, "Can only wake up on 8 bytes or 4 bytes value" ); |
| 130 | |
| 131 | if constexpr (_Size == 4) |
| 132 | __ulock_wake(UL_COMPARE_AND_WAIT | (__notify_one ? 0 : ULF_WAKE_ALL), const_cast<void*>(__ptr), 0); |
| 133 | else |
| 134 | __ulock_wake(UL_COMPARE_AND_WAIT64 | (__notify_one ? 0 : ULF_WAKE_ALL), const_cast<void*>(__ptr), 0); |
| 135 | } |
| 136 | |
| 137 | #elif defined(__FreeBSD__) && __SIZEOF_LONG__ == 8 |
| 138 | /* |
| 139 | * Since __cxx_contention_t is int64_t even on 32bit FreeBSD |
| 140 | * platforms, we have to use umtx ops that work on the long type, and |
| 141 | * limit its use to architectures where long and int64_t are synonyms. |
| 142 | */ |
| 143 | |
| 144 | template <std::size_t _Size, class MaybeTimeout> |
| 145 | static void __platform_wait_on_address(void const* __ptr, void const* __val, MaybeTimeout maybe_timeout_ns) { |
| 146 | static_assert(_Size == 8, "Can only wait on 8 bytes value" ); |
| 147 | // At the moment, FreeBSD is stuck on stable ABI, which only supports platform wait with __cxx_contention_t |
| 148 | // It is safe to reinterpret_cast the val as it is ever going to be passed a __cxx_contention_t under this ABI |
| 149 | // If in the future FreeBSD decides to experiment unstable ABI to support more types, this cast will no longer be |
| 150 | // safe. |
| 151 | __cxx_contention_t value = *reinterpret_cast<const __cxx_contention_t*>(__val); |
| 152 | if constexpr (is_same_v<MaybeTimeout, NoTimeout>) { |
| 153 | _umtx_op(const_cast<void*>(__ptr), UMTX_OP_WAIT, value, nullptr, nullptr); |
| 154 | } else { |
| 155 | timespec timeout{}; |
| 156 | timeout.tv_sec = maybe_timeout_ns / 1'000'000'000; |
| 157 | timeout.tv_nsec = maybe_timeout_ns % 1'000'000'000; |
| 158 | |
| 159 | _umtx_op(const_cast<void*>(__ptr), |
| 160 | UMTX_OP_WAIT, |
| 161 | value, |
| 162 | reinterpret_cast<void*>(static_cast<uintptr_t>(sizeof(timeout))), |
| 163 | &timeout); |
| 164 | } |
| 165 | } |
| 166 | |
| 167 | template <std::size_t _Size> |
| 168 | static void __platform_wake_by_address(void const* __ptr, bool __notify_one) { |
| 169 | static_assert(_Size == 8, "Can only wake up on 8 bytes value" ); |
| 170 | _umtx_op(const_cast<void*>(__ptr), UMTX_OP_WAKE, __notify_one ? 1 : INT_MAX, nullptr, nullptr); |
| 171 | } |
| 172 | |
| 173 | #elif defined(_WIN32) |
| 174 | |
| 175 | static void* win32_get_synch_api_function(const char* function_name) { |
| 176 | // Attempt to load the API set. Note that as per the Microsoft STL implementation, we assume this API is already |
| 177 | // loaded and accessible. While this isn't explicitly guaranteed by publicly available Win32 API documentation, it is |
| 178 | // true in practice, and may be guaranteed by internal documentation not released publicly. In any case the fact that |
| 179 | // the Microsoft STL made this assumption is reasonable basis to say that we can too. The alternative to this would be |
| 180 | // to use LoadLibrary, but then leak the module handle. We can't call FreeLibrary, as this would have to be triggered |
| 181 | // by a global static destructor, which would hang off DllMain, and calling FreeLibrary from DllMain is explicitly |
| 182 | // mentioned as not being allowed: |
| 183 | // https://learn.microsoft.com/en-us/windows/win32/dlls/dllmain |
| 184 | // Given the range of bad options here, we have chosen to mirror what Microsoft did, as it seems fair to assume that |
| 185 | // Microsoft will guarantee compatibility for us, as we are exposed to the same conditions as all existing Windows |
| 186 | // apps using the Microsoft STL VS2015/2017/2019/2022 runtimes, where Windows 7 support has not been excluded at |
| 187 | // compile time. |
| 188 | static auto module_handle = GetModuleHandleW(L"api-ms-win-core-synch-l1-2-0.dll" ); |
| 189 | if (module_handle == nullptr) { |
| 190 | return nullptr; |
| 191 | } |
| 192 | |
| 193 | // Attempt to locate the function in the API and return the result to the caller. Note that the NULL return from this |
| 194 | // method is documented as being interchangeable with nullptr. |
| 195 | // https://devblogs.microsoft.com/oldnewthing/20180307-00/?p=98175 |
| 196 | return reinterpret_cast<void*>(GetProcAddress(module_handle, function_name)); |
| 197 | } |
| 198 | |
| 199 | template <std::size_t _Size, class MaybeTimeout> |
| 200 | static void __platform_wait_on_address(void const* __ptr, void const* __val, MaybeTimeout maybe_timeout_ns) { |
| 201 | static_assert(_Size == 8, "Can only wait on 8 bytes value" ); |
| 202 | // WaitOnAddress was added in Windows 8 (build 9200) |
| 203 | static auto wait_on_address = |
| 204 | reinterpret_cast<BOOL(WINAPI*)(void*, PVOID, SIZE_T, DWORD)>(win32_get_synch_api_function("WaitOnAddress" )); |
| 205 | if (wait_on_address != nullptr) { |
| 206 | auto timeout_ms = [&]() -> DWORD { |
| 207 | if constexpr (is_same_v<MaybeTimeout, NoTimeout>) { |
| 208 | return INFINITE; |
| 209 | } else { |
| 210 | uint64_t ms = maybe_timeout_ns / 1'000'000; |
| 211 | if (ms == 0 && maybe_timeout_ns > 100'000) |
| 212 | // Round up to 1ms if requested between 100us - 1ms |
| 213 | return 1; |
| 214 | |
| 215 | return static_cast<DWORD>(std::min(static_cast<uint64_t>(INFINITE), ms)); |
| 216 | } |
| 217 | }(); |
| 218 | wait_on_address(const_cast<void*>(__ptr), const_cast<void*>(__val), _Size, timeout_ms); |
| 219 | } else { |
| 220 | std::chrono::nanoseconds timeout = std::chrono::nanoseconds(0); |
| 221 | if constexpr (!is_same_v<MaybeTimeout, NoTimeout>) { |
| 222 | timeout = std::chrono::nanoseconds(maybe_timeout_ns); |
| 223 | } |
| 224 | __libcpp_thread_poll_with_backoff( |
| 225 | [=]() -> bool { return std::memcmp(const_cast<const void*>(__ptr), __val, _Size) != 0; }, |
| 226 | __libcpp_timed_backoff_policy(), |
| 227 | timeout); |
| 228 | } |
| 229 | } |
| 230 | |
| 231 | template <std::size_t _Size> |
| 232 | static void __platform_wake_by_address(void const* __ptr, bool __notify_one) { |
| 233 | static_assert(_Size == 8, "Can only wake up on 8 bytes value" ); |
| 234 | if (__notify_one) { |
| 235 | // WakeByAddressSingle was added in Windows 8 (build 9200) |
| 236 | static auto wake_by_address_single = |
| 237 | reinterpret_cast<void(WINAPI*)(PVOID)>(win32_get_synch_api_function("WakeByAddressSingle" )); |
| 238 | if (wake_by_address_single != nullptr) { |
| 239 | wake_by_address_single(const_cast<void*>(__ptr)); |
| 240 | } else { |
| 241 | // The fallback implementation of waking does nothing, as the fallback wait implementation just does polling, so |
| 242 | // there's nothing to do here. |
| 243 | } |
| 244 | } else { |
| 245 | // WakeByAddressAll was added in Windows 8 (build 9200) |
| 246 | static auto wake_by_address_all = |
| 247 | reinterpret_cast<void(WINAPI*)(PVOID)>(win32_get_synch_api_function("WakeByAddressAll" )); |
| 248 | if (wake_by_address_all != nullptr) { |
| 249 | wake_by_address_all(const_cast<void*>(__ptr)); |
| 250 | } else { |
| 251 | // The fallback implementation of waking does nothing, as the fallback wait implementation just does polling, so |
| 252 | // there's nothing to do here. |
| 253 | } |
| 254 | } |
| 255 | } |
| 256 | |
| 257 | #else // <- Add other operating systems here |
| 258 | |
| 259 | // Baseline is just a timed backoff |
| 260 | |
| 261 | template <std::size_t _Size, class MaybeTimeout> |
| 262 | static void __platform_wait_on_address(void const* __ptr, void const* __val, MaybeTimeout maybe_timeout_ns) { |
| 263 | std::chrono::nanoseconds timeout = std::chrono::nanoseconds(0); |
| 264 | if constexpr (!is_same_v<MaybeTimeout, NoTimeout>) { |
| 265 | timeout = std::chrono::nanoseconds(maybe_timeout_ns); |
| 266 | } |
| 267 | __libcpp_thread_poll_with_backoff( |
| 268 | [=]() -> bool { return std::memcmp(const_cast<const void*>(__ptr), __val, _Size) != 0; }, |
| 269 | __libcpp_timed_backoff_policy(), |
| 270 | timeout); |
| 271 | } |
| 272 | |
| 273 | template <std::size_t _Size> |
| 274 | static void __platform_wake_by_address(void const*, bool) {} |
| 275 | |
| 276 | #endif // __linux__ |
| 277 | |
| 278 | // ============================= |
| 279 | // Local hidden helper functions |
| 280 | // ============================= |
| 281 | |
| 282 | /* Given an atomic to track contention and an atomic to actually wait on, which may be |
| 283 | the same atomic, we try to detect contention to avoid spuriously calling the platform. */ |
| 284 | |
| 285 | template <std::size_t _Size> |
| 286 | static void |
| 287 | __contention_notify(__cxx_atomic_contention_t* __waiter_count, void const* __address_to_notify, bool __notify_one) { |
| 288 | if (0 != __cxx_atomic_load(a: __waiter_count, order: memory_order_seq_cst)) |
| 289 | // We only call 'wake' if we consumed a contention bit here. |
| 290 | __platform_wake_by_address<_Size>(__address_to_notify, __notify_one); |
| 291 | } |
| 292 | |
| 293 | template <std::size_t _Size, class MaybeTimeout> |
| 294 | static void __contention_wait(__cxx_atomic_contention_t* __waiter_count, |
| 295 | void const* __address_to_wait, |
| 296 | void const* __old_value, |
| 297 | MaybeTimeout maybe_timeout_ns) { |
| 298 | __cxx_atomic_fetch_add(a: __waiter_count, delta: __cxx_contention_t(1), order: memory_order_relaxed); |
| 299 | // https://llvm.org/PR109290 |
| 300 | // There are no platform guarantees of a memory barrier in the platform wait implementation |
| 301 | __cxx_atomic_thread_fence(order: memory_order_seq_cst); |
| 302 | // We sleep as long as the monitored value hasn't changed. |
| 303 | __platform_wait_on_address<_Size>(__address_to_wait, __old_value, maybe_timeout_ns); |
| 304 | __cxx_atomic_fetch_sub(a: __waiter_count, delta: __cxx_contention_t(1), order: memory_order_release); |
| 305 | } |
| 306 | |
| 307 | static constexpr size_t __contention_table_size = (1 << 8); /* < there's no magic in this number */ |
| 308 | |
| 309 | static constexpr hash<void const*> __contention_hasher; |
| 310 | |
| 311 | // Waiter count table for all atomics with the correct size that use itself as the wait/notify address. |
| 312 | |
| 313 | struct alignas( |
| 314 | std::hardware_constructive_interference_size) /* aim to avoid false sharing */ __contention_state_native { |
| 315 | __cxx_atomic_contention_t __waiter_count; |
| 316 | constexpr __contention_state_native() : __waiter_count(0) {} |
| 317 | }; |
| 318 | |
| 319 | static __contention_state_native __contention_table_native[__contention_table_size]; |
| 320 | |
| 321 | static __cxx_atomic_contention_t* __get_native_waiter_count(void const* p) { |
| 322 | return &__contention_table_native[__contention_hasher(p) & (__contention_table_size - 1)].__waiter_count; |
| 323 | } |
| 324 | |
| 325 | // Global contention table for all atomics with the wrong size that use the global table's atomic as wait/notify |
| 326 | // address. |
| 327 | |
| 328 | struct alignas( |
| 329 | std::hardware_constructive_interference_size) /* aim to avoid false sharing */ __contention_state_global { |
| 330 | __cxx_atomic_contention_t __waiter_count; |
| 331 | __cxx_atomic_contention_t __platform_state; |
| 332 | constexpr __contention_state_global() : __waiter_count(0), __platform_state(0) {} |
| 333 | }; |
| 334 | |
| 335 | static __contention_state_global __contention_table_global[__contention_table_size]; |
| 336 | |
| 337 | static __contention_state_global* __get_global_contention_state(void const* p) { |
| 338 | return &__contention_table_global[__contention_hasher(p) & (__contention_table_size - 1)]; |
| 339 | } |
| 340 | |
| 341 | /* When the incoming atomic is the wrong size for the platform wait size, need to |
| 342 | launder the value sequence through an atomic from our table. */ |
| 343 | |
| 344 | static void __atomic_notify_global_table(void const* __location) { |
| 345 | auto const __entry = __get_global_contention_state(p: __location); |
| 346 | // The value sequence laundering happens on the next line below. |
| 347 | __cxx_atomic_fetch_add(a: &__entry->__platform_state, delta: __cxx_contention_t(1), order: memory_order_seq_cst); |
| 348 | __contention_notify<sizeof(__cxx_atomic_contention_t)>( |
| 349 | waiter_count: &__entry->__waiter_count, address_to_notify: &__entry->__platform_state, notify_one: false /* when laundering, we can't handle notify_one */); |
| 350 | } |
| 351 | |
| 352 | // ============================= |
| 353 | // New dylib exported symbols |
| 354 | // ============================= |
| 355 | |
| 356 | // global |
| 357 | _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __atomic_monitor_global(void const* __location) noexcept { |
| 358 | auto const __entry = __get_global_contention_state(p: __location); |
| 359 | return __cxx_atomic_load(a: &__entry->__platform_state, order: memory_order_acquire); |
| 360 | } |
| 361 | |
| 362 | _LIBCPP_EXPORTED_FROM_ABI void |
| 363 | __atomic_wait_global_table(void const* __location, __cxx_contention_t __old_value) noexcept { |
| 364 | auto const __entry = __get_global_contention_state(p: __location); |
| 365 | __contention_wait<sizeof(__cxx_atomic_contention_t)>( |
| 366 | waiter_count: &__entry->__waiter_count, address_to_wait: &__entry->__platform_state, old_value: &__old_value, maybe_timeout_ns: NoTimeout{}); |
| 367 | } |
| 368 | |
| 369 | _LIBCPP_EXPORTED_FROM_ABI void __atomic_wait_global_table_with_timeout( |
| 370 | void const* __location, __cxx_contention_t __old_value, uint64_t __timeout_ns) _NOEXCEPT { |
| 371 | auto const __entry = __get_global_contention_state(p: __location); |
| 372 | __contention_wait<sizeof(__cxx_atomic_contention_t)>( |
| 373 | waiter_count: &__entry->__waiter_count, address_to_wait: &__entry->__platform_state, old_value: &__old_value, maybe_timeout_ns: __timeout_ns); |
| 374 | } |
| 375 | |
| 376 | _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_one_global_table(void const* __location) noexcept { |
| 377 | __atomic_notify_global_table(__location); |
| 378 | } |
| 379 | |
| 380 | _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_all_global_table(void const* __location) noexcept { |
| 381 | __atomic_notify_global_table(__location); |
| 382 | } |
| 383 | |
| 384 | // native |
| 385 | |
| 386 | template <std::size_t _Size> |
| 387 | _LIBCPP_EXPORTED_FROM_ABI void __atomic_wait_native(void const* __address, void const* __old_value) noexcept { |
| 388 | __contention_wait<_Size>(__get_native_waiter_count(p: __address), __address, __old_value, NoTimeout{}); |
| 389 | } |
| 390 | |
| 391 | template <std::size_t _Size> |
| 392 | _LIBCPP_EXPORTED_FROM_ABI void |
| 393 | __atomic_wait_native_with_timeout(void const* __address, void const* __old_value, uint64_t __timeout_ns) noexcept { |
| 394 | __contention_wait<_Size>(__get_native_waiter_count(p: __address), __address, __old_value, __timeout_ns); |
| 395 | } |
| 396 | |
| 397 | template <std::size_t _Size> |
| 398 | _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_one_native(void const* __location) noexcept { |
| 399 | __contention_notify<_Size>(__get_native_waiter_count(p: __location), __location, true); |
| 400 | } |
| 401 | |
| 402 | template <std::size_t _Size> |
| 403 | _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_all_native(void const* __location) noexcept { |
| 404 | __contention_notify<_Size>(__get_native_waiter_count(p: __location), __location, false); |
| 405 | } |
| 406 | |
| 407 | // ================================================== |
| 408 | // Instantiation of the templates with supported size |
| 409 | // ================================================== |
| 410 | |
| 411 | #if defined(_LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE) |
| 412 | |
| 413 | # define _INSTANTIATE(_SIZE) \ |
| 414 | template _LIBCPP_EXPORTED_FROM_ABI void __atomic_wait_native<_SIZE>(void const*, void const*) noexcept; \ |
| 415 | template _LIBCPP_EXPORTED_FROM_ABI void __atomic_wait_native_with_timeout<_SIZE>( \ |
| 416 | void const*, void const*, uint64_t) noexcept; \ |
| 417 | template _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_one_native<_SIZE>(void const*) noexcept; \ |
| 418 | template _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_all_native<_SIZE>(void const*) noexcept; |
| 419 | |
| 420 | _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_INSTANTIATE) |
| 421 | |
| 422 | # undef _INSTANTIATE |
| 423 | |
| 424 | #else // _LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE |
| 425 | |
| 426 | template _LIBCPP_EXPORTED_FROM_ABI void |
| 427 | __atomic_wait_native<sizeof(__cxx_contention_t)>(void const* __address, void const* __old_value) noexcept; |
| 428 | |
| 429 | template _LIBCPP_EXPORTED_FROM_ABI void __atomic_wait_native_with_timeout<sizeof(__cxx_contention_t)>( |
| 430 | void const* __address, void const* __old_value, uint64_t) noexcept; |
| 431 | |
| 432 | template _LIBCPP_EXPORTED_FROM_ABI void |
| 433 | __atomic_notify_one_native<sizeof(__cxx_contention_t)>(void const* __location) noexcept; |
| 434 | |
| 435 | template _LIBCPP_EXPORTED_FROM_ABI void |
| 436 | __atomic_notify_all_native<sizeof(__cxx_contention_t)>(void const* __location) noexcept; |
| 437 | |
| 438 | #endif // _LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE |
| 439 | |
| 440 | // ============================================================= |
| 441 | // Old dylib exported symbols, for backwards compatibility |
| 442 | // ============================================================= |
| 443 | _LIBCPP_DIAGNOSTIC_PUSH |
| 444 | _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wmissing-prototypes" ) |
| 445 | |
| 446 | _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(void const volatile* __location) noexcept { |
| 447 | __atomic_notify_global_table(location: const_cast<void const*>(__location)); |
| 448 | } |
| 449 | |
| 450 | _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(void const volatile* __location) noexcept { |
| 451 | __atomic_notify_global_table(location: const_cast<void const*>(__location)); |
| 452 | } |
| 453 | |
| 454 | _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __libcpp_atomic_monitor(void const volatile* __location) noexcept { |
| 455 | auto const __entry = __get_global_contention_state(p: const_cast<void const*>(__location)); |
| 456 | return __cxx_atomic_load(a: &__entry->__platform_state, order: memory_order_acquire); |
| 457 | } |
| 458 | |
| 459 | _LIBCPP_EXPORTED_FROM_ABI void |
| 460 | __libcpp_atomic_wait(void const volatile* __location, __cxx_contention_t __old_value) noexcept { |
| 461 | auto const __entry = __get_global_contention_state(p: const_cast<void const*>(__location)); |
| 462 | __contention_wait<sizeof(__cxx_atomic_contention_t)>( |
| 463 | waiter_count: &__entry->__waiter_count, address_to_wait: &__entry->__platform_state, old_value: &__old_value, maybe_timeout_ns: NoTimeout{}); |
| 464 | } |
| 465 | |
| 466 | _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(__cxx_atomic_contention_t const volatile* __location) noexcept { |
| 467 | auto __location_cast = const_cast<const void*>(static_cast<const volatile void*>(__location)); |
| 468 | __contention_notify<sizeof(__cxx_atomic_contention_t)>( |
| 469 | waiter_count: __get_native_waiter_count(p: __location_cast), address_to_notify: __location_cast, notify_one: true); |
| 470 | } |
| 471 | |
| 472 | _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(__cxx_atomic_contention_t const volatile* __location) noexcept { |
| 473 | auto __location_cast = const_cast<const void*>(static_cast<const volatile void*>(__location)); |
| 474 | __contention_notify<sizeof(__cxx_atomic_contention_t)>( |
| 475 | waiter_count: __get_native_waiter_count(p: __location_cast), address_to_notify: __location_cast, notify_one: false); |
| 476 | } |
| 477 | |
| 478 | _LIBCPP_EXPORTED_FROM_ABI void |
| 479 | __libcpp_atomic_wait(__cxx_atomic_contention_t const volatile* __location, __cxx_contention_t __old_value) noexcept { |
| 480 | auto __location_cast = const_cast<const void*>(static_cast<const volatile void*>(__location)); |
| 481 | __contention_wait<sizeof(__cxx_atomic_contention_t)>( |
| 482 | waiter_count: __get_native_waiter_count(p: __location_cast), address_to_wait: __location_cast, old_value: &__old_value, maybe_timeout_ns: NoTimeout{}); |
| 483 | } |
| 484 | |
| 485 | // this function is even unused in the old ABI |
| 486 | _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t |
| 487 | __libcpp_atomic_monitor(__cxx_atomic_contention_t const volatile* __location) noexcept { |
| 488 | return __cxx_atomic_load(a: __location, order: memory_order_acquire); |
| 489 | } |
| 490 | |
| 491 | _LIBCPP_DIAGNOSTIC_POP |
| 492 | |
| 493 | _LIBCPP_END_NAMESPACE_STD |
| 494 | |
| 495 | _LIBCPP_POP_MACROS |
| 496 | |