1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include <__config>
10#ifdef _LIBCPP_DEPRECATED_ABI_LEGACY_LIBRARY_DEFINITIONS_FOR_INLINE_FUNCTIONS
11# define _LIBCPP_SHARED_PTR_DEFINE_LEGACY_INLINE_FUNCTIONS
12#endif
13
14#include <__functional/hash.h>
15#include <memory>
16#include <typeinfo>
17
18#if _LIBCPP_HAS_THREADS
19# include <mutex>
20# include <thread>
21# if defined(__ELF__) && defined(_LIBCPP_LINK_PTHREAD_LIB)
22# pragma comment(lib, "pthread")
23# endif
24#endif
25
26#include "include/atomic_support.h"
27
28_LIBCPP_BEGIN_NAMESPACE_STD
29
30bad_weak_ptr::~bad_weak_ptr() noexcept {}
31
32const char* bad_weak_ptr::what() const noexcept { return "bad_weak_ptr"; }
33
34__shared_count::~__shared_count() {}
35
36__shared_weak_count::~__shared_weak_count() {}
37
38#if defined(_LIBCPP_SHARED_PTR_DEFINE_LEGACY_INLINE_FUNCTIONS)
39void __shared_count::__add_shared() noexcept { __libcpp_atomic_refcount_increment(t&: __shared_owners_); }
40
41bool __shared_count::__release_shared() noexcept {
42 if (__libcpp_atomic_refcount_decrement(t&: __shared_owners_) == -1) {
43 __on_zero_shared();
44 return true;
45 }
46 return false;
47}
48
49void __shared_weak_count::__add_shared() noexcept { __shared_count::__add_shared(); }
50
51void __shared_weak_count::__add_weak() noexcept { __libcpp_atomic_refcount_increment(t&: __shared_weak_owners_); }
52
53void __shared_weak_count::__release_shared() noexcept {
54 if (__shared_count::__release_shared())
55 __release_weak();
56}
57#endif // _LIBCPP_SHARED_PTR_DEFINE_LEGACY_INLINE_FUNCTIONS
58
59void __shared_weak_count::__release_weak() noexcept {
60 // NOTE: The acquire load here is an optimization of the very
61 // common case where a shared pointer is being destructed while
62 // having no other contended references.
63 //
64 // BENEFIT: We avoid expensive atomic stores like XADD and STREX
65 // in a common case. Those instructions are slow and do nasty
66 // things to caches.
67 //
68 // IS THIS SAFE? Yes. During weak destruction, if we see that we
69 // are the last reference, we know that no-one else is accessing
70 // us. If someone were accessing us, then they would be doing so
71 // while the last shared / weak_ptr was being destructed, and
72 // that's undefined anyway.
73 //
74 // If we see anything other than a 0, then we have possible
75 // contention, and need to use an atomicrmw primitive.
76 // The same arguments don't apply for increment, where it is legal
77 // (though inadvisable) to share shared_ptr references between
78 // threads, and have them all get copied at once. The argument
79 // also doesn't apply for __release_shared, because an outstanding
80 // weak_ptr::lock() could read / modify the shared count.
81 if (__libcpp_atomic_load(val: &__shared_weak_owners_, order: _AO_Acquire) == 0) {
82 // no need to do this store, because we are about
83 // to destroy everything.
84 //__libcpp_atomic_store(&__shared_weak_owners_, -1, _AO_Release);
85 __on_zero_shared_weak();
86 } else if (__libcpp_atomic_refcount_decrement(t&: __shared_weak_owners_) == -1)
87 __on_zero_shared_weak();
88}
89
90__shared_weak_count* __shared_weak_count::lock() noexcept {
91 long object_owners = __libcpp_atomic_load(val: &__shared_owners_);
92 while (object_owners != -1) {
93 if (__libcpp_atomic_compare_exchange(val: &__shared_owners_, expected: &object_owners, after: object_owners + 1))
94 return this;
95 }
96 return nullptr;
97}
98
99const void* __shared_weak_count::__get_deleter(const type_info&) const noexcept { return nullptr; }
100
101#if _LIBCPP_HAS_THREADS
102
103static constexpr std::size_t __sp_mut_count = 32;
104static constinit __libcpp_mutex_t mut_back[__sp_mut_count] = {
105 _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER,
106 _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER,
107 _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER,
108 _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER,
109 _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER,
110 _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER,
111 _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER,
112 _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER, _LIBCPP_MUTEX_INITIALIZER};
113
114constexpr __sp_mut::__sp_mut(void* p) noexcept : __lx_(p) {}
115
116void __sp_mut::lock() noexcept {
117 auto m = static_cast<__libcpp_mutex_t*>(__lx_);
118 __libcpp_mutex_lock(m: m);
119}
120
121void __sp_mut::unlock() noexcept { __libcpp_mutex_unlock(m: static_cast<__libcpp_mutex_t*>(__lx_)); }
122
123__sp_mut& __get_sp_mut(const void* p) {
124 static constinit __sp_mut muts[__sp_mut_count] = {
125 &mut_back[0], &mut_back[1], &mut_back[2], &mut_back[3], &mut_back[4], &mut_back[5], &mut_back[6],
126 &mut_back[7], &mut_back[8], &mut_back[9], &mut_back[10], &mut_back[11], &mut_back[12], &mut_back[13],
127 &mut_back[14], &mut_back[15], &mut_back[16], &mut_back[17], &mut_back[18], &mut_back[19], &mut_back[20],
128 &mut_back[21], &mut_back[22], &mut_back[23], &mut_back[24], &mut_back[25], &mut_back[26], &mut_back[27],
129 &mut_back[28], &mut_back[29], &mut_back[30], &mut_back[31]};
130 return muts[hash<const void*>()(p) & (__sp_mut_count - 1)];
131}
132
133#endif // _LIBCPP_HAS_THREADS
134
135void* align(size_t alignment, size_t size, void*& ptr, size_t& space) {
136 void* r = nullptr;
137 if (size <= space) {
138 char* p1 = static_cast<char*>(ptr);
139 char* p2 = reinterpret_cast<char*>(reinterpret_cast<uintptr_t>(p1 + (alignment - 1)) & -alignment);
140 size_t d = static_cast<size_t>(p2 - p1);
141 if (d <= space - size) {
142 r = p2;
143 ptr = r;
144 space -= d;
145 }
146 }
147 return r;
148}
149
150_LIBCPP_END_NAMESPACE_STD
151