1//===-- tsd_shared.h --------------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#ifndef SCUDO_TSD_SHARED_H_
10#define SCUDO_TSD_SHARED_H_
11
12#include "tsd.h"
13
14#include "string_utils.h"
15
16#if SCUDO_HAS_PLATFORM_TLS_SLOT
17// This is a platform-provided header that needs to be on the include path when
18// Scudo is compiled. It must declare a function with the prototype:
19// uintptr_t *getPlatformAllocatorTlsSlot()
20// that returns the address of a thread-local word of storage reserved for
21// Scudo, that must be zero-initialized in newly created threads.
22#include "scudo_platform_tls_slot.h"
23#endif
24
25namespace scudo {
26
27template <class Allocator, u32 TSDsArraySize, u32 DefaultTSDCount>
28struct TSDRegistrySharedT {
29 using ThisT = TSDRegistrySharedT<Allocator, TSDsArraySize, DefaultTSDCount>;
30
31 struct ScopedTSD {
32 ALWAYS_INLINE ScopedTSD(ThisT &TSDRegistry) {
33 CurrentTSD = TSDRegistry.getTSDAndLock();
34 DCHECK_NE(CurrentTSD, nullptr);
35 }
36
37 ~ScopedTSD() { CurrentTSD->unlock(); }
38
39 TSD<Allocator> &operator*() { return *CurrentTSD; }
40
41 TSD<Allocator> *operator->() {
42 CurrentTSD->assertLocked(/*BypassCheck=*/false);
43 return CurrentTSD;
44 }
45
46 private:
47 TSD<Allocator> *CurrentTSD;
48 };
49
50 void init(Allocator *Instance) EXCLUDES(Mutex) {
51 ScopedLock L(Mutex);
52 // If more than one thread is initializing at the exact same moment, the
53 // threads that lose don't need to do anything.
54 if (UNLIKELY(atomic_load_relaxed(&Initialized) != 0))
55 return;
56
57 Instance->init();
58 for (u32 I = 0; I < TSDsArraySize; I++)
59 TSDs[I].init(Instance);
60 const u32 NumberOfCPUs = getNumberOfCPUs();
61 setNumberOfTSDs((NumberOfCPUs == 0) ? DefaultTSDCount
62 : Min(A: NumberOfCPUs, B: DefaultTSDCount));
63 atomic_store_relaxed(A: &Initialized, V: 1);
64 }
65
66 void initOnceMaybe(Allocator *Instance) {
67 if (LIKELY(atomic_load_relaxed(&Initialized) != 0))
68 return;
69 init(Instance); // Sets Initialized.
70 }
71
72 void unmapTestOnly(Allocator *Instance) {
73 ScopedLock L(Mutex);
74 for (u32 I = 0; I < TSDsArraySize; I++) {
75 TSDs[I].commitBack(Instance);
76 TSDs[I] = {};
77 }
78 setCurrentTSD(nullptr);
79 atomic_store_relaxed(A: &Initialized, V: 0);
80 }
81
82 void drainCaches(Allocator *Instance) {
83 u32 TotalTSDs = atomic_load_relaxed(A: &NumberOfTSDs);
84 for (uptr I = 0; I < TotalTSDs; ++I) {
85 TSDs[I].lock();
86 Instance->drainCache(&TSDs[I]);
87 TSDs[I].unlock();
88 }
89 }
90
91 ALWAYS_INLINE void initThreadMaybe(Allocator *Instance,
92 UNUSED bool MinimalInit) {
93 if (LIKELY(getCurrentTSD()))
94 return;
95 initThread(Instance);
96 }
97
98 // Only the first NumberOfTSDs TSDs are ever handed out, so skip the rest to
99 // avoid dirtying their pages around fork(). setNumberOfTSDs() only grows the
100 // count under Mutex, which is held until enable(), so both loops match.
101 void disable() NO_THREAD_SAFETY_ANALYSIS {
102 Mutex.lock();
103 const u32 TotalTSDs = atomic_load_relaxed(A: &NumberOfTSDs);
104 for (u32 I = 0; I < TotalTSDs; I++)
105 TSDs[I].lock();
106 }
107
108 void enable() NO_THREAD_SAFETY_ANALYSIS {
109 const u32 TotalTSDs = atomic_load_relaxed(A: &NumberOfTSDs);
110 for (s32 I = static_cast<s32>(TotalTSDs - 1); I >= 0; I--)
111 TSDs[I].unlock();
112 Mutex.unlock();
113 }
114
115 bool setOption(Option O, sptr Value) {
116 if (O == Option::MaxTSDsCount) {
117 ScopedLock L(Mutex);
118 return setNumberOfTSDs(static_cast<u32>(Value));
119 }
120 if (O == Option::ThreadDisableMemInit)
121 setDisableMemInit(Value);
122 // Not supported by the TSD Registry, but not an error either.
123 return true;
124 }
125
126 bool getDisableMemInit() const { return *getTlsPtr() & 1; }
127
128 void getStats(ScopedString *Str) {
129 u32 TotalTSDs = atomic_load_relaxed(A: &NumberOfTSDs);
130 Str->append(Format: "Stats: SharedTSDs: %u available; total %u\n", TotalTSDs,
131 TSDsArraySize);
132 for (uptr I = 0; I < TotalTSDs; ++I) {
133 TSDs[I].lock();
134 // Theoretically, we want to mark TSD::lock()/TSD::unlock() with proper
135 // thread annotations. However, given the TSD is only locked on shared
136 // path, do the assertion in a separate path to avoid confusing the
137 // analyzer.
138 TSDs[I].assertLocked(/*BypassCheck=*/true);
139 Str->append(Format: " Shared TSD[%zu]:\n", I);
140 TSDs[I].getSizeClassAllocator().getStats(Str);
141 TSDs[I].unlock();
142 }
143 }
144
145private:
146 ALWAYS_INLINE TSD<Allocator> *getTSDAndLock() NO_THREAD_SAFETY_ANALYSIS {
147 TSD<Allocator> *TSD = getCurrentTSD();
148 DCHECK(TSD);
149 if (TSDsArraySize == 1U) {
150 // Only 1 TSD, lock and return.
151 TSD->lock();
152 return TSD;
153 }
154
155 if (TSD->tryLock())
156 return TSD;
157 return getTSDSlow(CurrentTSD: TSD);
158 }
159
160 ALWAYS_INLINE uptr *getTlsPtr() const {
161#if SCUDO_HAS_PLATFORM_TLS_SLOT
162 return reinterpret_cast<uptr *>(getPlatformAllocatorTlsSlot());
163#else
164 static thread_local uptr ThreadTSD;
165 return &ThreadTSD;
166#endif
167 }
168
169 static_assert(alignof(TSD<Allocator>) >= 2, "");
170
171 ALWAYS_INLINE void setCurrentTSD(TSD<Allocator> *CurrentTSD) {
172 *getTlsPtr() &= 1;
173 *getTlsPtr() |= reinterpret_cast<uptr>(CurrentTSD);
174 }
175
176 ALWAYS_INLINE TSD<Allocator> *getCurrentTSD() {
177 return reinterpret_cast<TSD<Allocator> *>(*getTlsPtr() & ~1ULL);
178 }
179
180 // Requires a lock to avoid multiple threads trying to set the TSDs at once.
181 bool setNumberOfTSDs(u32 N) REQUIRES(Mutex) {
182 DCHECK_GT(N, 0);
183 const u32 TotalTSDs = atomic_load_relaxed(A: &NumberOfTSDs);
184 // In order to avoid needing locks for these values, the number of TSDs
185 // can never be decreased.
186 if (N < TotalTSDs)
187 return false;
188
189 if (N > TSDsArraySize)
190 N = TSDsArraySize;
191
192 // In order to avoid locks around the CoPrimes, compute all of the
193 // new values and then set them in place after.
194 u32 NewCoPrimes[TSDsArraySize] = {};
195 u32 NewNumberOfCoPrimes = 0;
196
197 // Compute all the coprimes of the TSDs. This will be used to walk the
198 // array of TSDs in a random order. For details, see:
199 // https://lemire.me/blog/2017/09/18/visiting-all-values-in-an-array-exactly-once-in-random-order/
200 for (u32 I = 0; I < N; I++) {
201 u32 A = I + 1;
202 u32 B = N;
203 // Find the GCD between I + 1 and N. If 1, they are coprimes.
204 while (B != 0) {
205 const u32 T = A;
206 A = B;
207 B = T % B;
208 }
209 if (A == 1)
210 NewCoPrimes[NewNumberOfCoPrimes++] = I + 1;
211 }
212
213 // Set these values in this particular order so that if getTSDSlow
214 // is called while they are being modified, nothing crashes, or encounters
215 // a zero CoPrime value.
216 for (u32 I = 0; I < NewNumberOfCoPrimes; I++) {
217 atomic_store(&CoPrimes[I], NewCoPrimes[I], memory_order_release);
218 }
219 atomic_store(A: &NumberOfCoPrimes, V: NewNumberOfCoPrimes, MO: memory_order_release);
220 atomic_store(A: &NumberOfTSDs, V: N, MO: memory_order_release);
221 return true;
222 }
223
224 void setDisableMemInit(bool B) {
225 *getTlsPtr() &= ~1ULL;
226 *getTlsPtr() |= B;
227 }
228
229 NOINLINE void initThread(Allocator *Instance) {
230 initOnceMaybe(Instance);
231 // Initial context assignment is done in a plain round-robin fashion.
232 const u32 Index = atomic_fetch_add(A: &CurrentIndex, V: 1U, MO: memory_order_relaxed);
233 setCurrentTSD(&TSDs[Index % atomic_load_relaxed(A: &NumberOfTSDs)]);
234 Instance->callPostInitCallback();
235 }
236
237 // TSDs is an array of locks which is not supported for marking thread-safety
238 // capability.
239 NOINLINE TSD<Allocator> *getTSDSlow(TSD<Allocator> *CurrentTSD) {
240 const u32 TotalTSDs = atomic_load_relaxed(A: &NumberOfTSDs);
241 if (UNLIKELY(TotalTSDs == 1U)) {
242 CurrentTSD->lock();
243 return CurrentTSD;
244 }
245
246 // Use the Precedence of the current TSD as our random seed. Since we are
247 // in the slow path, it means that tryLock failed, and as a result it's
248 // very likely that said Precedence is non-zero.
249 const u32 R = static_cast<u32>(CurrentTSD->getPrecedence());
250 const u32 TotalCoPrimes = atomic_load_relaxed(A: &NumberOfCoPrimes);
251 DCHECK_NE(TotalCoPrimes, 0U);
252 const u32 Inc = atomic_load_relaxed(&CoPrimes[R % TotalCoPrimes]);
253
254 u32 Index = R % TotalTSDs;
255 uptr LowestPrecedence = UINTPTR_MAX;
256 TSD<Allocator> *CandidateTSD = nullptr;
257 // Go randomly through at most 4 contexts and find a candidate.
258 for (u32 I = 0; I < Min(A: 4U, B: TotalTSDs); I++) {
259 if (TSDs[Index].tryLock()) {
260 setCurrentTSD(&TSDs[Index]);
261 return &TSDs[Index];
262 }
263 const uptr Precedence = TSDs[Index].getPrecedence();
264 // A 0 precedence here means another thread just locked this TSD.
265 if (Precedence && Precedence < LowestPrecedence) {
266 CandidateTSD = &TSDs[Index];
267 LowestPrecedence = Precedence;
268 }
269 Index += Inc;
270 if (Index >= TotalTSDs)
271 Index -= TotalTSDs;
272 }
273 if (CandidateTSD) {
274 CandidateTSD->lock();
275 setCurrentTSD(CandidateTSD);
276 return CandidateTSD;
277 }
278 // Last resort, stick with the current one.
279 CurrentTSD->lock();
280 return CurrentTSD;
281 }
282
283 atomic_u32 CurrentIndex = {};
284 atomic_u32 NumberOfTSDs = {};
285 atomic_u32 NumberOfCoPrimes = {};
286 atomic_u32 CoPrimes[TSDsArraySize] = {};
287 atomic_u8 Initialized = {};
288 // Used for global initialization and TSDs access.
289 // Acquiring the global initialization should only lock once in normal
290 // operation, which is why using it for TSDs access should not cause
291 // any interference.
292 HybridMutex Mutex;
293 TSD<Allocator> TSDs[TSDsArraySize];
294};
295
296} // namespace scudo
297
298#endif // SCUDO_TSD_SHARED_H_
299