1//===-- quarantine.h --------------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#ifndef SCUDO_QUARANTINE_H_
10#define SCUDO_QUARANTINE_H_
11
12#include "list.h"
13#include "mutex.h"
14#include "string_utils.h"
15#include "thread_annotations.h"
16
17namespace scudo {
18
19struct QuarantineBatch {
20 // With the following count, a batch (and the header that protects it) occupy
21 // 4096 bytes on 32-bit platforms, and 8192 bytes on 64-bit.
22 static const u32 MaxCount = 1019;
23 QuarantineBatch *Next;
24 uptr Size;
25 u32 Count;
26 void *Batch[MaxCount];
27
28 void init(void *Ptr, uptr Size) {
29 Count = 1;
30 Batch[0] = Ptr;
31 this->Size = Size + sizeof(QuarantineBatch); // Account for the Batch Size.
32 }
33
34 // The total size of quarantined nodes recorded in this batch.
35 uptr getQuarantinedSize() const { return Size - sizeof(QuarantineBatch); }
36
37 void push_back(void *Ptr, uptr Size) {
38 CHECK_LT(Count, MaxCount);
39 Batch[Count++] = Ptr;
40 this->Size += Size;
41 }
42
43 bool canMerge(const QuarantineBatch *const From) const {
44 // Validate both counts before computing the available capacity.
45 CHECK_LE(Count, MaxCount);
46 CHECK_LE(From->Count, MaxCount);
47 return From->Count <= MaxCount - Count;
48 }
49
50 void merge(QuarantineBatch *const From) {
51 CHECK(canMerge(From));
52 DCHECK_GE(Size, sizeof(QuarantineBatch));
53
54 for (uptr I = 0; I < From->Count; ++I)
55 Batch[Count + I] = From->Batch[I];
56 Count += From->Count;
57 Size += From->getQuarantinedSize();
58
59 From->Count = 0;
60 From->Size = sizeof(QuarantineBatch);
61 }
62
63 void shuffle(u32 State) {
64 CHECK_LE(Count, MaxCount);
65 ::scudo::shuffle(A: Batch, N: Count, RandState: &State);
66 }
67};
68
69static_assert(sizeof(QuarantineBatch) <= (1U << 13), ""); // 8Kb.
70
71// Per-thread cache of memory blocks.
72template <typename Callback> class QuarantineCache {
73public:
74 void init() { DCHECK_EQ(atomic_load_relaxed(&Size), 0U); }
75
76 // Total memory used, including internal accounting.
77 uptr getSize() const { return atomic_load_relaxed(A: &Size); }
78 // Memory used for internal accounting.
79 uptr getOverheadSize() const { return List.size() * sizeof(QuarantineBatch); }
80
81 void enqueue(Callback Cb, void *Ptr, uptr Size) {
82 if (List.empty() || List.back()->Count == QuarantineBatch::MaxCount) {
83 QuarantineBatch *B =
84 reinterpret_cast<QuarantineBatch *>(Cb.allocate(sizeof(*B)));
85 DCHECK(B);
86 B->init(Ptr, Size);
87 enqueueBatch(B);
88 } else {
89 List.back()->push_back(Ptr, Size);
90 addToSize(add: Size);
91 }
92 }
93
94 void transfer(QuarantineCache *From) {
95 List.append_back(L: &From->List);
96 addToSize(add: From->getSize());
97 atomic_store_relaxed(A: &From->Size, V: 0);
98 }
99
100 void enqueueBatch(QuarantineBatch *B) {
101 List.push_back(X: B);
102 addToSize(add: B->Size);
103 }
104
105 QuarantineBatch *dequeueBatch() {
106 if (List.empty())
107 return nullptr;
108 QuarantineBatch *B = List.front();
109 List.pop_front();
110 subFromSize(sub: B->Size);
111 return B;
112 }
113
114 void mergeBatches(QuarantineCache *ToDeallocate) {
115 uptr ExtractedSize = 0;
116 QuarantineBatch *Current = List.front();
117 while (Current && Current->Next) {
118 if (Current->canMerge(From: Current->Next)) {
119 QuarantineBatch *Extracted = Current->Next;
120 // Move all the chunks into the current batch.
121 Current->merge(From: Extracted);
122 DCHECK_EQ(Extracted->Count, 0);
123 DCHECK_EQ(Extracted->Size, sizeof(QuarantineBatch));
124 // Remove the next batch From the list and account for its Size.
125 List.extract(Prev: Current, X: Extracted);
126 ExtractedSize += Extracted->Size;
127 // Add it to deallocation list.
128 ToDeallocate->enqueueBatch(B: Extracted);
129 } else {
130 Current = Current->Next;
131 }
132 }
133 subFromSize(sub: ExtractedSize);
134 }
135
136 void getStats(ScopedString *Str) const {
137 uptr BatchCount = 0;
138 uptr TotalOverheadBytes = 0;
139 uptr TotalBytes = 0;
140 uptr TotalQuarantineChunks = 0;
141 for (const QuarantineBatch &Batch : List) {
142 BatchCount++;
143 TotalBytes += Batch.Size;
144 TotalOverheadBytes += Batch.Size - Batch.getQuarantinedSize();
145 TotalQuarantineChunks += Batch.Count;
146 }
147 const uptr QuarantineChunksCapacity =
148 BatchCount * QuarantineBatch::MaxCount;
149 const uptr ChunksUsagePercent =
150 (QuarantineChunksCapacity == 0)
151 ? 0
152 : TotalQuarantineChunks * 100 / QuarantineChunksCapacity;
153 const uptr TotalQuarantinedBytes = TotalBytes - TotalOverheadBytes;
154 const uptr MemoryOverheadPercent =
155 (TotalQuarantinedBytes == 0)
156 ? 0
157 : TotalOverheadBytes * 100 / TotalQuarantinedBytes;
158 Str->append(
159 Format: "Stats: Quarantine: batches: %zu; bytes: %zu (user: %zu); chunks: %zu "
160 "(capacity: %zu); %zu%% chunks used; %zu%% memory overhead\n",
161 BatchCount, TotalBytes, TotalQuarantinedBytes, TotalQuarantineChunks,
162 QuarantineChunksCapacity, ChunksUsagePercent, MemoryOverheadPercent);
163 }
164
165private:
166 SinglyLinkedList<QuarantineBatch> List;
167 atomic_uptr Size = {};
168
169 void addToSize(uptr add) { atomic_store_relaxed(&Size, getSize() + add); }
170 void subFromSize(uptr sub) { atomic_store_relaxed(&Size, getSize() - sub); }
171};
172
173// The callback interface is:
174// void Callback::recycle(Node *Ptr);
175// void *Callback::allocate(uptr Size);
176// void Callback::deallocate(void *Ptr);
177template <typename Callback, typename Node> class GlobalQuarantine {
178public:
179 typedef QuarantineCache<Callback> CacheT;
180 using ThisT = GlobalQuarantine<Callback, Node>;
181
182 void init(uptr Size, uptr CacheSize) NO_THREAD_SAFETY_ANALYSIS {
183 DCHECK(isAligned(reinterpret_cast<uptr>(this), alignof(ThisT)));
184 DCHECK_EQ(atomic_load_relaxed(&MaxSize), 0U);
185 DCHECK_EQ(atomic_load_relaxed(&MinSize), 0U);
186 DCHECK_EQ(atomic_load_relaxed(&MaxCacheSize), 0U);
187 // Thread local quarantine size can be zero only when global quarantine size
188 // is zero (it allows us to perform just one atomic read per put() call).
189 CHECK((Size == 0 && CacheSize == 0) || CacheSize != 0);
190
191 atomic_store_relaxed(A: &MaxSize, V: Size);
192 atomic_store_relaxed(A: &MinSize, V: Size / 10 * 9); // 90% of max size.
193 atomic_store_relaxed(A: &MaxCacheSize, V: CacheSize);
194
195 Cache.init();
196 }
197
198 uptr getMaxSize() const { return atomic_load_relaxed(A: &MaxSize); }
199 uptr getCacheSize() const { return atomic_load_relaxed(A: &MaxCacheSize); }
200
201 // This is supposed to be used in test only.
202 bool isEmpty() {
203 ScopedLock L(CacheMutex);
204 return Cache.getSize() == 0U;
205 }
206
207 void put(CacheT *C, Callback Cb, Node *Ptr, uptr Size) {
208 C->enqueue(Cb, Ptr, Size);
209 if (C->getSize() > getCacheSize())
210 drain(C, Cb);
211 }
212
213 void NOINLINE drain(CacheT *C, Callback Cb) EXCLUDES(CacheMutex) {
214 bool needRecycle = false;
215 {
216 ScopedLock L(CacheMutex);
217 Cache.transfer(C);
218 needRecycle = Cache.getSize() > getMaxSize();
219 }
220
221 if (needRecycle && RecycleMutex.tryLock())
222 recycle(MinSize: atomic_load_relaxed(A: &MinSize), Cb);
223 }
224
225 void NOINLINE drainAndRecycle(CacheT *C, Callback Cb) EXCLUDES(CacheMutex) {
226 {
227 ScopedLock L(CacheMutex);
228 Cache.transfer(C);
229 }
230 RecycleMutex.lock();
231 recycle(MinSize: 0, Cb);
232 }
233
234 void getStats(ScopedString *Str) EXCLUDES(CacheMutex) {
235 ScopedLock L(CacheMutex);
236 // It assumes that the world is stopped, just as the allocator's printStats.
237 Cache.getStats(Str);
238 Str->append("Quarantine limits: global: %zuK; thread local: %zuK\n",
239 getMaxSize() >> 10, getCacheSize() >> 10);
240 }
241
242 void disable() NO_THREAD_SAFETY_ANALYSIS {
243 // RecycleMutex must be locked 1st since we grab CacheMutex within recycle.
244 RecycleMutex.lock();
245 CacheMutex.lock();
246 }
247
248 void enable() NO_THREAD_SAFETY_ANALYSIS {
249 CacheMutex.unlock();
250 RecycleMutex.unlock();
251 }
252
253private:
254 // Read-only data.
255 alignas(SCUDO_CACHE_LINE_SIZE) HybridMutex CacheMutex;
256 CacheT Cache GUARDED_BY(CacheMutex);
257 alignas(SCUDO_CACHE_LINE_SIZE) HybridMutex RecycleMutex;
258 atomic_uptr MinSize = {};
259 atomic_uptr MaxSize = {};
260 alignas(SCUDO_CACHE_LINE_SIZE) atomic_uptr MaxCacheSize = {};
261
262 void NOINLINE recycle(uptr MinSize, Callback Cb) RELEASE(RecycleMutex)
263 EXCLUDES(CacheMutex) {
264 CacheT Tmp;
265 Tmp.init();
266 {
267 ScopedLock L(CacheMutex);
268 // Go over the batches and merge partially filled ones to
269 // save some memory, otherwise batches themselves (since the memory used
270 // by them is counted against quarantine limit) can overcome the actual
271 // user's quarantined chunks, which diminishes the purpose of the
272 // quarantine.
273 const uptr CacheSize = Cache.getSize();
274 const uptr OverheadSize = Cache.getOverheadSize();
275 DCHECK_GE(CacheSize, OverheadSize);
276 // Do the merge only when overhead exceeds this predefined limit (might
277 // require some tuning). It saves us merge attempt when the batch list
278 // quarantine is unlikely to contain batches suitable for merge.
279 constexpr uptr OverheadThresholdPercents = 100;
280 if (CacheSize > OverheadSize &&
281 OverheadSize * (100 + OverheadThresholdPercents) >
282 CacheSize * OverheadThresholdPercents) {
283 Cache.mergeBatches(&Tmp);
284 }
285 // Extract enough chunks from the quarantine to get below the max
286 // quarantine size and leave some leeway for the newly quarantined chunks.
287 while (Cache.getSize() > MinSize)
288 Tmp.enqueueBatch(Cache.dequeueBatch());
289 }
290 RecycleMutex.unlock();
291 doRecycle(C: &Tmp, Cb);
292 }
293
294 void NOINLINE doRecycle(CacheT *C, Callback Cb) {
295 while (QuarantineBatch *B = C->dequeueBatch()) {
296 const u32 Seed = static_cast<u32>(
297 (reinterpret_cast<uptr>(B) ^ reinterpret_cast<uptr>(C)) >> 4);
298 B->shuffle(State: Seed);
299 constexpr uptr NumberOfPrefetch = 8UL;
300 CHECK(NumberOfPrefetch <= ARRAY_SIZE(B->Batch));
301 for (uptr I = 0; I < NumberOfPrefetch; I++)
302 PREFETCH(B->Batch[I]);
303 for (uptr I = 0, Count = B->Count; I < Count; I++) {
304 if (I + NumberOfPrefetch < Count)
305 PREFETCH(B->Batch[I + NumberOfPrefetch]);
306 Cb.recycle(reinterpret_cast<Node *>(B->Batch[I]));
307 }
308 Cb.deallocate(B);
309 }
310 }
311};
312
313} // namespace scudo
314
315#endif // SCUDO_QUARANTINE_H_
316