1//===-- secondary.h ---------------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#ifndef SCUDO_SECONDARY_H_
10#define SCUDO_SECONDARY_H_
11
12#ifndef __STDC_FORMAT_MACROS
13// Ensure PRId64 macro is available
14#define __STDC_FORMAT_MACROS 1
15#endif
16#include <inttypes.h>
17
18#include "chunk.h"
19#include "common.h"
20#include "list.h"
21#include "mem_map.h"
22#include "memtag.h"
23#include "mutex.h"
24#include "options.h"
25#include "stats.h"
26#include "string_utils.h"
27#include "thread_annotations.h"
28#include "tracing.h"
29#include "vector.h"
30
31namespace scudo {
32
33// This allocator wraps the platform allocation primitives, and as such is on
34// the slower side and should preferably be used for larger sized allocations.
35// Blocks allocated will be preceded and followed by a guard page, and hold
36// their own header that is not checksummed: the guard pages and the Combined
37// header should be enough for our purpose.
38
39namespace LargeBlock {
40
41struct alignas(Max<uptr>(A: archSupportsMemoryTagging()
42 ? archMemoryTagGranuleSize()
43 : 1,
44 B: 1U << SCUDO_MIN_ALIGNMENT_LOG)) Header {
45 LargeBlock::Header *Prev;
46 LargeBlock::Header *Next;
47 uptr CommitBase;
48 uptr CommitSize;
49 MemMapT MemMap;
50};
51
52static_assert(sizeof(Header) % (1U << SCUDO_MIN_ALIGNMENT_LOG) == 0, "");
53static_assert(!archSupportsMemoryTagging() ||
54 sizeof(Header) % archMemoryTagGranuleSize() == 0,
55 "");
56
57constexpr uptr getHeaderSize() { return sizeof(Header); }
58
59template <typename Config> uptr addHeaderTag(uptr Ptr) {
60 if (allocatorSupportsMemoryTagging<Config>())
61 return addFixedTag(Ptr, Tag: 1);
62 return Ptr;
63}
64
65template <typename Config> Header *getHeader(uptr Ptr) {
66 return reinterpret_cast<Header *>(addHeaderTag<Config>(Ptr)) - 1;
67}
68
69template <typename Config> Header *getHeader(const void *Ptr) {
70 return getHeader<Config>(reinterpret_cast<uptr>(Ptr));
71}
72
73} // namespace LargeBlock
74
75static inline void unmap(MemMapT &MemMap) { MemMap.unmap(); }
76
77namespace {
78
79struct CachedBlock {
80 static constexpr u16 CacheIndexMax = UINT16_MAX;
81 static constexpr u16 EndOfListVal = CacheIndexMax;
82
83 // We allow a certain amount of fragmentation and part of the fragmented bytes
84 // will be released by `releaseAndZeroPagesToOS()`. This increases the chance
85 // of cache hit rate and reduces the overhead to the RSS at the same time. See
86 // more details in the `MapAllocatorCache::retrieve()` section.
87 //
88 // We arrived at this default value after noticing that mapping in larger
89 // memory regions performs better than releasing memory and forcing a cache
90 // hit. According to the data, it suggests that beyond 4 pages, the release
91 // execution time is longer than the map execution time. In this way,
92 // the default is dependent on the platform.
93 static constexpr uptr MaxReleasedCachePages = 4U;
94
95 uptr CommitBase = 0;
96 uptr CommitSize = 0;
97 uptr BlockBegin = 0;
98 MemMapT MemMap = {};
99 u64 Time = 0;
100 u16 Next = 0;
101 u16 Prev = 0;
102
103 enum CacheFlags : u16 {
104 None = 0,
105 NoAccess = 0x1,
106 Zeroed = 0x2,
107 };
108 u16 Flags = CachedBlock::None;
109
110 bool isZero() { return (Flags & CacheFlags::Zeroed) != 0; }
111
112 bool isValid() { return CommitBase != 0; }
113
114 void invalidate() { CommitBase = 0; }
115};
116} // namespace
117
118template <typename Config> class MapAllocatorNoCache {
119public:
120 void init(UNUSED s32 ReleaseToOsInterval) {}
121 CachedBlock retrieve(UNUSED uptr MaxAllowedFragmentedBytes, UNUSED uptr Size,
122 UNUSED uptr Alignment, UNUSED uptr HeadersSize,
123 UNUSED uptr &EntryHeaderPos) {
124 return {};
125 }
126 void store(UNUSED Options Options, UNUSED uptr CommitBase,
127 UNUSED uptr CommitSize, UNUSED uptr BlockBegin,
128 UNUSED MemMapT MemMap, UNUSED bool Zero) {
129 // This should never be called since canCache always returns false.
130 UNREACHABLE(
131 "It is not valid to call store on MapAllocatorNoCache objects.");
132 }
133
134 bool canCache(UNUSED uptr Size) { return false; }
135 void disable() {}
136 void enable() {}
137 void releaseToOS(ReleaseToOS) {}
138 void disableMemoryTagging() {}
139 void unmapTestOnly() {}
140 bool setOption(Option O, UNUSED sptr Value) {
141 if (O == Option::ReleaseInterval || O == Option::MaxCacheEntriesCount ||
142 O == Option::MaxCacheEntrySize || O == Option::MaxCacheResidentBytes)
143 return false;
144 // Not supported by the Secondary Cache, but not an error either.
145 return true;
146 }
147
148 uptr getMaxResidentBytesTestOnly() const { return 0; }
149 uptr getCurrentResidentBytesTestOnly() const { return 0; }
150
151 void getStats(UNUSED ScopedString *Str) {
152 Str->append(Format: "Secondary Cache Disabled\n");
153 }
154};
155
156static const uptr MaxUnreleasedCachePages = 4U;
157
158template <typename Config>
159bool mapSecondary(const Options &Options, uptr CommitBase, uptr CommitSize,
160 uptr AllocPos, uptr Flags, MemMapT &MemMap) {
161 Flags |= MAP_RESIZABLE;
162 Flags |= MAP_ALLOWNOMEM;
163
164 const uptr PageSize = getPageSizeCached();
165 if (SCUDO_TRUSTY) {
166 /*
167 * On Trusty we need AllocPos to be usable for shared memory, which cannot
168 * cross multiple mappings. This means we need to split around AllocPos
169 * and not over it. We can only do this if the address is page-aligned.
170 */
171 const uptr TaggedSize = AllocPos - CommitBase;
172 if (useMemoryTagging<Config>(Options) && isAligned(X: TaggedSize, Alignment: PageSize)) {
173 DCHECK_GT(TaggedSize, 0);
174 return MemMap.remap(Addr: CommitBase, Size: TaggedSize, Name: "scudo:secondary",
175 MAP_MEMTAG | Flags) &&
176 MemMap.remap(Addr: AllocPos, Size: CommitSize - TaggedSize, Name: "scudo:secondary",
177 Flags);
178 } else {
179 const uptr RemapFlags =
180 (useMemoryTagging<Config>(Options) ? MAP_MEMTAG : 0) | Flags;
181 return MemMap.remap(Addr: CommitBase, Size: CommitSize, Name: "scudo:secondary",
182 Flags: RemapFlags);
183 }
184 }
185
186 // AllocPos is assumed to be page-aligned when memory tagging is enabled.
187 // Therefore the page right before AllocPos is MTE-tagged and the header
188 // resides in this MTE-tagged page.
189 if (useMemoryTagging<Config>(Options)) {
190 const uptr PageSize = getPageSizeCached();
191 const uptr MteStart = AllocPos - PageSize;
192
193 DCHECK(AllocPos % PageSize == 0U);
194 DCHECK(MteStart % PageSize == 0U);
195
196 DCHECK_GE(MteStart, CommitBase);
197 DCHECK_LE(AllocPos, CommitBase + CommitSize);
198 return MemMap.remap(Addr: MteStart, Size: PageSize, Name: "scudo:secondary",
199 MAP_MEMTAG | Flags) &&
200 MemMap.remap(Addr: AllocPos, Size: CommitBase + CommitSize - AllocPos,
201 Name: "scudo:secondary", Flags);
202 } else {
203 const uptr RemapFlags =
204 (useMemoryTagging<Config>(Options) ? MAP_MEMTAG : 0) | Flags;
205 return MemMap.remap(Addr: CommitBase, Size: CommitSize, Name: "scudo:secondary", Flags: RemapFlags);
206 }
207}
208
209// Template specialization to avoid producing zero-length array
210template <typename T, size_t Size> class NonZeroLengthArray {
211public:
212 T &operator[](uptr Idx) { return values[Idx]; }
213
214private:
215 T values[Size];
216};
217template <typename T> class NonZeroLengthArray<T, 0> {
218public:
219 T &operator[](uptr UNUSED Idx) { UNREACHABLE("Unsupported!"); }
220};
221
222// The default unmap callback is simply scudo::unmap.
223// In testing, a different unmap callback is used to
224// record information about unmaps in the cache
225template <typename Config, void (*unmapCallBack)(MemMapT &) = unmap>
226class MapAllocatorCache {
227public:
228 void getStats(ScopedString *Str) {
229 ScopedLock L(Mutex);
230 Str->append(Format: "Config Stats Secondary: ");
231 Config::getConfigValues(Str);
232 uptr Integral;
233 uptr Fractional;
234 computePercentage(Numerator: SuccessfulRetrieves, Denominator: CallsToRetrieve, Integral: &Integral,
235 Fractional: &Fractional);
236 const s32 Interval = atomic_load_relaxed(A: &ReleaseToOsIntervalMs);
237 Str->append(
238 Format: "Stats: MapAllocatorCache: EntriesCount: %zu, "
239 "MaxEntriesCount: %u, MaxEntrySize: %zu, ReleaseToOsSkips: "
240 "%zu, ReleaseToOsIntervalMs = %d, Unmapped due to eviction: %u, "
241 "MaxResidentBytes: %zu, CurrentResidentBytes: %zu\n",
242 LRUEntries.size(), atomic_load_relaxed(A: &MaxEntriesCount),
243 atomic_load_relaxed(A: &MaxEntrySize),
244 atomic_load_relaxed(A: &ReleaseToOsSkips), Interval >= 0 ? Interval : -1,
245 EvictedCount, MaxResidentBytes, CurrentResidentBytes);
246 Str->append(Format: "Stats: CacheRetrievalStats: SuccessRate: %u/%u "
247 "(%zu.%02zu%%)\n",
248 SuccessfulRetrieves, CallsToRetrieve, Integral, Fractional);
249 Str->append(Format: "Cache Entry Info (Most Recent -> Least Recent):\n");
250
251 for (CachedBlock &Entry : LRUEntries) {
252 Str->append(Format: " StartBlockAddress: 0x%zx, EndBlockAddress: 0x%zx, "
253 "BlockSize: %zu%s, Flags: %s",
254 Entry.CommitBase, Entry.CommitBase + Entry.CommitSize,
255 Entry.CommitSize, Entry.Time == 0 ? " [R]" : "",
256 Entry.Flags & CachedBlock::NoAccess ? "NoAccess" : "None");
257 const s64 ResidentPages =
258 Entry.MemMap.getResidentPages(From: Entry.CommitBase, Size: Entry.CommitSize);
259
260 if (ResidentPages >= 0) {
261 Str->append(Format: ", Resident Pages: %" PRId64 "/%zu", ResidentPages,
262 Entry.CommitSize / getPageSizeCached());
263 }
264 Str->append(Format: "\n");
265 }
266 }
267
268 // Ensure the default maximum specified fits the array.
269 static_assert(Config::getDefaultMaxEntriesCount() <=
270 Config::getEntriesArraySize(),
271 "");
272 // Ensure the cache entry array size fits in the LRU list Next and Prev
273 // index fields
274 static_assert(Config::getEntriesArraySize() <= CachedBlock::CacheIndexMax,
275 "Cache entry array is too large to be indexed.");
276
277 void init(s32 ReleaseToOsInterval) NO_THREAD_SAFETY_ANALYSIS {
278 DCHECK_EQ(LRUEntries.size(), 0U);
279 setOption(O: Option::MaxCacheEntriesCount,
280 Value: static_cast<sptr>(Config::getDefaultMaxEntriesCount()));
281 setOption(O: Option::MaxCacheEntrySize,
282 Value: static_cast<sptr>(Config::getDefaultMaxEntrySize()));
283 setOption(O: Option::MaxCacheResidentBytes,
284 Value: static_cast<sptr>(Config::getDefaultMaxCacheResidentBytes()));
285 // The default value in the cache config has the higher priority.
286 if (Config::getDefaultReleaseToOsIntervalMs() != INT32_MIN)
287 ReleaseToOsInterval = Config::getDefaultReleaseToOsIntervalMs();
288 setOption(O: Option::ReleaseInterval, Value: static_cast<sptr>(ReleaseToOsInterval));
289
290 LRUEntries.clear();
291 LRUEntries.init(Base: Entries, BaseSize: sizeof(Entries));
292 OldestPresentEntry = nullptr;
293
294 AvailEntries.clear();
295 AvailEntries.init(Base: Entries, BaseSize: sizeof(Entries));
296 for (u32 I = 0; I < Config::getEntriesArraySize(); I++)
297 AvailEntries.push_back(X: &Entries[I]);
298 }
299
300 void store(const Options &Options, uptr CommitBase, uptr CommitSize,
301 uptr BlockBegin, MemMapT MemMap, bool Zero) EXCLUDES(Mutex) {
302 DCHECK(canCache(CommitSize));
303
304 const s32 Interval = atomic_load_relaxed(A: &ReleaseToOsIntervalMs);
305 u64 Time;
306 CachedBlock Entry;
307
308 Entry.CommitBase = CommitBase;
309 Entry.CommitSize = CommitSize;
310 Entry.BlockBegin = BlockBegin;
311 Entry.MemMap = MemMap;
312 Entry.Time = UINT64_MAX;
313 Entry.Flags = CachedBlock::None;
314
315 bool MemoryTaggingEnabled = useMemoryTagging<Config>(Options);
316 if (MemoryTaggingEnabled) {
317 if ((Interval == 0 || Zero) && !SCUDO_FUCHSIA) {
318 Entry.MemMap.releaseAndZeroPagesToOS(From: Entry.CommitBase,
319 Size: Entry.CommitSize);
320 Entry.Time = 0;
321 Entry.Flags |= CachedBlock::Zeroed;
322 }
323 // MAP_NOACCESS or PROT_NONE does not strip PROT_MTE.
324 Entry.MemMap.setMemoryPermission(Addr: Entry.CommitBase, Size: Entry.CommitSize,
325 MAP_NOACCESS);
326 Entry.Flags |= CachedBlock::NoAccess;
327 } else if (Zero) {
328 // Make sure to zero outside of the lock.
329 memset(s: reinterpret_cast<void *>(Entry.CommitBase), c: 0, n: Entry.CommitSize);
330 Entry.Flags |= CachedBlock::Zeroed;
331 }
332
333 // Usually only one entry will be evicted from the cache.
334 // Only in the rare event that the cache shrinks in real-time
335 // due to a decrease in the configurable value MaxEntriesCount
336 // will more than one cache entry be evicted.
337 // The vector is used to save the MemMaps of evicted entries so
338 // that the unmap call can be performed outside the lock
339 Vector<MemMapT, 1U> EvictionMemMaps;
340
341 do {
342 ScopedLock L(Mutex);
343
344 // Time must be computed under the lock to ensure
345 // that the LRU cache remains sorted with respect to
346 // time in a multithreaded environment
347 Time = getMonotonicTimeFast();
348 if (Entry.Time != 0)
349 Entry.Time = Time;
350
351 if (MemoryTaggingEnabled && !useMemoryTagging<Config>(Options)) {
352 // If we get here then memory tagging was disabled in between when we
353 // read Options and when we locked Mutex. We can't insert our entry into
354 // the quarantine or the cache because the permissions would be wrong so
355 // just unmap it.
356 unmapCallBack(Entry.MemMap);
357 break;
358 }
359
360 if (!Config::getQuarantineDisabled() && Config::getQuarantineSize()) {
361 QuarantinePos =
362 (QuarantinePos + 1) % Max(Config::getQuarantineSize(), 1u);
363 if (!Quarantine[QuarantinePos].isValid()) {
364 Quarantine[QuarantinePos] = Entry;
365 return;
366 }
367 CachedBlock PrevEntry = Quarantine[QuarantinePos];
368 Quarantine[QuarantinePos] = Entry;
369 Entry = PrevEntry;
370 }
371
372 // All excess entries are evicted from the cache. Note that when
373 // `MaxEntriesCount` is zero, cache storing shouldn't happen and it's
374 // guarded by the `DCHECK(canCache(CommitSize))` above. As a result, we
375 // won't try to pop `LRUEntries` when it's empty.
376 while (LRUEntries.size() >= atomic_load_relaxed(A: &MaxEntriesCount)) {
377 // Save MemMaps of evicted entries to perform unmap outside of lock
378 CachedBlock *Entry = LRUEntries.back();
379 EvictedCount++;
380 EvictionMemMaps.push_back(Element: Entry->MemMap);
381 remove(Entry);
382 }
383
384 insert(Entry);
385 trimResidentBytes(MaxResidentBytesLimit: atomic_load_relaxed(A: &MaxCacheResidentBytes));
386 } while (0);
387
388 for (MemMapT &EvictMemMap : EvictionMemMaps)
389 unmapCallBack(EvictMemMap);
390
391 if (Interval >= 0) {
392 // It is very likely that multiple threads trying to do a release at the
393 // same time will not actually release any extra elements. Therefore,
394 // let any other thread continue, skipping the release.
395 const u64 IntervalTime = static_cast<u64>(Interval) * 1000000;
396 if (LIKELY(Time > IntervalTime) && Mutex.tryLock()) {
397 SCUDO_SCOPED_TRACE(
398 GetSecondaryReleaseToOSTraceName(ReleaseToOS::Normal));
399
400 releaseOlderThan(ReleaseTime: Time - IntervalTime);
401 Mutex.unlock();
402 } else
403 atomic_fetch_add(A: &ReleaseToOsSkips, V: 1U, MO: memory_order_relaxed);
404 }
405 }
406
407 CachedBlock retrieve(uptr MaxAllowedFragmentedPages, uptr Size,
408 uptr Alignment, uptr HeadersSize, uptr &EntryHeaderPos)
409 EXCLUDES(Mutex) {
410 const uptr PageSize = getPageSizeCached();
411 // 10% of the requested size proved to be the optimal choice for
412 // retrieving cached blocks after testing several options.
413 constexpr u32 FragmentedBytesDivisor = 10;
414 CachedBlock Entry;
415 EntryHeaderPos = 0;
416 {
417 ScopedLock L(Mutex);
418 CallsToRetrieve++;
419 if (LRUEntries.size() == 0)
420 return {};
421 CachedBlock *RetrievedEntry = nullptr;
422 uptr MinDiff = UINTPTR_MAX;
423
424 // Since allocation sizes don't always match cached memory chunk sizes
425 // we allow some memory to be unused (called fragmented bytes). The
426 // amount of unused bytes is exactly EntryHeaderPos - CommitBase.
427 //
428 // CommitBase CommitBase + CommitSize
429 // V V
430 // +---+------------+-----------------+---+
431 // | | | | |
432 // +---+------------+-----------------+---+
433 // ^ ^ ^
434 // Guard EntryHeaderPos Guard-page-end
435 // page-begin
436 //
437 // [EntryHeaderPos, CommitBase + CommitSize) contains the user data as
438 // well as the header metadata. If EntryHeaderPos - CommitBase exceeds
439 // MaxAllowedFragmentedPages * PageSize, the cached memory chunk is
440 // not considered valid for retrieval.
441 for (CachedBlock &Entry : LRUEntries) {
442 const uptr CommitBase = Entry.CommitBase;
443 const uptr CommitSize = Entry.CommitSize;
444 const uptr AllocPos =
445 roundDown(X: CommitBase + CommitSize - Size, Boundary: Alignment);
446 const uptr HeaderPos = AllocPos - HeadersSize;
447 const uptr MaxAllowedFragmentedBytes =
448 MaxAllowedFragmentedPages * PageSize;
449 if (HeaderPos > CommitBase + CommitSize)
450 continue;
451 // TODO: Remove AllocPos > CommitBase + MaxAllowedFragmentedBytes
452 // and replace with Diff > MaxAllowedFragmentedBytes
453 if (HeaderPos < CommitBase ||
454 AllocPos > CommitBase + MaxAllowedFragmentedBytes) {
455 continue;
456 }
457
458 const uptr Diff = roundDown(X: HeaderPos, Boundary: PageSize) - CommitBase;
459
460 // Keep track of the smallest cached block
461 // that is greater than (AllocSize + HeaderSize)
462 if (Diff >= MinDiff)
463 continue;
464
465 MinDiff = Diff;
466 RetrievedEntry = &Entry;
467 EntryHeaderPos = HeaderPos;
468
469 // Immediately use a cached block if its size is close enough to the
470 // requested size
471 const uptr OptimalFitThesholdBytes =
472 (CommitBase + CommitSize - HeaderPos) / FragmentedBytesDivisor;
473 if (Diff <= OptimalFitThesholdBytes)
474 break;
475 }
476
477 if (RetrievedEntry != nullptr) {
478 Entry = *RetrievedEntry;
479 remove(Entry: RetrievedEntry);
480 SuccessfulRetrieves++;
481 }
482 }
483
484 // The difference between the retrieved memory chunk and the request
485 // size is at most MaxAllowedFragmentedPages
486 //
487 // +- MaxAllowedFragmentedPages * PageSize -+
488 // +--------------------------+-------------+
489 // | | |
490 // +--------------------------+-------------+
491 // \ Bytes to be released / ^
492 // |
493 // (may or may not be committed)
494 //
495 // The maximum number of bytes released to the OS is capped by
496 // MaxReleasedCachePages
497 //
498 // TODO : Consider making MaxReleasedCachePages configurable since
499 // the release to OS API can vary across systems.
500 if (Entry.Time != 0) {
501 const uptr FragmentedBytes =
502 roundDown(X: EntryHeaderPos, Boundary: PageSize) - Entry.CommitBase;
503 const uptr MaxUnreleasedCacheBytes = MaxUnreleasedCachePages * PageSize;
504 if (FragmentedBytes > MaxUnreleasedCacheBytes) {
505 const uptr MaxReleasedCacheBytes =
506 CachedBlock::MaxReleasedCachePages * PageSize;
507 uptr BytesToRelease =
508 roundUp(X: Min<uptr>(A: MaxReleasedCacheBytes,
509 B: FragmentedBytes - MaxUnreleasedCacheBytes),
510 Boundary: PageSize);
511 Entry.MemMap.releaseAndZeroPagesToOS(From: Entry.CommitBase, Size: BytesToRelease);
512 }
513 }
514
515 return Entry;
516 }
517
518 bool canCache(uptr Size) {
519 return atomic_load_relaxed(A: &MaxEntriesCount) != 0U &&
520 Size <= atomic_load_relaxed(A: &MaxEntrySize);
521 }
522
523 bool setOption(Option O, sptr Value) {
524 if (O == Option::ReleaseInterval) {
525 const s32 Interval = Max(
526 Min(static_cast<s32>(Value), Config::getMaxReleaseToOsIntervalMs()),
527 Config::getMinReleaseToOsIntervalMs());
528 atomic_store_relaxed(A: &ReleaseToOsIntervalMs, V: Interval);
529 if (Interval >= 0) {
530 // Always trigger a trim if the interval is not being disabled.
531 ScopedLock L(Mutex);
532 trimResidentBytes(MaxResidentBytesLimit: atomic_load_relaxed(A: &MaxCacheResidentBytes));
533 }
534 return true;
535 }
536 if (O == Option::MaxCacheEntriesCount) {
537 if (Value < 0)
538 return false;
539 atomic_store_relaxed(
540 &MaxEntriesCount,
541 Min<u32>(static_cast<u32>(Value), Config::getEntriesArraySize()));
542 return true;
543 }
544 if (O == Option::MaxCacheEntrySize) {
545 atomic_store_relaxed(A: &MaxEntrySize, V: static_cast<uptr>(Value));
546 return true;
547 }
548 if (O == Option::MaxCacheResidentBytes) {
549 if (Value < 0)
550 return false;
551 const uptr NewMaxResidentBytes = static_cast<uptr>(Value);
552 atomic_store_relaxed(A: &MaxCacheResidentBytes, V: NewMaxResidentBytes);
553 if (NewMaxResidentBytes != 0) {
554 ScopedLock L(Mutex);
555 trimResidentBytes(MaxResidentBytesLimit: NewMaxResidentBytes);
556 }
557 return true;
558 }
559 // Not supported by the Secondary Cache, but not an error either.
560 return true;
561 }
562
563 void releaseToOS([[maybe_unused]] ReleaseToOS ReleaseType) EXCLUDES(Mutex) {
564 SCUDO_SCOPED_TRACE(GetSecondaryReleaseToOSTraceName(ReleaseType));
565
566 if (ReleaseType == ReleaseToOS::ForceFast) {
567 // Never wait for the lock, always move on if there is already
568 // a release operation in progress.
569 if (Mutex.tryLock()) {
570 releaseOlderThan(UINT64_MAX);
571 Mutex.unlock();
572 }
573 } else {
574 // Since this is a request to release everything, always wait for the
575 // lock so that we guarantee all entries are released after this call.
576 ScopedLock L(Mutex);
577 releaseOlderThan(UINT64_MAX);
578 }
579 }
580
581 void disableMemoryTagging() EXCLUDES(Mutex) {
582 if (Config::getQuarantineDisabled())
583 return;
584
585 ScopedLock L(Mutex);
586 for (u32 I = 0; I != Config::getQuarantineSize(); ++I) {
587 if (Quarantine[I].isValid()) {
588 MemMapT &MemMap = Quarantine[I].MemMap;
589 unmapCallBack(MemMap);
590 Quarantine[I].invalidate();
591 }
592 }
593 // Reset to 0 (matching initial state) rather than UINT32_MAX so that
594 // (QuarantinePos + 1) in store() does not overflow u32.
595 QuarantinePos = 0;
596 }
597
598 void disable() NO_THREAD_SAFETY_ANALYSIS { Mutex.lock(); }
599
600 void enable() NO_THREAD_SAFETY_ANALYSIS { Mutex.unlock(); }
601
602 uptr getMaxResidentBytesTestOnly() {
603 ScopedLock L(Mutex);
604 return MaxResidentBytes;
605 }
606
607 uptr getCurrentResidentBytesTestOnly() {
608 ScopedLock L(Mutex);
609 return CurrentResidentBytes;
610 }
611
612 void unmapTestOnly() { empty(); }
613
614 void releaseOlderThanTestOnly(u64 ReleaseTime) {
615 ScopedLock L(Mutex);
616 releaseOlderThan(ReleaseTime);
617 }
618
619private:
620 void insert(const CachedBlock &Entry) REQUIRES(Mutex) {
621 CachedBlock *AvailEntry = AvailEntries.front();
622 AvailEntries.pop_front();
623
624 *AvailEntry = Entry;
625 LRUEntries.push_front(X: AvailEntry);
626 if (OldestPresentEntry == nullptr && AvailEntry->Time != 0)
627 OldestPresentEntry = AvailEntry;
628 if (AvailEntry->Time != 0) {
629 CurrentResidentBytes += Entry.CommitSize;
630 if (CurrentResidentBytes > MaxResidentBytes)
631 MaxResidentBytes = CurrentResidentBytes;
632 }
633 }
634
635 void remove(CachedBlock *Entry) REQUIRES(Mutex) {
636 DCHECK(Entry->isValid());
637 if (OldestPresentEntry == Entry) {
638 OldestPresentEntry = LRUEntries.getPrev(X: Entry);
639 DCHECK(OldestPresentEntry == nullptr || OldestPresentEntry->Time != 0);
640 }
641 LRUEntries.remove(X: Entry);
642 if (Entry->Time != 0)
643 CurrentResidentBytes -= Entry->CommitSize;
644 Entry->invalidate();
645 AvailEntries.push_front(X: Entry);
646 }
647
648 ALWAYS_INLINE void trimResidentBytes(uptr MaxResidentBytesLimit)
649 REQUIRES(Mutex) {
650 if (MaxResidentBytesLimit == 0 ||
651 atomic_load_relaxed(A: &ReleaseToOsIntervalMs) < 0)
652 return;
653 while (CurrentResidentBytes > MaxResidentBytesLimit &&
654 OldestPresentEntry != nullptr) {
655 CachedBlock *Entry = OldestPresentEntry;
656 OldestPresentEntry = LRUEntries.getPrev(X: Entry);
657 Entry->MemMap.releaseAndZeroPagesToOS(From: Entry->CommitBase,
658 Size: Entry->CommitSize);
659 CurrentResidentBytes -= Entry->CommitSize;
660 Entry->Time = 0;
661 Entry->Flags |= CachedBlock::Zeroed;
662 }
663 }
664
665 void empty() {
666 MemMapT MapInfo[Config::getEntriesArraySize()];
667 uptr N = 0;
668 {
669 ScopedLock L(Mutex);
670
671 for (CachedBlock &Entry : LRUEntries)
672 MapInfo[N++] = Entry.MemMap;
673 LRUEntries.clear();
674 OldestPresentEntry = nullptr;
675 CurrentResidentBytes = 0;
676 }
677 for (uptr I = 0; I < N; I++) {
678 MemMapT &MemMap = MapInfo[I];
679 unmapCallBack(MemMap);
680 }
681 }
682
683 void releaseOlderThan(u64 ReleaseTime) REQUIRES(Mutex) {
684 SCUDO_SCOPED_TRACE(GetSecondaryReleaseOlderThanTraceName());
685
686 if (!Config::getQuarantineDisabled()) {
687 for (uptr I = 0; I < Config::getQuarantineSize(); I++) {
688 auto &Entry = Quarantine[I];
689 if (!Entry.isValid() || Entry.Time == 0 || Entry.Time > ReleaseTime)
690 continue;
691 Entry.MemMap.releaseAndZeroPagesToOS(Entry.CommitBase,
692 Entry.CommitSize);
693 Entry.Time = 0;
694 Entry.Flags |= CachedBlock::Zeroed;
695 }
696 }
697
698 for (CachedBlock *Entry = OldestPresentEntry; Entry != nullptr;
699 Entry = LRUEntries.getPrev(X: Entry)) {
700 DCHECK(Entry->isValid());
701 DCHECK(Entry->Time != 0);
702
703 if (Entry->Time > ReleaseTime) {
704 // All entries are newer than this, so no need to keep scanning.
705 OldestPresentEntry = Entry;
706 return;
707 }
708
709 Entry->MemMap.releaseAndZeroPagesToOS(From: Entry->CommitBase,
710 Size: Entry->CommitSize);
711 CurrentResidentBytes -= Entry->CommitSize;
712 Entry->Time = 0;
713 Entry->Flags |= CachedBlock::Zeroed;
714 }
715 OldestPresentEntry = nullptr;
716 }
717
718 HybridMutex Mutex;
719 u32 QuarantinePos GUARDED_BY(Mutex) = 0;
720 atomic_u32 MaxEntriesCount = {};
721 atomic_uptr MaxEntrySize = {};
722 atomic_uptr MaxCacheResidentBytes = {};
723 atomic_s32 ReleaseToOsIntervalMs = {};
724 u32 CallsToRetrieve GUARDED_BY(Mutex) = 0;
725 u32 SuccessfulRetrieves GUARDED_BY(Mutex) = 0;
726 u32 EvictedCount GUARDED_BY(Mutex) = 0;
727 uptr CurrentResidentBytes GUARDED_BY(Mutex) = 0;
728 uptr MaxResidentBytes GUARDED_BY(Mutex) = 0;
729 atomic_uptr ReleaseToOsSkips = {};
730
731 CachedBlock Entries[Config::getEntriesArraySize()] GUARDED_BY(Mutex) = {};
732 NonZeroLengthArray<CachedBlock, Config::getQuarantineSize()>
733 Quarantine GUARDED_BY(Mutex) = {};
734
735 // The oldest entry in the LRUEntries that has Time non-zero.
736 CachedBlock *OldestPresentEntry GUARDED_BY(Mutex) = nullptr;
737 // Cached blocks stored in LRU order
738 DoublyLinkedList<CachedBlock> LRUEntries GUARDED_BY(Mutex);
739 // The unused Entries
740 SinglyLinkedList<CachedBlock> AvailEntries GUARDED_BY(Mutex);
741};
742
743template <typename Config> class MapAllocator {
744public:
745 void init(GlobalStats *S,
746 s32 ReleaseToOsInterval = -1) NO_THREAD_SAFETY_ANALYSIS {
747 DCHECK_EQ(AllocatedBytes, 0U);
748 DCHECK_EQ(FreedBytes, 0U);
749 Cache.init(ReleaseToOsInterval);
750 Stats.init();
751 if (LIKELY(S))
752 S->link(S: &Stats);
753 }
754
755 void *allocate(const Options &Options, uptr Size, uptr AlignmentHint = 0,
756 uptr *BlockEnd = nullptr,
757 FillContentsMode FillContents = NoFill);
758
759 void deallocate(const Options &Options, void *Ptr, bool Zero);
760
761 void *tryAllocateFromCache(const Options &Options, uptr Size, uptr Alignment,
762 uptr *BlockEndPtr, FillContentsMode FillContents);
763
764 static uptr getBlockEnd(void *Ptr) {
765 auto *B = LargeBlock::getHeader<Config>(Ptr);
766 return B->CommitBase + B->CommitSize;
767 }
768
769 static uptr getBlockSize(void *Ptr) {
770 return getBlockEnd(Ptr) - reinterpret_cast<uptr>(Ptr);
771 }
772
773 static uptr getGuardPageSize() {
774 if (Config::getEnableGuardPages())
775 return getPageSizeCached();
776 return 0U;
777 }
778
779 static constexpr uptr getHeadersSize() {
780 return Chunk::getHeaderSize() + LargeBlock::getHeaderSize();
781 }
782
783 void disable() NO_THREAD_SAFETY_ANALYSIS {
784 Mutex.lock();
785 Cache.disable();
786 }
787
788 void enable() NO_THREAD_SAFETY_ANALYSIS {
789 Cache.enable();
790 Mutex.unlock();
791 }
792
793 template <typename F> void iterateOverBlocks(F Callback) const {
794 Mutex.assertHeld();
795
796 for (const auto &H : InUseBlocks) {
797 uptr Ptr = reinterpret_cast<uptr>(&H) + LargeBlock::getHeaderSize();
798 if (allocatorSupportsMemoryTagging<Config>())
799 Ptr = untagPointer(Ptr);
800 Callback(Ptr);
801 }
802 }
803
804 bool canCache(uptr Size) { return Cache.canCache(Size); }
805
806 bool setOption(Option O, sptr Value) { return Cache.setOption(O, Value); }
807
808 void releaseToOS(ReleaseToOS ReleaseType) { Cache.releaseToOS(ReleaseType); }
809
810 void disableMemoryTagging() { Cache.disableMemoryTagging(); }
811
812 void unmapTestOnly() { Cache.unmapTestOnly(); }
813
814 uptr getMaxResidentBytesTestOnly() {
815 return Cache.getMaxResidentBytesTestOnly();
816 }
817
818 uptr getCurrentResidentBytesTestOnly() {
819 return Cache.getCurrentResidentBytesTestOnly();
820 }
821
822 void getStats(ScopedString *Str);
823
824private:
825 typename Config::template CacheT<typename Config::CacheConfig> Cache;
826
827 mutable HybridMutex Mutex;
828 DoublyLinkedList<LargeBlock::Header> InUseBlocks GUARDED_BY(Mutex);
829 uptr AllocatedBytes GUARDED_BY(Mutex) = 0;
830 uptr FreedBytes GUARDED_BY(Mutex) = 0;
831 uptr FragmentedBytes GUARDED_BY(Mutex) = 0;
832 uptr LargestSize GUARDED_BY(Mutex) = 0;
833 u32 UncacheableUnmaps GUARDED_BY(Mutex) = 0;
834 u32 NumberOfAllocs GUARDED_BY(Mutex) = 0;
835 u32 NumberOfFrees GUARDED_BY(Mutex) = 0;
836 LocalStats Stats GUARDED_BY(Mutex);
837};
838
839template <typename Config>
840void *
841MapAllocator<Config>::tryAllocateFromCache(const Options &Options, uptr Size,
842 uptr Alignment, uptr *BlockEndPtr,
843 FillContentsMode FillContents) {
844 CachedBlock Entry;
845 uptr EntryHeaderPos;
846 uptr MaxAllowedFragmentedPages = MaxUnreleasedCachePages;
847
848 if (LIKELY(!useMemoryTagging<Config>(Options))) {
849 MaxAllowedFragmentedPages += CachedBlock::MaxReleasedCachePages;
850 } else {
851 // TODO: Enable MaxReleasedCachePages may result in pages for an entry being
852 // partially released and it erases the tag of those pages as well. To
853 // support this feature for MTE, we need to tag those pages again.
854 DCHECK_EQ(MaxAllowedFragmentedPages, MaxUnreleasedCachePages);
855 }
856
857 Entry = Cache.retrieve(MaxAllowedFragmentedPages, Size, Alignment,
858 getHeadersSize(), EntryHeaderPos);
859 if (!Entry.isValid())
860 return nullptr;
861
862 LargeBlock::Header *H = reinterpret_cast<LargeBlock::Header *>(
863 LargeBlock::addHeaderTag<Config>(EntryHeaderPos));
864
865 if (UNLIKELY(Entry.Flags & CachedBlock::NoAccess)) {
866 // NOTE: Flags set to 0 actually restores read-write.
867 Entry.MemMap.setMemoryPermission(Addr: Entry.CommitBase, Size: Entry.CommitSize,
868 /*Flags=*/Flags: 0);
869 }
870
871 if (useMemoryTagging<Config>(Options)) {
872 const uptr PageSize = getPageSizeCached();
873 const uptr OldAllocPos =
874 untagPointer(Ptr: Entry.BlockBegin) + Chunk::getHeaderSize();
875 const uptr NewAllocPos = untagPointer(Ptr: roundUp(X: EntryHeaderPos, Boundary: Alignment));
876 DCHECK_GE(Alignment, PageSize);
877
878 const uptr OldHeaderPage = OldAllocPos - PageSize;
879 const uptr NewHeaderPage = NewAllocPos - PageSize;
880
881 // Enabling or disabling memory tagging at runtime is unsupported.
882 // If MTE is enabled now, the cached entry was also allocated with MTE
883 // enabled, guaranteeing that both OldAllocPos and NewAllocPos are
884 // page-aligned.
885 CHECK(OldAllocPos % PageSize == 0U);
886 DCHECK(NewAllocPos % PageSize == 0U);
887
888 if (NewAllocPos != OldAllocPos) {
889 // The shift distance must be a multiple of PageSize
890 DCHECK_EQ((NewAllocPos > OldAllocPos ? NewAllocPos - OldAllocPos
891 : OldAllocPos - NewAllocPos) %
892 PageSize,
893 0U);
894
895 const uptr MappingFlags = MAP_RESIZABLE | MAP_ALLOWNOMEM;
896
897 if (!Entry.MemMap.remap(Addr: NewHeaderPage, Size: PageSize, Name: "scudo:secondary",
898 MAP_MEMTAG | MappingFlags)) {
899 unmap(MemMap&: Entry.MemMap);
900 return nullptr;
901 }
902 // Since PROT_MTE is sticky, setting memory permissions to MAP_NOACCESS
903 // (PROT_NONE) when caching the block does not clear the PROT_MTE flag
904 // from the kernel VMA. When we make the block RW again, the old header
905 // page still has MTE enabled. If the allocation shifted, we must
906 // explicitly remap the old header page without MAP_MEMTAG to disable MTE,
907 // as this page may now be part of the user payload or padding.
908 if (!Entry.MemMap.remap(Addr: OldHeaderPage, Size: PageSize, Name: "scudo:secondary",
909 Flags: MappingFlags)) {
910 unmap(MemMap&: Entry.MemMap);
911 return nullptr;
912 }
913 }
914
915 uptr NewBlockBegin = reinterpret_cast<uptr>(H + 1);
916 storeTags(Begin: reinterpret_cast<uptr>(H), End: NewBlockBegin);
917 }
918
919 H->CommitBase = Entry.CommitBase;
920 H->CommitSize = Entry.CommitSize;
921 H->MemMap = Entry.MemMap;
922
923 const uptr BlockEnd = H->CommitBase + H->CommitSize;
924 if (BlockEndPtr)
925 *BlockEndPtr = BlockEnd;
926 uptr HInt = reinterpret_cast<uptr>(H);
927 if (allocatorSupportsMemoryTagging<Config>())
928 HInt = untagPointer(Ptr: HInt);
929 const uptr PtrInt = HInt + LargeBlock::getHeaderSize();
930 void *Ptr = reinterpret_cast<void *>(PtrInt);
931 if (FillContents != NoFill && !Entry.isZero()) {
932 memset(s: Ptr, c: FillContents == ZeroFill ? 0 : PatternFillByte,
933 n: BlockEnd - PtrInt);
934 }
935
936 {
937 ScopedLock L(Mutex);
938 InUseBlocks.push_back(X: H);
939 AllocatedBytes += H->CommitSize;
940 FragmentedBytes += H->MemMap.getCapacity() - H->CommitSize;
941 NumberOfAllocs++;
942 Stats.add(I: StatAllocated, V: H->CommitSize);
943 Stats.add(I: StatMapped, V: H->MemMap.getCapacity());
944 }
945 return Ptr;
946}
947// As with the Primary, the size passed to this function includes any desired
948// alignment, so that the frontend can align the user allocation. The hint
949// parameter allows us to unmap spurious memory when dealing with larger
950// (greater than a page) alignments on 32-bit platforms.
951// Due to the sparsity of address space available on those platforms, requesting
952// an allocation from the Secondary with a large alignment would end up wasting
953// VA space (even though we are not committing the whole thing), hence the need
954// to trim off some of the reserved space.
955// For allocations requested with an alignment greater than or equal to a page,
956// the committed memory will amount to something close to Size - AlignmentHint
957// (pending rounding and headers).
958template <typename Config>
959void *MapAllocator<Config>::allocate(const Options &Options, uptr Size,
960 uptr Alignment, uptr *BlockEndPtr,
961 FillContentsMode FillContents) {
962 if (Options.get(Opt: OptionBit::AddLargeAllocationSlack))
963 Size += 1UL << SCUDO_MIN_ALIGNMENT_LOG;
964 Alignment = Max(A: Alignment, B: uptr(1U) << SCUDO_MIN_ALIGNMENT_LOG);
965 const uptr PageSize = getPageSizeCached();
966
967 if (useMemoryTagging<Config>(Options))
968 Alignment = Max(A: Alignment, B: PageSize);
969
970 // Note that cached blocks may have aligned address already. Thus we simply
971 // pass the required size (`Size` + `getHeadersSize()`) to do cache look up.
972 const uptr MinNeededSizeForCache = roundUp(X: Size + getHeadersSize(), Boundary: PageSize);
973
974 if (Alignment <= PageSize && Cache.canCache(MinNeededSizeForCache)) {
975 void *Ptr = tryAllocateFromCache(Options, Size, Alignment, BlockEndPtr,
976 FillContents);
977 if (Ptr != nullptr)
978 return Ptr;
979 }
980
981 uptr RoundedSize =
982 roundUp(X: roundUp(X: Size, Boundary: Alignment) + getHeadersSize(), Boundary: PageSize);
983 if (UNLIKELY(Alignment > PageSize))
984 RoundedSize += Alignment - PageSize;
985
986 ReservedMemoryT ReservedMemory;
987 const uptr MapSize = RoundedSize + 2 * getGuardPageSize();
988 if (UNLIKELY(!ReservedMemory.create(/*Addr=*/0U, MapSize, nullptr,
989 MAP_ALLOWNOMEM))) {
990 return nullptr;
991 }
992
993 // Take the entire ownership of reserved region.
994 MemMapT MemMap = ReservedMemory.dispatch(Addr: ReservedMemory.getBase(),
995 Size: ReservedMemory.getCapacity());
996 uptr MapBase = MemMap.getBase();
997 uptr CommitBase = MapBase + getGuardPageSize();
998 uptr MapEnd = MapBase + MapSize;
999
1000 // In the unlikely event of alignments larger than a page, adjust the amount
1001 // of memory we want to commit, and trim the extra memory.
1002 if (UNLIKELY(Alignment >= PageSize)) {
1003 // For alignments greater than or equal to a page, the user pointer (eg:
1004 // the pointer that is returned by the C or C++ allocation APIs) ends up
1005 // on a page boundary , and our headers will live in the preceding page.
1006 CommitBase =
1007 roundUp(X: MapBase + getGuardPageSize() + 1, Boundary: Alignment) - PageSize;
1008 // We only trim the extra memory on 32-bit platforms: 64-bit platforms
1009 // are less constrained memory wise, and that saves us two syscalls.
1010 if (SCUDO_WORDSIZE == 32U) {
1011 const uptr NewMapBase = CommitBase - getGuardPageSize();
1012 DCHECK_GE(NewMapBase, MapBase);
1013 if (NewMapBase != MapBase) {
1014 MemMap.unmap(Addr: MapBase, Size: NewMapBase - MapBase);
1015 MapBase = NewMapBase;
1016 }
1017 // CommitBase is past the first guard page, but this computation needs
1018 // to include a page where the header lives.
1019 const uptr NewMapEnd =
1020 CommitBase + PageSize + roundUp(X: Size, Boundary: PageSize) + getGuardPageSize();
1021 DCHECK_LE(NewMapEnd, MapEnd);
1022 if (NewMapEnd != MapEnd) {
1023 MemMap.unmap(Addr: NewMapEnd, Size: MapEnd - NewMapEnd);
1024 MapEnd = NewMapEnd;
1025 }
1026 }
1027 }
1028
1029 const uptr CommitSize = MapEnd - getGuardPageSize() - CommitBase;
1030 const uptr AllocPos = roundDown(X: CommitBase + CommitSize - Size, Boundary: Alignment);
1031 if (!mapSecondary<Config>(Options, CommitBase, CommitSize, AllocPos, 0,
1032 MemMap)) {
1033 MemMap.unmap();
1034 return nullptr;
1035 }
1036 const uptr HeaderPos = AllocPos - getHeadersSize();
1037 // Make sure that the header is not in the guard page or before the base.
1038 DCHECK_GE(HeaderPos, MapBase + getGuardPageSize());
1039 LargeBlock::Header *H = reinterpret_cast<LargeBlock::Header *>(
1040 LargeBlock::addHeaderTag<Config>(HeaderPos));
1041 if (useMemoryTagging<Config>(Options))
1042 storeTags(Begin: reinterpret_cast<uptr>(H), End: reinterpret_cast<uptr>(H + 1));
1043 H->CommitBase = CommitBase;
1044 H->CommitSize = CommitSize;
1045 H->MemMap = MemMap;
1046 if (BlockEndPtr)
1047 *BlockEndPtr = CommitBase + CommitSize;
1048 {
1049 ScopedLock L(Mutex);
1050 InUseBlocks.push_back(X: H);
1051 AllocatedBytes += CommitSize;
1052 FragmentedBytes += H->MemMap.getCapacity() - CommitSize;
1053 if (LargestSize < CommitSize)
1054 LargestSize = CommitSize;
1055 NumberOfAllocs++;
1056 Stats.add(I: StatAllocated, V: CommitSize);
1057 Stats.add(I: StatMapped, V: H->MemMap.getCapacity());
1058 }
1059 return reinterpret_cast<void *>(HeaderPos + LargeBlock::getHeaderSize());
1060}
1061
1062template <typename Config>
1063void MapAllocator<Config>::deallocate(const Options &Options, void *Ptr,
1064 bool Zero) EXCLUDES(Mutex) {
1065 LargeBlock::Header *H = LargeBlock::getHeader<Config>(Ptr);
1066 const uptr CommitSize = H->CommitSize;
1067 {
1068 ScopedLock L(Mutex);
1069 InUseBlocks.remove(X: H);
1070 FreedBytes += CommitSize;
1071 FragmentedBytes -= H->MemMap.getCapacity() - CommitSize;
1072 NumberOfFrees++;
1073 Stats.sub(I: StatAllocated, V: CommitSize);
1074 Stats.sub(I: StatMapped, V: H->MemMap.getCapacity());
1075 }
1076
1077 if (Cache.canCache(H->CommitSize)) {
1078 Cache.store(Options, H->CommitBase, H->CommitSize,
1079 reinterpret_cast<uptr>(H + 1), H->MemMap, Zero);
1080 } else {
1081 // Note that the `H->MemMap` is stored on the pages managed by itself. Take
1082 // over the ownership before unmap() so that any operation along with
1083 // unmap() won't touch inaccessible pages.
1084 MemMapT MemMap = H->MemMap;
1085 unmap(MemMap);
1086 ScopedLock L(Mutex);
1087 UncacheableUnmaps++;
1088 }
1089}
1090
1091template <typename Config>
1092void MapAllocator<Config>::getStats(ScopedString *Str) EXCLUDES(Mutex) {
1093 ScopedLock L(Mutex);
1094 Str->append(
1095 Format: "Stats: MapAllocator: allocated %u times (%zuK), freed %u times (%zuK), "
1096 "remains %u (%zuK) max %zuM, Fragmented %zuK, Uncacheable unmaps: %u\n",
1097 NumberOfAllocs, AllocatedBytes >> 10, NumberOfFrees, FreedBytes >> 10,
1098 NumberOfAllocs - NumberOfFrees, (AllocatedBytes - FreedBytes) >> 10,
1099 LargestSize >> 20, FragmentedBytes >> 10, UncacheableUnmaps);
1100 Cache.getStats(Str);
1101}
1102
1103} // namespace scudo
1104
1105#endif // SCUDO_SECONDARY_H_
1106