1//===-- combined.h ----------------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#ifndef SCUDO_COMBINED_H_
10#define SCUDO_COMBINED_H_
11
12#include "allocator_config_wrapper.h"
13#include "atomic_helpers.h"
14#include "chunk.h"
15#include "common.h"
16#include "flags.h"
17#include "flags_parser.h"
18#include "mem_map.h"
19#include "memtag.h"
20#include "mutex.h"
21#include "options.h"
22#include "quarantine.h"
23#include "report.h"
24#include "secondary.h"
25#include "size_class_allocator.h"
26#include "stack_depot.h"
27#include "string_utils.h"
28#include "tracing.h"
29#include "tsd.h"
30
31#include "scudo/interface.h"
32
33#ifdef GWP_ASAN_HOOKS
34#include "gwp_asan/guarded_pool_allocator.h"
35#include "gwp_asan/optional/backtrace.h"
36#include "gwp_asan/optional/segv_handler.h"
37#endif // GWP_ASAN_HOOKS
38
39extern "C" inline void EmptyCallback() {}
40
41#ifdef HAVE_ANDROID_UNSAFE_FRAME_POINTER_CHASE
42// This function is not part of the NDK so it does not appear in any public
43// header files. We only declare/use it when targeting the platform.
44extern "C" size_t android_unsafe_frame_pointer_chase(scudo::uptr *buf,
45 size_t num_entries);
46#endif
47
48namespace scudo {
49
50template <class Config, void (*PostInitCallback)(void) = EmptyCallback>
51class Allocator {
52public:
53 using AllocatorConfig = BaseConfig<Config>;
54 using PrimaryT =
55 typename AllocatorConfig::template PrimaryT<PrimaryConfig<Config>>;
56 using SecondaryT =
57 typename AllocatorConfig::template SecondaryT<SecondaryConfig<Config>>;
58 using SizeClassAllocatorT = typename PrimaryT::SizeClassAllocatorT;
59 typedef Allocator<Config, PostInitCallback> ThisT;
60 typedef typename AllocatorConfig::template TSDRegistryT<ThisT> TSDRegistryT;
61
62 void callPostInitCallback() {
63 pthread_once(once_control: &PostInitNonce, init_routine: PostInitCallback);
64 }
65
66 struct QuarantineCallback {
67 explicit QuarantineCallback(ThisT &Instance,
68 SizeClassAllocatorT &SizeClassAllocator)
69 : Allocator(Instance), SizeClassAllocator(SizeClassAllocator) {}
70
71 // Chunk recycling function, returns a quarantined chunk to the backend,
72 // first making sure it hasn't been tampered with.
73 void recycle(void *Ptr) {
74 Chunk::UnpackedHeader Header;
75 Chunk::loadHeader(Cookie: Allocator.Cookie, Ptr, NewUnpackedHeader: &Header);
76 if (UNLIKELY(Header.State != Chunk::State::Quarantined))
77 reportInvalidChunkState(Action: AllocatorAction::Recycling, Ptr);
78
79 Header.State = Chunk::State::Available;
80 Chunk::storeHeader(Cookie: Allocator.Cookie, Ptr, NewUnpackedHeader: &Header);
81
82 if (allocatorSupportsMemoryTagging<AllocatorConfig>())
83 Ptr = untagPointer(Ptr);
84 void *BlockBegin = Allocator::getBlockBegin(Ptr, Header: &Header);
85 SizeClassAllocator.deallocate(Header.ClassId, BlockBegin);
86 }
87
88 // We take a shortcut when allocating a quarantine batch by working with the
89 // appropriate class ID instead of using Size. The compiler should optimize
90 // the class ID computation and work with the associated cache directly.
91 void *allocate(UNUSED uptr Size) {
92 const uptr QuarantineClassId = SizeClassMap::getClassIdBySize(
93 sizeof(QuarantineBatch) + Chunk::getHeaderSize());
94 void *Ptr = SizeClassAllocator.allocate(QuarantineClassId);
95 // Quarantine batch allocation failure is fatal.
96 if (UNLIKELY(!Ptr))
97 reportOutOfMemory(SizeClassMap::getSizeByClassId(QuarantineClassId));
98
99 Ptr = reinterpret_cast<void *>(reinterpret_cast<uptr>(Ptr) +
100 Chunk::getHeaderSize());
101 Chunk::UnpackedHeader Header = {};
102 Header.ClassId = QuarantineClassId & Chunk::ClassIdMask;
103 Header.SizeOrUnusedBytes = sizeof(QuarantineBatch);
104 Header.State = Chunk::State::Quarantined;
105 Chunk::storeHeader(Cookie: Allocator.Cookie, Ptr, NewUnpackedHeader: &Header);
106
107 // Reset tag to 0 as this chunk may have been previously used for a tagged
108 // user allocation.
109 if (UNLIKELY(useMemoryTagging<AllocatorConfig>(
110 Allocator.Primary.Options.load())))
111 storeTags(Begin: reinterpret_cast<uptr>(Ptr),
112 End: reinterpret_cast<uptr>(Ptr) + sizeof(QuarantineBatch));
113
114 return Ptr;
115 }
116
117 void deallocate(void *Ptr) {
118 const uptr QuarantineClassId = SizeClassMap::getClassIdBySize(
119 sizeof(QuarantineBatch) + Chunk::getHeaderSize());
120 Chunk::UnpackedHeader Header;
121 Chunk::loadHeader(Cookie: Allocator.Cookie, Ptr, NewUnpackedHeader: &Header);
122
123 if (UNLIKELY(Header.State != Chunk::State::Quarantined))
124 reportInvalidChunkState(Action: AllocatorAction::Deallocating, Ptr);
125 DCHECK_EQ(Header.ClassId, QuarantineClassId);
126 DCHECK_EQ(Header.Offset, 0);
127 DCHECK_EQ(Header.SizeOrUnusedBytes, sizeof(QuarantineBatch));
128
129 Header.State = Chunk::State::Available;
130 Chunk::storeHeader(Cookie: Allocator.Cookie, Ptr, NewUnpackedHeader: &Header);
131 SizeClassAllocator.deallocate(
132 QuarantineClassId,
133 reinterpret_cast<void *>(reinterpret_cast<uptr>(Ptr) -
134 Chunk::getHeaderSize()));
135 }
136
137 private:
138 ThisT &Allocator;
139 SizeClassAllocatorT &SizeClassAllocator;
140 };
141
142 typedef GlobalQuarantine<QuarantineCallback, void,
143 AllocatorConfig::getQuarantineDisabled()>
144 QuarantineT;
145 typedef typename QuarantineT::CacheT QuarantineCacheT;
146
147 void init() {
148 // Make sure that the page size is initialized if it's not a constant.
149 CHECK_NE(getPageSizeCached(), 0U);
150
151 performSanityChecks();
152
153 // Check if hardware CRC32 is supported in the binary and by the platform,
154 // if so, opt for the CRC32 hardware version of the checksum.
155 if (&computeHardwareCRC32 && hasHardwareCRC32())
156 HashAlgorithm = Checksum::HardwareCRC32;
157
158 if (UNLIKELY(!getRandom(&Cookie, sizeof(Cookie)))) {
159 ScopedString Str;
160 Str.append(Format: "Randomness degraded.\n");
161 Str.output();
162 Cookie = static_cast<u32>(getMonotonicTime() ^
163 (reinterpret_cast<uptr>(this) >> 4));
164 }
165
166 initFlags();
167 reportUnrecognizedFlags();
168
169 // Store some flags locally.
170 if (getFlags()->may_return_null)
171 Primary.Options.set(OptionBit::MayReturnNull);
172 if (getFlags()->zero_contents)
173 Primary.Options.setFillContentsMode(ZeroFill);
174 else if (getFlags()->pattern_fill_contents)
175 Primary.Options.setFillContentsMode(PatternOrZeroFill);
176 if (getFlags()->dealloc_type_mismatch)
177 Primary.Options.set(OptionBit::DeallocTypeMismatch);
178 if (getFlags()->dealloc_align_mismatch)
179 Primary.Options.set(OptionBit::DeallocAlignMismatch);
180 if (getFlags()->delete_size_mismatch)
181 Primary.Options.set(OptionBit::DeleteSizeMismatch);
182 if (systemSupportsMemoryTagging())
183 Primary.Options.set(OptionBit::UseMemoryTagging);
184
185 QuarantineMaxChunkSize =
186 static_cast<u32>(getFlags()->quarantine_max_chunk_size);
187#if SCUDO_FUCHSIA
188 ZeroOnDeallocMaxSize =
189 static_cast<u32>(getFlags()->zero_on_dealloc_max_size);
190#endif
191
192 Stats.init();
193 // TODO(chiahungduan): Given that we support setting the default value in
194 // the PrimaryConfig and CacheConfig, consider to deprecate the use of
195 // `release_to_os_interval_ms` flag.
196 const s32 ReleaseToOsIntervalMs = getFlags()->release_to_os_interval_ms;
197 Primary.init(ReleaseToOsIntervalMs);
198 Secondary.init(&Stats, ReleaseToOsIntervalMs);
199 if (!AllocatorConfig::getQuarantineDisabled()) {
200 Quarantine.init(
201 static_cast<uptr>(getFlags()->quarantine_size_kb << 10),
202 static_cast<uptr>(getFlags()->thread_local_quarantine_size_kb << 10));
203 }
204 }
205
206 void enableRingBuffer() NO_THREAD_SAFETY_ANALYSIS {
207 AllocationRingBuffer *RB = getRingBuffer();
208 if (RB)
209 RB->Depot->enable();
210 RingBufferInitLock.unlock();
211 }
212
213 void disableRingBuffer() NO_THREAD_SAFETY_ANALYSIS {
214 RingBufferInitLock.lock();
215 AllocationRingBuffer *RB = getRingBuffer();
216 if (RB)
217 RB->Depot->disable();
218 }
219
220 // Initialize the embedded GWP-ASan instance. Requires the main allocator to
221 // be functional, best called from PostInitCallback.
222 void initGwpAsan() {
223#ifdef GWP_ASAN_HOOKS
224 gwp_asan::options::Options Opt;
225 Opt.Enabled = getFlags()->GWP_ASAN_Enabled;
226 Opt.MaxSimultaneousAllocations =
227 getFlags()->GWP_ASAN_MaxSimultaneousAllocations;
228 Opt.SampleRate = getFlags()->GWP_ASAN_SampleRate;
229 Opt.InstallSignalHandlers = getFlags()->GWP_ASAN_InstallSignalHandlers;
230 Opt.Recoverable = getFlags()->GWP_ASAN_Recoverable;
231 // Embedded GWP-ASan is locked through the Scudo atfork handler (via
232 // Allocator::disable calling GWPASan.disable). Disable GWP-ASan's atfork
233 // handler.
234 Opt.InstallForkHandlers = false;
235 Opt.Backtrace = gwp_asan::backtrace::getBacktraceFunction();
236 GuardedAlloc.init(Opts: Opt);
237
238 if (Opt.InstallSignalHandlers)
239 gwp_asan::segv_handler::installSignalHandlers(
240 GPA: &GuardedAlloc, Printf,
241 PrintBacktrace: gwp_asan::backtrace::getPrintBacktraceFunction(),
242 SegvBacktrace: gwp_asan::backtrace::getSegvBacktraceFunction(),
243 Recoverable: Opt.Recoverable);
244
245 GuardedAllocSlotSize =
246 GuardedAlloc.getAllocatorState()->maximumAllocationSize();
247 Stats.add(I: StatFree, V: static_cast<uptr>(Opt.MaxSimultaneousAllocations) *
248 GuardedAllocSlotSize);
249#endif // GWP_ASAN_HOOKS
250 }
251
252#ifdef GWP_ASAN_HOOKS
253 const gwp_asan::AllocationMetadata *getGwpAsanAllocationMetadata() {
254 return GuardedAlloc.getMetadataRegion();
255 }
256
257 const gwp_asan::AllocatorState *getGwpAsanAllocatorState() {
258 return GuardedAlloc.getAllocatorState();
259 }
260#endif // GWP_ASAN_HOOKS
261
262 ALWAYS_INLINE void initThreadMaybe(bool MinimalInit = false) {
263 TSDRegistry.initThreadMaybe(this, MinimalInit);
264 }
265
266 void unmapTestOnly() {
267 unmapRingBuffer();
268 TSDRegistry.unmapTestOnly(this);
269 Primary.unmapTestOnly();
270 Secondary.unmapTestOnly();
271#ifdef GWP_ASAN_HOOKS
272 if (getFlags()->GWP_ASAN_InstallSignalHandlers)
273 gwp_asan::segv_handler::uninstallSignalHandlers();
274 GuardedAlloc.uninitTestOnly();
275#endif // GWP_ASAN_HOOKS
276 }
277
278 TSDRegistryT *getTSDRegistry() { return &TSDRegistry; }
279 QuarantineT *getQuarantine() { return &Quarantine; }
280
281 // The Cache must be provided zero-initialized.
282 void initAllocator(SizeClassAllocatorT *SizeClassAllocator) {
283 SizeClassAllocator->init(&Stats, &Primary);
284 }
285
286 // Release the resources used by a TSD, which involves:
287 // - draining the local quarantine cache to the global quarantine;
288 // - releasing the cached pointers back to the Primary;
289 // - unlinking the local stats from the global ones (destroying the cache does
290 // the last two items).
291 void commitBack(TSD<ThisT> *TSD) {
292 TSD->assertLocked(/*BypassCheck=*/true);
293 if (!AllocatorConfig::getQuarantineDisabled()) {
294 Quarantine.drain(&TSD->getQuarantineCache(),
295 QuarantineCallback(*this, TSD->getSizeClassAllocator()));
296 }
297 TSD->getSizeClassAllocator().destroy(&Stats);
298 }
299
300 void drainCache(TSD<ThisT> *TSD) {
301 TSD->assertLocked(/*BypassCheck=*/true);
302 if (!AllocatorConfig::getQuarantineDisabled()) {
303 Quarantine.drainAndRecycle(
304 &TSD->getQuarantineCache(),
305 QuarantineCallback(*this, TSD->getSizeClassAllocator()));
306 }
307 TSD->getSizeClassAllocator().drain();
308 }
309 void drainCaches() { TSDRegistry.drainCaches(this); }
310
311 ALWAYS_INLINE void *getHeaderTaggedPointer(void *Ptr) {
312 if (!allocatorSupportsMemoryTagging<AllocatorConfig>())
313 return Ptr;
314 auto UntaggedPtr = untagPointer(Ptr);
315 // If the untagged pointer differs from the tagged pointer, it means the
316 // original pointer was tagged and we have a primary allocation. So we want
317 // to return a pointer with tag 0 which is the header tag for primary
318 // allocations. It's the same as the untagged pointer. We use 0 for the
319 // primary allocation headers because it doesn't conflict with the tags used
320 // by the allocations themselves (1-15). This means that MTE will
321 // deterministically detect linear buffer overflows.
322 if (UntaggedPtr != Ptr)
323 return UntaggedPtr;
324 // Secondary, or pointer allocated while memory tagging is unsupported or
325 // disabled. The tag mismatch is okay in the latter case because tags will
326 // not be checked.
327 // If the untagged pointer is the same as the tagged pointer, it means the
328 // original pointer was untagged. This means one of the followings:
329 // 1. We have a secondary allocation so we apply a fixed tag of 2, the
330 // header tag for secondary allocation.
331 // 2. MTE is disabled at runtime. In this case it doesn't matter which tag
332 // we apply, so we arbitrarily apply a fixed tag of 2 in this case as well.
333 return addHeaderTag(Ptr);
334 }
335
336 ALWAYS_INLINE uptr addHeaderTag(uptr Ptr) {
337 if (!allocatorSupportsMemoryTagging<AllocatorConfig>())
338 return Ptr;
339 return addFixedTag(Ptr, Tag: 2);
340 }
341
342 ALWAYS_INLINE void *addHeaderTag(void *Ptr) {
343 return reinterpret_cast<void *>(addHeaderTag(reinterpret_cast<uptr>(Ptr)));
344 }
345
346 NOINLINE u32 collectStackTrace(UNUSED StackDepot *Depot) {
347#ifdef HAVE_ANDROID_UNSAFE_FRAME_POINTER_CHASE
348 // Discard collectStackTrace() frame and allocator function frame.
349 constexpr uptr DiscardFrames = 2;
350 uptr Stack[ScudoTraceSize + DiscardFrames];
351 uptr Size = android_unsafe_frame_pointer_chase(Stack, ScudoTraceSize +
352 DiscardFrames);
353 Size = Min<uptr>(Size, ScudoTraceSize + DiscardFrames);
354 return Depot->insert(Stack + Min<uptr>(DiscardFrames, Size), Stack + Size);
355#else
356 return 0;
357#endif
358 }
359
360 uptr computeOddEvenMaskForPointerMaybe(const Options &Options, uptr Ptr,
361 uptr ClassId) {
362 if (!Options.get(Opt: OptionBit::UseOddEvenTags))
363 return 0;
364
365 // If a chunk's tag is odd, we want the tags of the surrounding blocks to be
366 // even, and vice versa. Blocks are laid out Size bytes apart, and adding
367 // Size to Ptr will flip the least significant set bit of Size in Ptr, so
368 // that bit will have the pattern 010101... for consecutive blocks, which we
369 // can use to determine which tag mask to use.
370 return 0x5555U << ((Ptr >> SizeClassMap::getSizeLSBByClassId(ClassId)) & 1);
371 }
372
373 NOINLINE void *allocate(uptr Size, Chunk::Origin Origin,
374 uptr Alignment = MinAlignment,
375 bool ZeroContents = false) NO_THREAD_SAFETY_ANALYSIS {
376 initThreadMaybe();
377
378 const Options Options = Primary.Options.load();
379 if (UNLIKELY(Alignment > MaxAlignment)) {
380 if (Options.get(Opt: OptionBit::MayReturnNull))
381 return nullptr;
382 reportAlignmentTooBig(Alignment, MaxAlignment);
383 }
384 if (Alignment < MinAlignment)
385 Alignment = MinAlignment;
386
387#ifdef GWP_ASAN_HOOKS
388 if (UNLIKELY(GuardedAlloc.shouldSample())) {
389 if (void *Ptr = GuardedAlloc.allocate(Size, Alignment)) {
390 Stats.lock();
391 Stats.add(I: StatAllocated, V: GuardedAllocSlotSize);
392 Stats.sub(I: StatFree, V: GuardedAllocSlotSize);
393 Stats.unlock();
394 return Ptr;
395 }
396 }
397#endif // GWP_ASAN_HOOKS
398
399 // If the requested size happens to be 0 (more common than you might think),
400 // allocate MinAlignment bytes on top of the header. Then add the extra
401 // bytes required to fulfill the alignment requirements: we allocate enough
402 // to be sure that there will be an address in the block that will satisfy
403 // the alignment.
404 const uptr NeededSize =
405 roundUp(X: Size, Boundary: MinAlignment) +
406 ((Alignment > MinAlignment) ? Alignment : Chunk::getHeaderSize());
407
408 // Takes care of extravagantly large sizes as well as integer overflows.
409 static_assert(MaxAllowedMallocSize < UINTPTR_MAX - MaxAlignment, "");
410 if (UNLIKELY(Size >= MaxAllowedMallocSize)) {
411 if (Options.get(Opt: OptionBit::MayReturnNull))
412 return nullptr;
413 reportAllocationSizeTooBig(UserSize: Size, TotalSize: NeededSize, MaxSize: MaxAllowedMallocSize);
414 }
415 DCHECK_LE(Size, NeededSize);
416
417 void *Block = nullptr;
418 uptr ClassId = 0;
419 uptr SecondaryBlockEnd = 0;
420 if (LIKELY(PrimaryT::canAllocate(NeededSize))) {
421 ClassId = SizeClassMap::getClassIdBySize(NeededSize);
422 DCHECK_NE(ClassId, 0U);
423 typename TSDRegistryT::ScopedTSD TSD(TSDRegistry);
424 Block = TSD->getSizeClassAllocator().allocate(ClassId);
425 // If the allocation failed, retry in each successively larger class until
426 // it fits. If it fails to fit in the largest class, fallback to the
427 // Secondary.
428 if (UNLIKELY(!Block)) {
429 while (ClassId < SizeClassMap::LargestClassId && !Block)
430 Block = TSD->getSizeClassAllocator().allocate(++ClassId);
431 if (!Block)
432 ClassId = 0;
433 }
434 }
435
436 FillContentsMode FillContents;
437 if (ZeroContents)
438 FillContents = ZeroFill;
439 else if (TSDRegistry.getDisableMemInit())
440 FillContents = NoFill;
441 else {
442 FillContents = Options.getFillContentsMode();
443 if (FillContents == ZeroFill && AllocatorConfig::getDeallocZeroFill()) {
444 // The zeroing will happen in the deallocate.
445 FillContents = NoFill;
446 }
447 }
448
449 if (UNLIKELY(ClassId == 0)) {
450 Block = Secondary.allocate(Options, Size, Alignment, &SecondaryBlockEnd,
451 FillContents);
452 }
453
454 if (UNLIKELY(!Block)) {
455 if (Options.get(Opt: OptionBit::MayReturnNull))
456 return nullptr;
457 printStats();
458 reportOutOfMemory(RequestedSize: NeededSize);
459 }
460
461 const uptr UserPtr = roundUp(
462 X: reinterpret_cast<uptr>(Block) + Chunk::getHeaderSize(), Boundary: Alignment);
463 const uptr SizeOrUnusedBytes =
464 ClassId ? Size : SecondaryBlockEnd - (UserPtr + Size);
465
466 if (LIKELY(!useMemoryTagging<AllocatorConfig>(Options))) {
467 return initChunk(ClassId, Origin, Block, UserPtr, SizeOrUnusedBytes,
468 FillContents);
469 }
470 return initChunkWithMemoryTagging(ClassId, Origin, Block, UserPtr, Size,
471 SizeOrUnusedBytes, FillContents);
472 }
473
474 ALWAYS_INLINE void deallocate(void *Ptr, Chunk::Origin Origin) {
475 deallocate(Ptr, Origin, /*DeleteSize=*/0, /*DeleteAlignment=*/0);
476 }
477
478 ALWAYS_INLINE void deallocateSized(void *Ptr, Chunk::Origin Origin,
479 uptr DeleteSize) {
480 deallocate(Ptr, Origin | Chunk::Origin::Size, DeleteSize,
481 /*DeleteAlignment=*/0);
482 }
483
484 ALWAYS_INLINE void deallocateSizedAligned(void *Ptr, Chunk::Origin Origin,
485 uptr DeleteSize,
486 uptr DeleteAlignment) {
487 deallocate(Ptr, Origin | Chunk::Origin::Size | Chunk::Origin::Align,
488 DeleteSize, DeleteAlignment);
489 }
490
491 ALWAYS_INLINE void deallocateAligned(void *Ptr, Chunk::Origin Origin,
492 uptr DeleteAlignment) {
493 deallocate(Ptr, Origin | Chunk::Origin::Align,
494 /*DeleteSize=*/0, /*DeleteAlignment=*/DeleteAlignment);
495 }
496
497 ALWAYS_INLINE void checkSizeMatch(const void *Ptr,
498 Chunk::UnpackedHeader *Header, uptr Size,
499 uptr DeallocSize) {
500 if (AllocatorConfig::getExactUsableSize()) {
501 if (DeallocSize != Size)
502 reportDeleteSizeMismatch(Ptr, Size: DeallocSize, ExpectedSize: Size);
503 } else if (DeallocSize != Size && DeallocSize != getUsableSize(Ptr, Header))
504 reportDeleteSizeMismatch(Ptr, DeallocSize, Size,
505 getUsableSize(Ptr, Header));
506 }
507
508 ALWAYS_INLINE void checkTypeMatch(AllocatorAction Action, const void *Ptr,
509 u8 AllocOrigin, u8 DeallocOrigin) {
510 if (UNLIKELY(Chunk::originBaseType(AllocOrigin) !=
511 Chunk::originBaseType(DeallocOrigin)))
512 reportDeallocTypeMismatch(Action, Ptr, AllocOrigin, DeallocOrigin);
513
514 // There is no way to store that a new/new [] did an aligned allocate,
515 // so skip that part of the verification.
516 if (UNLIKELY(AllocOrigin == Chunk::Origin::New ||
517 AllocOrigin == Chunk::Origin::NewArray))
518 return;
519
520 if (Chunk::originAligned(Origin: AllocOrigin)) {
521 // Only disallow an aligned allocation and a non-aligned deallocation
522 // if this is a realloc.
523 if (Action == AllocatorAction::Reallocating &&
524 !Chunk::originAligned(Origin: DeallocOrigin))
525 reportDeallocTypeMismatch(Action, Ptr, AllocOrigin, DeallocOrigin);
526 } else if (Chunk::originAligned(Origin: DeallocOrigin)) {
527 // Origin not aligned, dealloc aligned.
528 reportDeallocTypeMismatch(Action, Ptr, AllocOrigin, DeallocOrigin);
529 }
530 }
531
532 NOINLINE void deallocate(void *Ptr, u8 DeallocOrigin, uptr DeleteSize,
533 uptr DeleteAlignment) {
534 if (UNLIKELY(!Ptr))
535 return;
536
537 // For a deallocation, we only ensure minimal initialization, meaning thread
538 // local data will be left uninitialized for now (when using ELF TLS). The
539 // fallback cache will be used instead. This is a workaround for a situation
540 // where the only heap operation performed in a thread would be a free past
541 // the TLS destructors, ending up in initialized thread specific data never
542 // being destroyed properly. Any other heap operation will do a full init.
543 initThreadMaybe(/*MinimalInit=*/MinimalInit: true);
544
545#ifdef GWP_ASAN_HOOKS
546 if (UNLIKELY(GuardedAlloc.pointerIsMine(Ptr))) {
547 GuardedAlloc.deallocate(Ptr);
548 Stats.lock();
549 Stats.add(I: StatFree, V: GuardedAllocSlotSize);
550 Stats.sub(I: StatAllocated, V: GuardedAllocSlotSize);
551 Stats.unlock();
552 return;
553 }
554#endif // GWP_ASAN_HOOKS
555
556 if (UNLIKELY(!isAligned(reinterpret_cast<uptr>(Ptr), MinAlignment)))
557 reportMisalignedPointer(Action: AllocatorAction::Deallocating, Ptr);
558
559 if (UNLIKELY(Chunk::originAligned(DeallocOrigin) &&
560 !isPowerOfTwo(DeleteAlignment)))
561 reportAlignmentNotPowerOfTwo(Alignment: DeleteAlignment);
562
563 void *TaggedPtr = Ptr;
564 Ptr = getHeaderTaggedPointer(Ptr);
565
566 Chunk::UnpackedHeader Header;
567 Chunk::loadHeader(Cookie, Ptr, NewUnpackedHeader: &Header);
568
569 if (UNLIKELY(Header.State != Chunk::State::Allocated))
570 reportInvalidChunkState(Action: AllocatorAction::Deallocating, Ptr);
571
572 const Options Options = Primary.Options.load();
573 const uptr Size = getSize(Ptr, Header: &Header);
574 if (AllocatorConfig::getAbortOnDeallocSizeMismatch() &&
575 Chunk::originSized(Origin: DeallocOrigin) &&
576 Options.get(Opt: OptionBit::DeleteSizeMismatch))
577 checkSizeMatch(Ptr, Header: &Header, Size, DeallocSize: DeleteSize);
578
579 if (AllocatorConfig::getAbortOnDeallocTypeMismatch() &&
580 Options.get(Opt: OptionBit::DeallocTypeMismatch))
581 checkTypeMatch(Action: AllocatorAction::Deallocating, Ptr, AllocOrigin: Header.getOrigin(),
582 DeallocOrigin);
583
584 if (UNLIKELY(AllocatorConfig::getAbortOnDeallocAlignmentMismatch() &&
585 Chunk::originAligned(DeallocOrigin) &&
586 Options.get(OptionBit::DeallocAlignMismatch) &&
587 !isAligned(reinterpret_cast<uptr>(Ptr), DeleteAlignment)))
588 reportDeleteAlignmentMismatch(Ptr, Alignment: DeleteAlignment);
589
590 quarantineOrDeallocateChunk(Options, TaggedPtr, Ptr, Header: &Header, Size);
591 }
592
593 void *reallocate(void *OldPtr, uptr NewSize, uptr Alignment = MinAlignment) {
594 initThreadMaybe();
595
596 const Options Options = Primary.Options.load();
597 if (UNLIKELY(NewSize >= MaxAllowedMallocSize)) {
598 if (Options.get(Opt: OptionBit::MayReturnNull))
599 return nullptr;
600 reportAllocationSizeTooBig(UserSize: NewSize, TotalSize: 0, MaxSize: MaxAllowedMallocSize);
601 }
602
603 // The following cases are handled by the C wrappers.
604 DCHECK_NE(OldPtr, nullptr);
605 DCHECK_NE(NewSize, 0);
606
607#ifdef GWP_ASAN_HOOKS
608 if (UNLIKELY(GuardedAlloc.pointerIsMine(OldPtr))) {
609 uptr OldSize = GuardedAlloc.getSize(Ptr: OldPtr);
610 void *NewPtr = allocate(Size: NewSize, Origin: Chunk::Origin::Malloc, Alignment);
611 if (NewPtr)
612 memcpy(dest: NewPtr, src: OldPtr, n: (NewSize < OldSize) ? NewSize : OldSize);
613 GuardedAlloc.deallocate(Ptr: OldPtr);
614 Stats.lock();
615 Stats.add(I: StatFree, V: GuardedAllocSlotSize);
616 Stats.sub(I: StatAllocated, V: GuardedAllocSlotSize);
617 Stats.unlock();
618 return NewPtr;
619 }
620#endif // GWP_ASAN_HOOKS
621
622 void *OldTaggedPtr = OldPtr;
623 OldPtr = getHeaderTaggedPointer(Ptr: OldPtr);
624
625 if (UNLIKELY(!isAligned(reinterpret_cast<uptr>(OldPtr), MinAlignment)))
626 reportMisalignedPointer(Action: AllocatorAction::Reallocating, Ptr: OldPtr);
627
628 Chunk::UnpackedHeader Header;
629 Chunk::loadHeader(Cookie, Ptr: OldPtr, NewUnpackedHeader: &Header);
630
631 if (UNLIKELY(Header.State != Chunk::State::Allocated))
632 reportInvalidChunkState(Action: AllocatorAction::Reallocating, Ptr: OldPtr);
633
634 // Pointer has to be allocated with a malloc-type function. Some
635 // applications think that it is OK to realloc a memalign'ed pointer, which
636 // will trigger this check. It really isn't.
637 if (AllocatorConfig::getAbortOnDeallocTypeMismatch() &&
638 Options.get(Opt: OptionBit::DeallocTypeMismatch))
639 checkTypeMatch(Action: AllocatorAction::Reallocating, Ptr: OldPtr, AllocOrigin: Header.getOrigin(),
640 DeallocOrigin: Chunk::Origin::Malloc);
641
642 void *BlockBegin = getBlockBegin(Ptr: OldTaggedPtr, Header: &Header);
643 uptr BlockEnd;
644 const uptr ClassId = Header.ClassId;
645 uptr OldSize;
646 uptr UsableSize;
647 if (LIKELY(ClassId)) {
648 BlockEnd = reinterpret_cast<uptr>(BlockBegin) +
649 SizeClassMap::getSizeByClassId(ClassId);
650 OldSize = Header.SizeOrUnusedBytes;
651 UsableSize = BlockEnd - reinterpret_cast<uptr>(OldTaggedPtr);
652 } else {
653 BlockEnd = SecondaryT::getBlockEnd(BlockBegin);
654 UsableSize = BlockEnd - reinterpret_cast<uptr>(OldTaggedPtr);
655 OldSize = UsableSize - Header.SizeOrUnusedBytes;
656 }
657
658 // If the new chunk fits in the previously allocated block, do nothing
659 // but update the header and return immediately.
660 if (NewSize <= UsableSize) {
661 Header.SizeOrUnusedBytes =
662 (ClassId
663 ? NewSize
664 : BlockEnd - (reinterpret_cast<uptr>(OldTaggedPtr) + NewSize)) &
665 Chunk::SizeOrUnusedBytesMask;
666 Chunk::storeHeader(Cookie, Ptr: OldPtr, NewUnpackedHeader: &Header);
667 if (UNLIKELY(useMemoryTagging<AllocatorConfig>(Options))) {
668 if (ClassId) {
669 resizeTaggedChunk</*ClearAllTags*/ true>(
670 reinterpret_cast<uptr>(OldTaggedPtr) + OldSize,
671 reinterpret_cast<uptr>(OldTaggedPtr) + NewSize, NewSize,
672 untagPointer(Ptr: BlockEnd));
673 storePrimaryAllocationStackMaybe(Options, Ptr: OldPtr);
674 } else {
675 storeSecondaryAllocationStackMaybe(Options, Ptr: OldPtr, Size: NewSize);
676 }
677 }
678 return OldTaggedPtr;
679 }
680
681 // Otherwise we allocate a new one, and deallocate the old one. Some
682 // allocators will allocate an even larger chunk (by a fixed factor) to
683 // allow for potential further in-place realloc. The gains of such a trick
684 // are currently unclear.
685 void *NewPtr = allocate(Size: NewSize, Origin: Chunk::Origin::Malloc, Alignment);
686 if (LIKELY(NewPtr)) {
687 bool ExactSize = AllocatorConfig::getExactUsableSize() ||
688 useMemoryTagging<AllocatorConfig>(Options);
689 memcpy(dest: NewPtr, src: OldTaggedPtr,
690 n: Min(A: NewSize, B: ExactSize ? OldSize : UsableSize));
691 quarantineOrDeallocateChunk(Options, TaggedPtr: OldTaggedPtr, Ptr: OldPtr, Header: &Header,
692 Size: OldSize);
693 }
694 return NewPtr;
695 }
696
697 // TODO(kostyak): disable() is currently best-effort. There are some small
698 // windows of time when an allocation could still succeed after
699 // this function finishes. We will revisit that later.
700 void disable() NO_THREAD_SAFETY_ANALYSIS {
701 initThreadMaybe();
702#ifdef GWP_ASAN_HOOKS
703 GuardedAlloc.disable();
704#endif
705 TSDRegistry.disable();
706 Stats.disable();
707 if (!AllocatorConfig::getQuarantineDisabled())
708 Quarantine.disable();
709 Primary.disable();
710 Secondary.disable();
711 disableRingBuffer();
712 }
713
714 void enable(bool IsChild) NO_THREAD_SAFETY_ANALYSIS {
715 initThreadMaybe();
716 enableRingBuffer();
717 Secondary.enable();
718 Primary.enable(IsChild);
719 if (!AllocatorConfig::getQuarantineDisabled())
720 Quarantine.enable();
721 Stats.enable();
722 TSDRegistry.enable();
723#ifdef GWP_ASAN_HOOKS
724 GuardedAlloc.enable();
725#endif
726 }
727
728 // The function returns the amount of bytes required to store the statistics,
729 // which might be larger than the amount of bytes provided. Note that the
730 // statistics buffer is not necessarily constant between calls to this
731 // function. This can be called with a null buffer or zero size for buffer
732 // sizing purposes.
733 uptr getStats(char *Buffer, uptr Size) {
734 ScopedString Str;
735 const uptr Length = getStats(&Str) + 1;
736 if (Length < Size)
737 Size = Length;
738 if (Buffer && Size) {
739 memcpy(dest: Buffer, src: Str.data(), n: Size);
740 Buffer[Size - 1] = '\0';
741 }
742 return Length;
743 }
744
745 void printStats() {
746 ScopedString Str;
747 getStats(&Str);
748 Str.output();
749 }
750
751 void printFragmentationInfo() {
752 ScopedString Str;
753 Primary.getFragmentationInfo(&Str);
754 // Secondary allocator dumps the fragmentation data in getStats().
755 Str.output();
756 }
757
758 void releaseToOS(ReleaseToOS ReleaseType) {
759 initThreadMaybe();
760 SCUDO_SCOPED_TRACE(GetReleaseToOSTraceName(ReleaseType));
761 if (ReleaseType == ReleaseToOS::ForceAll)
762 drainCaches();
763 Primary.releaseToOS(ReleaseType);
764 Secondary.releaseToOS(ReleaseType);
765 }
766
767 // Iterate over all chunks and call a callback for all busy chunks located
768 // within the provided memory range. Said callback must not use this allocator
769 // or a deadlock can ensue. This fits Android's malloc_iterate() needs.
770 void iterateOverChunks(uptr Base, uptr Size, iterate_callback Callback,
771 void *Arg) {
772 initThreadMaybe();
773 if (archSupportsMemoryTagging())
774 Base = untagPointer(Ptr: Base);
775 const uptr From = Base;
776 const uptr To = Base + Size;
777 const Options Options = Primary.Options.load();
778 bool MemoryTaggingEnabled = useMemoryTagging<AllocatorConfig>(Options);
779 auto Lambda = [this, From, To, MemoryTaggingEnabled, Callback,
780 Arg](uptr Block) {
781 if (Block < From || Block >= To)
782 return;
783 uptr Chunk;
784 Chunk::UnpackedHeader Header;
785 if (UNLIKELY(MemoryTaggingEnabled)) {
786 if (!getChunkFromBlock(Block: loadTag(Ptr: Block), Chunk: &Chunk, Header: &Header))
787 return;
788 } else if (!getChunkFromBlock(Block: addHeaderTag(Block), Chunk: &Chunk, Header: &Header)) {
789 return;
790 }
791
792 if (Header.State != Chunk::State::Allocated)
793 return;
794
795 uptr TaggedChunk = Chunk;
796 if (allocatorSupportsMemoryTagging<AllocatorConfig>())
797 TaggedChunk = untagPointer(Ptr: TaggedChunk);
798 uptr Size;
799 if (UNLIKELY(MemoryTaggingEnabled)) {
800 TaggedChunk = loadTag(Ptr: Chunk);
801 Size = getSize(Ptr: reinterpret_cast<void *>(Chunk), Header: &Header);
802 } else if (AllocatorConfig::getExactUsableSize()) {
803 Size = getSize(Ptr: reinterpret_cast<void *>(Chunk), Header: &Header);
804 } else {
805 Size = getUsableSize(reinterpret_cast<void *>(Chunk), &Header);
806 }
807 Callback(TaggedChunk, Size, Arg);
808 };
809 Primary.iterateOverBlocks(Lambda);
810 Secondary.iterateOverBlocks(Lambda);
811#ifdef GWP_ASAN_HOOKS
812 GuardedAlloc.iterate(Base: reinterpret_cast<void *>(Base), Size, Cb: Callback, Arg);
813#endif
814 }
815
816 bool canReturnNull() {
817 initThreadMaybe();
818 return Primary.Options.load().get(OptionBit::MayReturnNull);
819 }
820
821 bool setOption(Option O, sptr Value) {
822 initThreadMaybe();
823 if (O == Option::MemtagTuning) {
824 // Enabling odd/even tags involves a tradeoff between use-after-free
825 // detection and buffer overflow detection. Odd/even tags make it more
826 // likely for buffer overflows to be detected by increasing the size of
827 // the guaranteed "red zone" around the allocation, but on the other hand
828 // use-after-free is less likely to be detected because the tag space for
829 // any particular chunk is cut in half. Therefore we use this tuning
830 // setting to control whether odd/even tags are enabled.
831 if (Value == M_MEMTAG_TUNING_BUFFER_OVERFLOW)
832 Primary.Options.set(OptionBit::UseOddEvenTags);
833 else if (Value == M_MEMTAG_TUNING_UAF)
834 Primary.Options.clear(OptionBit::UseOddEvenTags);
835 return true;
836 } else {
837 // We leave it to the various sub-components to decide whether or not they
838 // want to handle the option, but we do not want to short-circuit
839 // execution if one of the setOption was to return false.
840 const bool PrimaryResult = Primary.setOption(O, Value);
841 const bool SecondaryResult = Secondary.setOption(O, Value);
842 const bool RegistryResult = TSDRegistry.setOption(O, Value);
843 return PrimaryResult && SecondaryResult && RegistryResult;
844 }
845 return false;
846 }
847
848 ALWAYS_INLINE uptr getUsableSize(const void *Ptr,
849 Chunk::UnpackedHeader *Header) {
850 void *BlockBegin = getBlockBegin(Ptr, Header);
851 if (LIKELY(Header->ClassId)) {
852 return SizeClassMap::getSizeByClassId(Header->ClassId) -
853 (reinterpret_cast<uptr>(Ptr) - reinterpret_cast<uptr>(BlockBegin));
854 }
855
856 uptr UntaggedPtr = reinterpret_cast<uptr>(Ptr);
857 if (allocatorSupportsMemoryTagging<AllocatorConfig>()) {
858 UntaggedPtr = untagPointer(Ptr: UntaggedPtr);
859 BlockBegin = untagPointer(Ptr: BlockBegin);
860 }
861 return SecondaryT::getBlockEnd(BlockBegin) - UntaggedPtr;
862 }
863
864 // Return the usable size for a given chunk. If MTE is enabled or if the
865 // ExactUsableSize config parameter is true, we report the exact size of
866 // the original allocation size. Otherwise, we will return the total
867 // actual usable size.
868 uptr getUsableSize(const void *Ptr) {
869 if (UNLIKELY(!Ptr))
870 return 0;
871
872 if (AllocatorConfig::getExactUsableSize() ||
873 UNLIKELY(useMemoryTagging<AllocatorConfig>(Primary.Options.load())))
874 return getAllocSize(Ptr);
875
876 initThreadMaybe();
877
878#ifdef GWP_ASAN_HOOKS
879 if (UNLIKELY(GuardedAlloc.pointerIsMine(Ptr)))
880 return GuardedAlloc.getSize(Ptr);
881#endif // GWP_ASAN_HOOKS
882
883 Ptr = getHeaderTaggedPointer(Ptr: const_cast<void *>(Ptr));
884 Chunk::UnpackedHeader Header;
885 Chunk::loadHeader(Cookie, Ptr, NewUnpackedHeader: &Header);
886
887 // Getting the alloc size of a chunk only makes sense if it's allocated.
888 if (UNLIKELY(Header.State != Chunk::State::Allocated))
889 reportInvalidChunkState(Action: AllocatorAction::Sizing, Ptr);
890
891 return getUsableSize(Ptr, &Header);
892 }
893
894 uptr getAllocSize(const void *Ptr) {
895 initThreadMaybe();
896
897#ifdef GWP_ASAN_HOOKS
898 if (UNLIKELY(GuardedAlloc.pointerIsMine(Ptr)))
899 return GuardedAlloc.getSize(Ptr);
900#endif // GWP_ASAN_HOOKS
901
902 Ptr = getHeaderTaggedPointer(Ptr: const_cast<void *>(Ptr));
903 Chunk::UnpackedHeader Header;
904 Chunk::loadHeader(Cookie, Ptr, NewUnpackedHeader: &Header);
905
906 // Getting the alloc size of a chunk only makes sense if it's allocated.
907 if (UNLIKELY(Header.State != Chunk::State::Allocated))
908 reportInvalidChunkState(Action: AllocatorAction::Sizing, Ptr);
909
910 return getSize(Ptr, Header: &Header);
911 }
912
913 void getStats(StatCounters S) {
914 initThreadMaybe();
915 Stats.get(S);
916 }
917
918 // Returns true if the pointer provided was allocated by the current
919 // allocator instance, which is compliant with tcmalloc's ownership concept.
920 // A corrupted chunk will not be reported as owned, which is WAI.
921 bool isOwnedTestOnly(const void *Ptr) {
922 initThreadMaybe();
923 // If the allocation is not owned, the tags could be wrong.
924 ScopedTestOnlyDisableMemoryTagChecks x(
925 useMemoryTagging<AllocatorConfig>(Primary.Options.load()));
926#ifdef GWP_ASAN_HOOKS
927 if (GuardedAlloc.pointerIsMine(Ptr))
928 return true;
929#endif // GWP_ASAN_HOOKS
930 if (!Ptr || !isAligned(X: reinterpret_cast<uptr>(Ptr), Alignment: MinAlignment))
931 return false;
932 Ptr = getHeaderTaggedPointer(Ptr: const_cast<void *>(Ptr));
933 Chunk::UnpackedHeader Header;
934 return Chunk::isValid(Cookie, Ptr, NewUnpackedHeader: &Header) &&
935 Header.State == Chunk::State::Allocated;
936 }
937
938 bool useMemoryTaggingTestOnly() const {
939 return useMemoryTagging<AllocatorConfig>(Primary.Options.load());
940 }
941
942 void disableMemoryTagging() {
943 // If we haven't been initialized yet, we need to initialize now in order to
944 // prevent a future call to initThreadMaybe() from enabling memory tagging
945 // based on feature detection. But don't call initThreadMaybe() because it
946 // may end up calling the allocator (via pthread_atfork, via the post-init
947 // callback), which may cause mappings to be created with memory tagging
948 // enabled.
949 TSDRegistry.initOnceMaybe(this);
950 if (allocatorSupportsMemoryTagging<AllocatorConfig>()) {
951 Secondary.disableMemoryTagging();
952 Primary.Options.clear(OptionBit::UseMemoryTagging);
953 }
954 }
955
956 void setTrackAllocationStacks(bool Track) {
957 initThreadMaybe();
958 if (getFlags()->allocation_ring_buffer_size <= 0) {
959 DCHECK(!Primary.Options.load().get(OptionBit::TrackAllocationStacks));
960 return;
961 }
962
963 if (Track) {
964 initRingBufferMaybe();
965 Primary.Options.set(OptionBit::TrackAllocationStacks);
966 } else
967 Primary.Options.clear(OptionBit::TrackAllocationStacks);
968 }
969
970 void setFillContents(FillContentsMode FillContents) {
971 initThreadMaybe();
972 Primary.Options.setFillContentsMode(FillContents);
973 }
974
975 void setAddLargeAllocationSlack(bool AddSlack) {
976 initThreadMaybe();
977 if (AddSlack)
978 Primary.Options.set(OptionBit::AddLargeAllocationSlack);
979 else
980 Primary.Options.clear(OptionBit::AddLargeAllocationSlack);
981 }
982
983 void getErrorInfo(uptr FaultAddr, size_t MinDistance, size_t MaxDistance,
984 scudo_error_report *Reports, size_t &ReportIndex) {
985 auto *RingBuffer = getRingBuffer();
986 if (RingBuffer == nullptr)
987 return;
988
989 // No more room for any more error reports.
990 if (ReportIndex == ScudoNumErrorReports)
991 return;
992
993 uptr UntaggedFaultAddr = untagPointer(Ptr: FaultAddr);
994 BlockInfo Info = Primary.findNearestBlock(UntaggedFaultAddr);
995
996 auto GetBlockInfo = [&](uptr Addr, Chunk::UnpackedHeader *Header,
997 uptr &ChunkAddr) {
998 uptr ChunkOffset = getChunkOffsetFromBlock(
999 Block: reinterpret_cast<const char *>(loadTagUnaligned(Ptr: Addr)));
1000 ChunkAddr = loadTagUnaligned(Ptr: Addr + ChunkOffset);
1001 uptr HeaderAddr = loadTag(Ptr: Addr + ChunkOffset - Chunk::getHeaderSize());
1002 *Header = *reinterpret_cast<const Chunk::UnpackedHeader *>(HeaderAddr);
1003 };
1004
1005 auto CheckOOB = [&](uptr BlockAddr) {
1006 if (BlockAddr < Info.RegionBegin || BlockAddr >= Info.RegionEnd)
1007 return false;
1008
1009 uptr ChunkAddr;
1010 Chunk::UnpackedHeader Header;
1011 GetBlockInfo(BlockAddr, &Header, ChunkAddr);
1012 if (Header.State != Chunk::State::Allocated)
1013 return false;
1014 const u8 ChunkTag = extractTag(Ptr: ChunkAddr);
1015 if (extractTag(Ptr: FaultAddr) != ChunkTag) {
1016 // If the allocated size is zero, then there isn't a tag on this
1017 // pointer, so allow a tag mismatch.
1018 if (ChunkTag != 0 || Header.SizeOrUnusedBytes != 0)
1019 return false;
1020 }
1021 auto *Report = &Reports[ReportIndex++];
1022 uptr UntaggedChunkAddr = untagPointer(Ptr: ChunkAddr);
1023 const u32 *ChunkData = reinterpret_cast<const u32 *>(UntaggedChunkAddr);
1024 Report->error_type = UntaggedFaultAddr < UntaggedChunkAddr
1025 ? BUFFER_UNDERFLOW
1026 : BUFFER_OVERFLOW;
1027 Report->allocation_address = UntaggedChunkAddr;
1028 Report->allocation_size = Header.SizeOrUnusedBytes;
1029 if (RingBuffer->Depot) {
1030 const u32 *TracePtr = reinterpret_cast<const u32 *>(loadTagUnaligned(
1031 Ptr: reinterpret_cast<uptr>(&ChunkData[MemTagAllocationTraceIndex])));
1032 collectTraceMaybe(Depot: RingBuffer->Depot, Trace&: Report->allocation_trace,
1033 Hash: *TracePtr);
1034 }
1035 const u32 *TidPtr = reinterpret_cast<const u32 *>(loadTagUnaligned(
1036 Ptr: reinterpret_cast<uptr>(&ChunkData[MemTagAllocationTidIndex])));
1037 Report->allocation_tid = *TidPtr;
1038 return ReportIndex == ScudoNumErrorReports;
1039 };
1040
1041 if (MinDistance == 0 && CheckOOB(Info.BlockBegin))
1042 return;
1043
1044 for (size_t I = Max<size_t>(A: MinDistance, B: 1); I != MaxDistance; ++I)
1045 if (CheckOOB(Info.BlockBegin + I * Info.BlockSize) ||
1046 CheckOOB(Info.BlockBegin - I * Info.BlockSize))
1047 return;
1048 }
1049
1050 void getRingBufferErrorInfo(uintptr_t FaultAddr, scudo_error_report *Reports,
1051 size_t &ReportIndex) {
1052 auto *RingBuffer = getRingBuffer();
1053 if (RingBuffer == nullptr)
1054 return;
1055
1056 // No more room for any more error reports.
1057 if (ReportIndex == ScudoNumErrorReports)
1058 return;
1059
1060 uptr Pos = atomic_load_relaxed(&RingBuffer->Pos);
1061 if (Pos == 0)
1062 return;
1063
1064 const uptr RingBufferElements = RingBuffer->RingBufferElements;
1065
1066 // Pos is a value that is always increasing.
1067 uptr LastPos;
1068 if (Pos < RingBufferElements) {
1069 LastPos = 0;
1070 } else {
1071 LastPos = Pos - RingBufferElements;
1072 }
1073 for (uptr I = Pos; I > LastPos; --I) {
1074 auto *Entry =
1075 getRingBufferEntry(RingBuffer, (I - 1) % RingBufferElements);
1076 uptr EntryPtr = atomic_load_relaxed(&Entry->Ptr);
1077 if (!EntryPtr)
1078 continue;
1079
1080 uptr UntaggedEntryPtr = untagPointer(Ptr: EntryPtr);
1081 uptr EntrySize = atomic_load_relaxed(&Entry->AllocationSize);
1082 u32 AllocationTrace = atomic_load_relaxed(&Entry->AllocationTrace);
1083 u32 AllocationTid = atomic_load_relaxed(&Entry->AllocationTid);
1084 u32 DeallocationTrace = atomic_load_relaxed(&Entry->DeallocationTrace);
1085 u32 DeallocationTid = atomic_load_relaxed(&Entry->DeallocationTid);
1086 if (DeallocationTid) {
1087 // For UAF we only consider in-bounds fault addresses because
1088 // out-of-bounds UAF is rare and attempting to detect it is very likely
1089 // to result in false positives.
1090 if (FaultAddr < EntryPtr || FaultAddr >= EntryPtr + EntrySize)
1091 continue;
1092 } else {
1093 // Ring buffer OOB is only possible with secondary allocations. In this
1094 // case we are guaranteed a guard region of at least a page on either
1095 // side of the allocation (guard page on the right, guard page + tagged
1096 // region on the left), so ignore any faults outside of that range.
1097 if (FaultAddr < EntryPtr - getPageSizeCached() ||
1098 FaultAddr >= EntryPtr + EntrySize + getPageSizeCached())
1099 continue;
1100
1101 // For UAF the ring buffer will contain two entries, one for the
1102 // allocation and another for the deallocation. Don't report buffer
1103 // overflow/underflow using the allocation entry if we have already
1104 // collected a report from the deallocation entry.
1105 bool Found = false;
1106 for (uptr J = 0; J != ReportIndex; ++J) {
1107 if (Reports[J].allocation_address == UntaggedEntryPtr) {
1108 Found = true;
1109 break;
1110 }
1111 }
1112 if (Found)
1113 continue;
1114 }
1115
1116 auto *Report = &Reports[ReportIndex++];
1117 if (DeallocationTid)
1118 Report->error_type = USE_AFTER_FREE;
1119 else if (FaultAddr < EntryPtr)
1120 Report->error_type = BUFFER_UNDERFLOW;
1121 else
1122 Report->error_type = BUFFER_OVERFLOW;
1123
1124 Report->allocation_address = UntaggedEntryPtr;
1125 Report->allocation_size = EntrySize;
1126 collectTraceMaybe(Depot: RingBuffer->Depot, Trace&: Report->allocation_trace,
1127 Hash: AllocationTrace);
1128 Report->allocation_tid = AllocationTid;
1129 collectTraceMaybe(Depot: RingBuffer->Depot, Trace&: Report->deallocation_trace,
1130 Hash: DeallocationTrace);
1131 Report->deallocation_tid = DeallocationTid;
1132 if (ReportIndex == ScudoNumErrorReports) {
1133 // No more report entries.
1134 return;
1135 }
1136 }
1137 }
1138
1139 void getErrorInfo(uintptr_t FaultAddr, struct scudo_error_info *ErrorInfo) {
1140 const Options Options = Primary.Options.load();
1141 if (!useMemoryTagging<AllocatorConfig>(Options))
1142 return;
1143
1144 bool TagExists = extractTag(Ptr: FaultAddr) != 0;
1145 size_t ReportIndex = 0;
1146 if (TagExists) {
1147 // Check for OOB in the current block and the two surrounding blocks.
1148 // Beyond that, UAF is more likely.
1149 getErrorInfo(FaultAddr, 0, 2, &ErrorInfo->reports[0], ReportIndex);
1150 }
1151
1152 // Check the ring buffer. For primary allocations this will only find UAF;
1153 // for secondary allocations we can find either UAF or OOB.
1154 getRingBufferErrorInfo(FaultAddr, Reports: &ErrorInfo->reports[0], ReportIndex);
1155
1156 if (TagExists) {
1157 // Check for OOB in the 28 blocks surrounding the 3 we checked earlier.
1158 // Beyond that we are likely to hit false positives.
1159 getErrorInfo(FaultAddr, 2, 16, &ErrorInfo->reports[0], ReportIndex);
1160 }
1161 }
1162
1163 static void collectTraceMaybe(const StackDepot *Depot,
1164 uintptr_t (&Trace)[ScudoTraceSize], u32 Hash) {
1165 uptr RingPos, Size;
1166 if (!Depot->find(Hash, RingPosPtr: &RingPos, SizePtr: &Size))
1167 return;
1168 for (unsigned I = 0; I != Size && I != ScudoTraceSize; ++I)
1169 Trace[I] = static_cast<uintptr_t>(Depot->at(RingPos: RingPos + I));
1170 }
1171
1172 uptr getBlockBeginTestOnly(const void *Ptr) {
1173 Chunk::UnpackedHeader Header;
1174 Chunk::loadHeader(Cookie, Ptr, NewUnpackedHeader: &Header);
1175 DCHECK(Header.State == Chunk::State::Allocated);
1176
1177 if (allocatorSupportsMemoryTagging<AllocatorConfig>())
1178 Ptr = untagPointer(Ptr: const_cast<void *>(Ptr));
1179 void *Begin = getBlockBegin(Ptr, Header: &Header);
1180 if (allocatorSupportsMemoryTagging<AllocatorConfig>())
1181 Begin = untagPointer(Ptr: Begin);
1182 return reinterpret_cast<uptr>(Begin);
1183 }
1184
1185 bool getRingBufferEnabledTestOnly() {
1186 initThreadMaybe();
1187 return getRingBuffer() != nullptr;
1188 }
1189
1190 const void *getRingBufferAddressTestOnly() {
1191 initThreadMaybe();
1192 return getRingBuffer();
1193 }
1194
1195 bool getStackDepotEnabledTestOnly() {
1196 initThreadMaybe();
1197 auto *RB = getRingBuffer();
1198 if (RB == nullptr)
1199 return false;
1200 return RB->Depot != nullptr;
1201 }
1202
1203 const void *getStackDepotAddressTestOnly() {
1204 initThreadMaybe();
1205 auto *RB = getRingBuffer();
1206 if (RB == nullptr)
1207 return nullptr;
1208 return RB->Depot;
1209 }
1210
1211private:
1212 typedef typename PrimaryT::SizeClassMap SizeClassMap;
1213
1214 static const uptr MinAlignmentLog = SCUDO_MIN_ALIGNMENT_LOG;
1215 static const uptr MaxAlignmentLog = 24U; // 16 MB seems reasonable.
1216 static const uptr MinAlignment = 1UL << MinAlignmentLog;
1217 static const uptr MaxAlignment = 1UL << MaxAlignmentLog;
1218 static const uptr MaxAllowedMallocSize =
1219 FIRST_32_SECOND_64(1UL << 31, 1ULL << 40);
1220
1221 static_assert(MinAlignment >= sizeof(Chunk::PackedHeader),
1222 "Minimal alignment must at least cover a chunk header.");
1223 static_assert(!allocatorSupportsMemoryTagging<AllocatorConfig>() ||
1224 MinAlignment >= archMemoryTagGranuleSize(),
1225 "");
1226
1227 static const u32 BlockMarker = 0x44554353U;
1228
1229 // These are indexes into an "array" of 32-bit values that store information
1230 // inline with a chunk that is relevant to diagnosing memory tag faults, where
1231 // 0 corresponds to the address of the user memory. This means that only
1232 // negative indexes may be used. The smallest index that may be used is -2,
1233 // which corresponds to 8 bytes before the user memory, because the chunk
1234 // header size is 8 bytes and in allocators that support memory tagging the
1235 // minimum alignment is at least the tag granule size (16 on aarch64).
1236 static const sptr MemTagAllocationTraceIndex = -2;
1237 static const sptr MemTagAllocationTidIndex = -1;
1238
1239 u32 Cookie = 0;
1240 u32 QuarantineMaxChunkSize = 0;
1241#if SCUDO_FUCHSIA
1242 u32 ZeroOnDeallocMaxSize = 0;
1243#endif
1244
1245 GlobalStats Stats;
1246 PrimaryT Primary;
1247 SecondaryT Secondary;
1248 [[no_unique_address]] QuarantineT Quarantine;
1249 TSDRegistryT TSDRegistry;
1250 pthread_once_t PostInitNonce = PTHREAD_ONCE_INIT;
1251
1252#ifdef GWP_ASAN_HOOKS
1253 gwp_asan::GuardedPoolAllocator GuardedAlloc;
1254 uptr GuardedAllocSlotSize = 0;
1255#endif // GWP_ASAN_HOOKS
1256
1257 struct AllocationRingBuffer {
1258 struct Entry {
1259 atomic_uptr Ptr;
1260 atomic_uptr AllocationSize;
1261 atomic_u32 AllocationTrace;
1262 atomic_u32 AllocationTid;
1263 atomic_u32 DeallocationTrace;
1264 atomic_u32 DeallocationTid;
1265 };
1266 StackDepot *Depot = nullptr;
1267 uptr StackDepotSize = 0;
1268 MemMapT RawRingBufferMap;
1269 MemMapT RawStackDepotMap;
1270 u32 RingBufferElements = 0;
1271 atomic_uptr Pos;
1272 // An array of Size (at least one) elements of type Entry is immediately
1273 // following to this struct.
1274 };
1275 static_assert(sizeof(AllocationRingBuffer) %
1276 alignof(typename AllocationRingBuffer::Entry) ==
1277 0,
1278 "invalid alignment");
1279
1280 // Lock to initialize the RingBuffer
1281 HybridMutex RingBufferInitLock;
1282
1283 // Pointer to memory mapped area starting with AllocationRingBuffer struct,
1284 // and immediately followed by Size elements of type Entry.
1285 atomic_uptr RingBufferAddress = {};
1286
1287 AllocationRingBuffer *getRingBuffer() {
1288 return reinterpret_cast<AllocationRingBuffer *>(
1289 atomic_load(A: &RingBufferAddress, MO: memory_order_acquire));
1290 }
1291
1292 // The following might get optimized out by the compiler.
1293 NOINLINE void performSanityChecks() {
1294 // Verify that the header offset field can hold the maximum offset. In the
1295 // case of the Secondary allocator, it takes care of alignment and the
1296 // offset will always be small. In the case of the Primary, the worst case
1297 // scenario happens in the last size class, when the backend allocation
1298 // would already be aligned on the requested alignment, which would happen
1299 // to be the maximum alignment that would fit in that size class. As a
1300 // result, the maximum offset will be at most the maximum alignment for the
1301 // last size class minus the header size, in multiples of MinAlignment.
1302 Chunk::UnpackedHeader Header = {};
1303 const uptr MaxPrimaryAlignment = 1UL << getMostSignificantSetBitIndex(
1304 SizeClassMap::MaxSize - MinAlignment);
1305 const uptr MaxOffset =
1306 (MaxPrimaryAlignment - Chunk::getHeaderSize()) >> MinAlignmentLog;
1307 Header.Offset = MaxOffset & Chunk::OffsetMask;
1308 if (UNLIKELY(Header.Offset != MaxOffset))
1309 reportSanityCheckError(Field: "offset");
1310
1311 // Verify that we can fit the maximum size or amount of unused bytes in the
1312 // header. Given that the Secondary fits the allocation to a page, the worst
1313 // case scenario happens in the Primary. It will depend on the second to
1314 // last and last class sizes, as well as the dynamic base for the Primary.
1315 // The following is an over-approximation that works for our needs.
1316 const uptr MaxSizeOrUnusedBytes = SizeClassMap::MaxSize - 1;
1317 Header.SizeOrUnusedBytes = MaxSizeOrUnusedBytes;
1318 if (UNLIKELY(Header.SizeOrUnusedBytes != MaxSizeOrUnusedBytes))
1319 reportSanityCheckError(Field: "size (or unused bytes)");
1320
1321 const uptr LargestClassId = SizeClassMap::LargestClassId;
1322 Header.ClassId = LargestClassId;
1323 if (UNLIKELY(Header.ClassId != LargestClassId))
1324 reportSanityCheckError(Field: "class ID");
1325 }
1326
1327 static inline void *getBlockBegin(const void *Ptr,
1328 Chunk::UnpackedHeader *Header) {
1329 return reinterpret_cast<void *>(
1330 reinterpret_cast<uptr>(Ptr) - Chunk::getHeaderSize() -
1331 (static_cast<uptr>(Header->Offset) << MinAlignmentLog));
1332 }
1333
1334 // Return the size of a chunk as requested during its allocation.
1335 inline uptr getSize(const void *Ptr, Chunk::UnpackedHeader *Header) {
1336 const uptr SizeOrUnusedBytes = Header->SizeOrUnusedBytes;
1337 if (LIKELY(Header->ClassId))
1338 return SizeOrUnusedBytes;
1339 if (allocatorSupportsMemoryTagging<AllocatorConfig>())
1340 Ptr = untagPointer(Ptr: const_cast<void *>(Ptr));
1341 return SecondaryT::getBlockEnd(getBlockBegin(Ptr, Header)) -
1342 reinterpret_cast<uptr>(Ptr) - SizeOrUnusedBytes;
1343 }
1344
1345 ALWAYS_INLINE void *initChunk(const uptr ClassId, const Chunk::Origin Origin,
1346 void *Block, const uptr UserPtr,
1347 const uptr SizeOrUnusedBytes,
1348 const FillContentsMode FillContents) {
1349 // Compute the default pointer before adding the header tag
1350 const uptr DefaultAlignedPtr =
1351 reinterpret_cast<uptr>(Block) + Chunk::getHeaderSize();
1352
1353 Block = addHeaderTag(Block);
1354 // Only do content fill when it's from the primary allocator because the
1355 // secondary allocator handles filling memory on its own.
1356 if (ClassId != 0 && UNLIKELY(FillContents != NoFill)) {
1357 memset(Block, FillContents == ZeroFill ? 0 : PatternFillByte,
1358 PrimaryT::getSizeByClassId(ClassId));
1359 }
1360
1361 Chunk::UnpackedHeader Header = {};
1362
1363 if (UNLIKELY(DefaultAlignedPtr != UserPtr)) {
1364 const uptr Offset = UserPtr - DefaultAlignedPtr;
1365 DCHECK_GE(Offset, 2 * sizeof(u32));
1366 // The BlockMarker has no security purpose, but is specifically meant for
1367 // the chunk iteration function that can be used in debugging situations.
1368 // It is the only situation where we have to locate the start of a chunk
1369 // based on its block address.
1370 reinterpret_cast<u32 *>(Block)[0] = BlockMarker;
1371 reinterpret_cast<u32 *>(Block)[1] = static_cast<u32>(Offset);
1372 Header.Offset = (Offset >> MinAlignmentLog) & Chunk::OffsetMask;
1373 }
1374
1375 Header.ClassId = ClassId & Chunk::ClassIdMask;
1376 Header.State = Chunk::State::Allocated;
1377 Header.setOrigin(Origin);
1378 Header.SizeOrUnusedBytes = SizeOrUnusedBytes & Chunk::SizeOrUnusedBytesMask;
1379 Chunk::storeHeader(Cookie, Ptr: reinterpret_cast<void *>(addHeaderTag(UserPtr)),
1380 NewUnpackedHeader: &Header);
1381
1382 return reinterpret_cast<void *>(UserPtr);
1383 }
1384
1385 NOINLINE void *
1386 initChunkWithMemoryTagging(const uptr ClassId, const Chunk::Origin Origin,
1387 void *Block, const uptr UserPtr, const uptr Size,
1388 const uptr SizeOrUnusedBytes,
1389 const FillContentsMode FillContents) {
1390 const Options Options = Primary.Options.load();
1391 DCHECK(useMemoryTagging<AllocatorConfig>(Options));
1392
1393 // Compute the default pointer before adding the header tag
1394 const uptr DefaultAlignedPtr =
1395 reinterpret_cast<uptr>(Block) + Chunk::getHeaderSize();
1396
1397 void *Ptr = reinterpret_cast<void *>(UserPtr);
1398 void *TaggedPtr = Ptr;
1399
1400 if (LIKELY(ClassId)) {
1401 // Init the primary chunk.
1402 //
1403 // We only need to zero or tag the contents for Primary backed
1404 // allocations. We only set tags for primary allocations in order to avoid
1405 // faulting potentially large numbers of pages for large secondary
1406 // allocations. We assume that guard pages are enough to protect these
1407 // allocations.
1408 //
1409 // FIXME: When the kernel provides a way to set the background tag of a
1410 // mapping, we should be able to tag secondary allocations as well.
1411 //
1412 // When memory tagging is enabled, zeroing the contents is done as part of
1413 // setting the tag.
1414
1415 Chunk::UnpackedHeader Header;
1416 const uptr BlockSize = PrimaryT::getSizeByClassId(ClassId);
1417 const uptr BlockUptr = reinterpret_cast<uptr>(Block);
1418 const uptr BlockEnd = BlockUptr + BlockSize;
1419 // If possible, try to reuse the UAF tag that was set by deallocate().
1420 // For simplicity, only reuse tags if we have the same start address as
1421 // the previous allocation. This handles the majority of cases since
1422 // most allocations will not be more aligned than the minimum alignment.
1423 //
1424 // We need to handle situations involving reclaimed chunks, and retag
1425 // the reclaimed portions if necessary. In the case where the chunk is
1426 // fully reclaimed, the chunk's header will be zero, which will trigger
1427 // the code path for new mappings and invalid chunks that prepares the
1428 // chunk from scratch. There are three possibilities for partial
1429 // reclaiming:
1430 //
1431 // (1) Header was reclaimed, data was partially reclaimed.
1432 // (2) Header was not reclaimed, all data was reclaimed (e.g. because
1433 // data started on a page boundary).
1434 // (3) Header was not reclaimed, data was partially reclaimed.
1435 //
1436 // Case (1) will be handled in the same way as for full reclaiming,
1437 // since the header will be zero.
1438 //
1439 // We can detect case (2) by loading the tag from the start
1440 // of the chunk. If it is zero, it means that either all data was
1441 // reclaimed (since we never use zero as the chunk tag), or that the
1442 // previous allocation was of size zero. Either way, we need to prepare
1443 // a new chunk from scratch.
1444 //
1445 // We can detect case (3) by moving to the next page (if covered by the
1446 // chunk) and loading the tag of its first granule. If it is zero, it
1447 // means that all following pages may need to be retagged. On the other
1448 // hand, if it is nonzero, we can assume that all following pages are
1449 // still tagged, according to the logic that if any of the pages
1450 // following the next page were reclaimed, the next page would have been
1451 // reclaimed as well.
1452 uptr TaggedUserPtr;
1453 uptr PrevUserPtr;
1454 if (getChunkFromBlock(Block: BlockUptr, Chunk: &PrevUserPtr, Header: &Header) &&
1455 PrevUserPtr == UserPtr &&
1456 (TaggedUserPtr = loadTag(Ptr: UserPtr)) != UserPtr) {
1457 uptr PrevEnd = TaggedUserPtr + Header.SizeOrUnusedBytes;
1458 const uptr NextPage = roundUp(X: TaggedUserPtr, Boundary: getPageSizeCached());
1459 if (NextPage < PrevEnd && loadTag(Ptr: NextPage) != NextPage)
1460 PrevEnd = NextPage;
1461 TaggedPtr = reinterpret_cast<void *>(TaggedUserPtr);
1462 resizeTaggedChunk</*ClearAllTags*/ false>(PrevEnd, TaggedUserPtr + Size,
1463 Size, BlockEnd);
1464 if (UNLIKELY(FillContents != NoFill && !Header.OriginOrWasZeroed)) {
1465 // If an allocation needs to be zeroed (i.e. calloc) we can normally
1466 // avoid zeroing the memory now since we can rely on memory having
1467 // been zeroed on free, as this is normally done while setting the
1468 // UAF tag. But if tagging was disabled per-thread when the memory
1469 // was freed, it would not have been retagged and thus zeroed, and
1470 // therefore it needs to be zeroed now.
1471 memset(s: TaggedPtr, c: 0,
1472 n: Min(A: Size, B: roundUp(X: PrevEnd - TaggedUserPtr,
1473 Boundary: archMemoryTagGranuleSize())));
1474 } else if (Size) {
1475 // Clear any stack metadata that may have previously been stored in
1476 // the chunk data.
1477 memset(s: TaggedPtr, c: 0, n: archMemoryTagGranuleSize());
1478 }
1479 } else {
1480 const uptr OddEvenMask =
1481 computeOddEvenMaskForPointerMaybe(Options, Ptr: BlockUptr, ClassId);
1482 TaggedPtr = prepareTaggedChunk(Ptr, Size, ExcludeMask: OddEvenMask, BlockEnd);
1483 }
1484 storePrimaryAllocationStackMaybe(Options, Ptr);
1485 } else {
1486 // Init the secondary chunk.
1487
1488 Block = addHeaderTag(Block);
1489 Ptr = addHeaderTag(Ptr);
1490 storeTags(Begin: reinterpret_cast<uptr>(Block), End: reinterpret_cast<uptr>(Ptr));
1491 storeSecondaryAllocationStackMaybe(Options, Ptr, Size);
1492 }
1493
1494 Chunk::UnpackedHeader Header = {};
1495
1496 if (UNLIKELY(DefaultAlignedPtr != UserPtr)) {
1497 const uptr Offset = UserPtr - DefaultAlignedPtr;
1498 DCHECK_GE(Offset, 2 * sizeof(u32));
1499 // The BlockMarker has no security purpose, but is specifically meant for
1500 // the chunk iteration function that can be used in debugging situations.
1501 // It is the only situation where we have to locate the start of a chunk
1502 // based on its block address.
1503 reinterpret_cast<u32 *>(Block)[0] = BlockMarker;
1504 reinterpret_cast<u32 *>(Block)[1] = static_cast<u32>(Offset);
1505 Header.Offset = (Offset >> MinAlignmentLog) & Chunk::OffsetMask;
1506 }
1507
1508 Header.ClassId = ClassId & Chunk::ClassIdMask;
1509 Header.State = Chunk::State::Allocated;
1510 Header.setOrigin(Origin);
1511 Header.SizeOrUnusedBytes = SizeOrUnusedBytes & Chunk::SizeOrUnusedBytesMask;
1512 Chunk::storeHeader(Cookie, Ptr, NewUnpackedHeader: &Header);
1513
1514 return TaggedPtr;
1515 }
1516
1517 __attribute__((no_sanitize("unsigned-integer-overflow"))) bool
1518 canQuarantine(uptr Size) const {
1519 // Only quarantine if quarantine is enabled, cache size is non-zero, and
1520 // the actual size of the chunk is between 1 and QuarantineMaxChunkSize.
1521 // This purposefully underflows for Size == 0.
1522 return !AllocatorConfig::getQuarantineDisabled() &&
1523 Quarantine.getCacheSize() && ((Size - 1) < QuarantineMaxChunkSize);
1524 }
1525
1526 void quarantineOrDeallocateChunk(const Options &Options, void *TaggedPtr,
1527 void *Ptr, Chunk::UnpackedHeader *Header,
1528 uptr Size) NO_THREAD_SAFETY_ANALYSIS {
1529 const bool BypassQuarantine = !canQuarantine(Size) || !Header->ClassId;
1530 bool MemoryTaggingEnabled = useMemoryTagging<AllocatorConfig>(Options);
1531 bool ZeroOnDeallocate = AllocatorConfig::getDeallocZeroFill() &&
1532 Options.getFillContentsMode() == ZeroFill;
1533 if (!ZeroOnDeallocate) {
1534 Header->State = BypassQuarantine ? Chunk::State::Available
1535 : Chunk::State::Quarantined;
1536 if (LIKELY(!MemoryTaggingEnabled)) {
1537 Header->OriginOrWasZeroed = 0U;
1538 } else {
1539 Header->OriginOrWasZeroed =
1540 Header->ClassId && !TSDRegistry.getDisableMemInit();
1541 }
1542 Chunk::storeHeader(Cookie, Ptr, NewUnpackedHeader: Header);
1543 }
1544
1545 if (BypassQuarantine) {
1546 const uptr ClassId = Header->ClassId;
1547 void *BlockBegin;
1548 if (LIKELY(!MemoryTaggingEnabled)) {
1549 // Must do this after storeHeader because loadHeader uses a tagged ptr.
1550 if (allocatorSupportsMemoryTagging<AllocatorConfig>())
1551 Ptr = untagPointer(Ptr);
1552 BlockBegin = getBlockBegin(Ptr, Header);
1553 if (ZeroOnDeallocate && ClassId != 0) {
1554 memset(BlockBegin, 0, SizeClassMap::getSizeByClassId(ClassId));
1555 }
1556 } else {
1557 // retagBlock always zeroes the allocation.
1558 BlockBegin =
1559 retagBlock(Options, TaggedPtr, Ptr, Header, Size,
1560 /*ClearHeader*/ ClearHeader: ZeroOnDeallocate || ClassId == 0);
1561 }
1562
1563#if SCUDO_FUCHSIA
1564 if (AllocatorConfig::getEnableZeroOnDealloc()) {
1565 // Clearing the header is incompatible with quarantine and tagging.
1566 // Hence, it is fine to implement it only when quarantine is bypassed.
1567 DCHECK(!MemoryTaggingEnabled);
1568 uptr length = reinterpret_cast<uptr>(Ptr) + Size -
1569 reinterpret_cast<uptr>(BlockBegin);
1570 if (length <= ZeroOnDeallocMaxSize)
1571 memset(BlockBegin, 0, length);
1572 }
1573#endif // SCUDO_FUCHSIA
1574
1575 if (LIKELY(ClassId)) {
1576 bool CacheDrained;
1577 {
1578 typename TSDRegistryT::ScopedTSD TSD(TSDRegistry);
1579 CacheDrained =
1580 TSD->getSizeClassAllocator().deallocate(ClassId, BlockBegin);
1581 }
1582 // When we have drained some blocks back to the Primary from TSD, that
1583 // implies that we may have the chance to release some pages as well.
1584 // Note that in order not to block other thread's accessing the TSD,
1585 // release the TSD first then try the page release.
1586 if (CacheDrained)
1587 Primary.tryReleaseToOS(ClassId, ReleaseToOS::Normal);
1588 } else {
1589 Secondary.deallocate(Options, BlockBegin, ZeroOnDeallocate);
1590 }
1591 } else {
1592 // TODO: ZeroOnDeallocate is not supported when quarantining is enabled.
1593 CHECK(!ZeroOnDeallocate);
1594 if (UNLIKELY(MemoryTaggingEnabled))
1595 retagBlock(Options, TaggedPtr, Ptr, Header, Size,
1596 /*ClearHeader*/ ClearHeader: false);
1597 typename TSDRegistryT::ScopedTSD TSD(TSDRegistry);
1598 Quarantine.put(&TSD->getQuarantineCache(),
1599 QuarantineCallback(*this, TSD->getSizeClassAllocator()),
1600 Ptr, Size);
1601 }
1602 }
1603
1604 NOINLINE void *retagBlock(const Options &Options, void *TaggedPtr, void *&Ptr,
1605 Chunk::UnpackedHeader *Header, const uptr Size,
1606 bool ClearHeader) {
1607 DCHECK(useMemoryTagging<AllocatorConfig>(Options));
1608
1609 const u8 PrevTag = extractTag(Ptr: reinterpret_cast<uptr>(TaggedPtr));
1610 storeDeallocationStackMaybe(Options, Ptr, PrevTag, Size);
1611 if (Header->ClassId && !TSDRegistry.getDisableMemInit()) {
1612 const uptr OddEvenMask = computeOddEvenMaskForPointerMaybe(
1613 Options, Ptr: reinterpret_cast<uptr>(getBlockBegin(Ptr, Header)),
1614 ClassId: Header->ClassId);
1615 // Exclude the previous tag so that immediate use after free is
1616 // detected 100% of the time.
1617 uptr TaggedBegin, TaggedEnd;
1618 setRandomTag(Ptr, Size, ExcludeMask: OddEvenMask | (1UL << PrevTag), TaggedBegin: &TaggedBegin,
1619 TaggedEnd: &TaggedEnd);
1620 }
1621
1622 Ptr = untagPointer(Ptr);
1623 void *BlockBegin = getBlockBegin(Ptr, Header);
1624 if (ClearHeader) {
1625 storeTags(Begin: reinterpret_cast<uptr>(BlockBegin),
1626 End: reinterpret_cast<uptr>(Ptr));
1627 }
1628
1629 return BlockBegin;
1630 }
1631
1632 bool getChunkFromBlock(uptr Block, uptr *Chunk,
1633 Chunk::UnpackedHeader *Header) {
1634 *Chunk =
1635 Block + getChunkOffsetFromBlock(Block: reinterpret_cast<const char *>(Block));
1636 return Chunk::isValid(Cookie, Ptr: reinterpret_cast<void *>(*Chunk), NewUnpackedHeader: Header);
1637 }
1638
1639 static uptr getChunkOffsetFromBlock(const char *Block) {
1640 u32 Offset = 0;
1641 if (reinterpret_cast<const u32 *>(Block)[0] == BlockMarker)
1642 Offset = reinterpret_cast<const u32 *>(Block)[1];
1643 return Offset + Chunk::getHeaderSize();
1644 }
1645
1646 // Set the tag of the granule past the end of the allocation to 0, to catch
1647 // linear overflows even if a previous larger allocation used the same block
1648 // and tag. Only do this if the granule past the end is in our block, because
1649 // this would otherwise lead to a SEGV if the allocation covers the entire
1650 // block and our block is at the end of a mapping. The tag of the next block's
1651 // header granule will be set to 0, so it will serve the purpose of catching
1652 // linear overflows in this case.
1653 //
1654 // For allocations of size 0 we do not end up storing the address tag to the
1655 // memory tag space, which getInlineErrorInfo() normally relies on to match
1656 // address tags against chunks. To allow matching in this case we store the
1657 // address tag in the first byte of the chunk.
1658 void storeEndMarker(uptr End, uptr Size, uptr BlockEnd) {
1659 DCHECK_EQ(BlockEnd, untagPointer(BlockEnd));
1660 uptr UntaggedEnd = untagPointer(Ptr: End);
1661 if (UntaggedEnd != BlockEnd) {
1662 storeTag(Ptr: UntaggedEnd);
1663 if (Size == 0)
1664 *reinterpret_cast<u8 *>(UntaggedEnd) = extractTag(Ptr: End);
1665 }
1666 }
1667
1668 void *prepareTaggedChunk(void *Ptr, uptr Size, uptr ExcludeMask,
1669 uptr BlockEnd) {
1670 // Prepare the granule before the chunk to store the chunk header by setting
1671 // its tag to 0. Normally its tag will already be 0, but in the case where a
1672 // chunk holding a low alignment allocation is reused for a higher alignment
1673 // allocation, the chunk may already have a non-zero tag from the previous
1674 // allocation.
1675 storeTag(Ptr: reinterpret_cast<uptr>(Ptr) - archMemoryTagGranuleSize());
1676
1677 uptr TaggedBegin, TaggedEnd;
1678 setRandomTag(Ptr, Size, ExcludeMask, TaggedBegin: &TaggedBegin, TaggedEnd: &TaggedEnd);
1679
1680 storeEndMarker(End: TaggedEnd, Size, BlockEnd);
1681 return reinterpret_cast<void *>(TaggedBegin);
1682 }
1683
1684 template <bool ClearAllTags>
1685 void resizeTaggedChunk(uptr OldPtr, uptr NewPtr, uptr NewSize,
1686 uptr BlockEnd) {
1687 uptr RoundOldPtr = roundUp(X: OldPtr, Boundary: archMemoryTagGranuleSize());
1688 uptr RoundNewPtr;
1689 if (RoundOldPtr >= NewPtr) {
1690 RoundNewPtr = roundUp(X: NewPtr, Boundary: archMemoryTagGranuleSize());
1691 if (ClearAllTags) {
1692 // Shrinking resize, set the tag to zero for the rest of the allocation.
1693 // This prevents use after realloc of the excess memory.
1694 storeTags(Begin: untagPointer(Ptr: RoundNewPtr), End: untagPointer(Ptr: RoundOldPtr));
1695 } else {
1696 // Add a zero tag right after the pointer to prevent overflow.
1697 storeEndMarker(End: RoundNewPtr, Size: NewSize, BlockEnd);
1698 }
1699 } else {
1700 // Set the memory tag of the region
1701 // [RoundOldPtr, roundUp(NewPtr, archMemoryTagGranuleSize()))
1702 // to the pointer tag stored in OldPtr.
1703 RoundNewPtr = storeTags(Begin: RoundOldPtr, End: NewPtr);
1704 // Add a zero tag right after the pointer to prevent overflow.
1705 storeEndMarker(End: RoundNewPtr, Size: NewSize, BlockEnd);
1706 }
1707 }
1708
1709 void storePrimaryAllocationStackMaybe(const Options &Options, void *Ptr) {
1710 if (!UNLIKELY(Options.get(OptionBit::TrackAllocationStacks)))
1711 return;
1712 AllocationRingBuffer *RB = getRingBuffer();
1713 if (!RB)
1714 return;
1715 auto *Ptr32 = reinterpret_cast<u32 *>(Ptr);
1716 Ptr32[MemTagAllocationTraceIndex] = collectStackTrace(Depot: RB->Depot);
1717 Ptr32[MemTagAllocationTidIndex] = getThreadID();
1718 }
1719
1720 void storeRingBufferEntry(AllocationRingBuffer *RB, void *Ptr,
1721 u32 AllocationTrace, u32 AllocationTid,
1722 uptr AllocationSize, u32 DeallocationTrace,
1723 u32 DeallocationTid) {
1724 uptr Pos = atomic_fetch_add(&RB->Pos, 1, memory_order_relaxed);
1725 typename AllocationRingBuffer::Entry *Entry =
1726 getRingBufferEntry(RB, Pos % RB->RingBufferElements);
1727
1728 // First invalidate our entry so that we don't attempt to interpret a
1729 // partially written state in getSecondaryErrorInfo(). The fences below
1730 // ensure that the compiler does not move the stores to Ptr in between the
1731 // stores to the other fields.
1732 atomic_store_relaxed(&Entry->Ptr, 0);
1733
1734 __atomic_signal_fence(__ATOMIC_SEQ_CST);
1735 atomic_store_relaxed(&Entry->AllocationTrace, AllocationTrace);
1736 atomic_store_relaxed(&Entry->AllocationTid, AllocationTid);
1737 atomic_store_relaxed(&Entry->AllocationSize, AllocationSize);
1738 atomic_store_relaxed(&Entry->DeallocationTrace, DeallocationTrace);
1739 atomic_store_relaxed(&Entry->DeallocationTid, DeallocationTid);
1740 __atomic_signal_fence(__ATOMIC_SEQ_CST);
1741
1742 atomic_store_relaxed(&Entry->Ptr, reinterpret_cast<uptr>(Ptr));
1743 }
1744
1745 void storeSecondaryAllocationStackMaybe(const Options &Options, void *Ptr,
1746 uptr Size) {
1747 if (!UNLIKELY(Options.get(OptionBit::TrackAllocationStacks)))
1748 return;
1749 AllocationRingBuffer *RB = getRingBuffer();
1750 if (!RB)
1751 return;
1752 u32 Trace = collectStackTrace(Depot: RB->Depot);
1753 u32 Tid = getThreadID();
1754
1755 auto *Ptr32 = reinterpret_cast<u32 *>(Ptr);
1756 Ptr32[MemTagAllocationTraceIndex] = Trace;
1757 Ptr32[MemTagAllocationTidIndex] = Tid;
1758
1759 storeRingBufferEntry(RB, Ptr: untagPointer(Ptr), AllocationTrace: Trace, AllocationTid: Tid, AllocationSize: Size, DeallocationTrace: 0, DeallocationTid: 0);
1760 }
1761
1762 void storeDeallocationStackMaybe(const Options &Options, void *Ptr,
1763 u8 PrevTag, uptr Size) {
1764 if (!UNLIKELY(Options.get(OptionBit::TrackAllocationStacks)))
1765 return;
1766 AllocationRingBuffer *RB = getRingBuffer();
1767 if (!RB)
1768 return;
1769 auto *Ptr32 = reinterpret_cast<u32 *>(Ptr);
1770 u32 AllocationTrace = Ptr32[MemTagAllocationTraceIndex];
1771 u32 AllocationTid = Ptr32[MemTagAllocationTidIndex];
1772
1773 u32 DeallocationTrace = collectStackTrace(Depot: RB->Depot);
1774 u32 DeallocationTid = getThreadID();
1775
1776 storeRingBufferEntry(RB, Ptr: addFixedTag(Ptr: untagPointer(Ptr), Tag: PrevTag),
1777 AllocationTrace, AllocationTid, AllocationSize: Size,
1778 DeallocationTrace, DeallocationTid);
1779 }
1780
1781 uptr getStats(ScopedString *Str) {
1782 Str->append(Format: "Config Stats Base: ");
1783 AllocatorConfig::getConfigValues(Str);
1784 Primary.getStats(Str);
1785 Secondary.getStats(Str);
1786 if (!AllocatorConfig::getQuarantineDisabled())
1787 Quarantine.getStats(Str);
1788 TSDRegistry.getStats(Str);
1789 return Str->length();
1790 }
1791
1792 static typename AllocationRingBuffer::Entry *
1793 getRingBufferEntry(AllocationRingBuffer *RB, uptr N) {
1794 char *RBEntryStart =
1795 &reinterpret_cast<char *>(RB)[sizeof(AllocationRingBuffer)];
1796 return &reinterpret_cast<typename AllocationRingBuffer::Entry *>(
1797 RBEntryStart)[N];
1798 }
1799 static const typename AllocationRingBuffer::Entry *
1800 getRingBufferEntry(const AllocationRingBuffer *RB, uptr N) {
1801 const char *RBEntryStart =
1802 &reinterpret_cast<const char *>(RB)[sizeof(AllocationRingBuffer)];
1803 return &reinterpret_cast<const typename AllocationRingBuffer::Entry *>(
1804 RBEntryStart)[N];
1805 }
1806
1807 void initRingBufferMaybe() {
1808 ScopedLock L(RingBufferInitLock);
1809 if (getRingBuffer() != nullptr)
1810 return;
1811
1812 int ring_buffer_size = getFlags()->allocation_ring_buffer_size;
1813 if (ring_buffer_size <= 0)
1814 return;
1815
1816 u32 AllocationRingBufferSize = static_cast<u32>(ring_buffer_size);
1817
1818 // We store alloc and free stacks for each entry.
1819 constexpr u32 kStacksPerRingBufferEntry = 2;
1820 constexpr u32 kMaxU32Pow2 = ~(UINT32_MAX >> 1);
1821 static_assert(isPowerOfTwo(X: kMaxU32Pow2));
1822 // On Android we always have 3 frames at the bottom: __start_main,
1823 // __libc_init, main, and 3 at the top: malloc, scudo_malloc and
1824 // Allocator::allocate. This leaves 10 frames for the user app. The next
1825 // smallest power of two (8) would only leave 2, which is clearly too
1826 // little.
1827 constexpr u32 kFramesPerStack = 16;
1828 static_assert(isPowerOfTwo(X: kFramesPerStack));
1829
1830 if (AllocationRingBufferSize > kMaxU32Pow2 / kStacksPerRingBufferEntry)
1831 return;
1832 u32 TabSize = static_cast<u32>(roundUpPowerOfTwo(Size: kStacksPerRingBufferEntry *
1833 AllocationRingBufferSize));
1834 if (TabSize > UINT32_MAX / kFramesPerStack)
1835 return;
1836 u32 RingSize = static_cast<u32>(TabSize * kFramesPerStack);
1837
1838 uptr StackDepotSize = sizeof(StackDepot) + sizeof(atomic_u64) * RingSize +
1839 sizeof(atomic_u32) * TabSize;
1840 MemMapT DepotMap;
1841 DepotMap.map(
1842 /*Addr=*/Addr: 0U, Size: roundUp(X: StackDepotSize, Boundary: getPageSizeCached()),
1843 Name: "scudo:stack_depot");
1844 auto *Depot = reinterpret_cast<StackDepot *>(DepotMap.getBase());
1845 Depot->init(RingSz: RingSize, TabSz: TabSize);
1846
1847 MemMapT MemMap;
1848 MemMap.map(
1849 /*Addr=*/Addr: 0U,
1850 Size: roundUp(X: sizeof(AllocationRingBuffer) +
1851 AllocationRingBufferSize *
1852 sizeof(typename AllocationRingBuffer::Entry),
1853 Boundary: getPageSizeCached()),
1854 Name: "scudo:ring_buffer");
1855 auto *RB = reinterpret_cast<AllocationRingBuffer *>(MemMap.getBase());
1856 RB->RawRingBufferMap = MemMap;
1857 RB->RingBufferElements = AllocationRingBufferSize;
1858 RB->Depot = Depot;
1859 RB->StackDepotSize = StackDepotSize;
1860 RB->RawStackDepotMap = DepotMap;
1861
1862 atomic_store(A: &RingBufferAddress, V: reinterpret_cast<uptr>(RB),
1863 MO: memory_order_release);
1864 }
1865
1866 void unmapRingBuffer() {
1867 AllocationRingBuffer *RB = getRingBuffer();
1868 if (RB == nullptr)
1869 return;
1870 // N.B. because RawStackDepotMap is part of RawRingBufferMap, the order
1871 // is very important.
1872 RB->RawStackDepotMap.unmap();
1873 // Note that the `RB->RawRingBufferMap` is stored on the pages managed by
1874 // itself. Take over the ownership before calling unmap() so that any
1875 // operation along with unmap() won't touch inaccessible pages.
1876 MemMapT RawRingBufferMap = RB->RawRingBufferMap;
1877 RawRingBufferMap.unmap();
1878 atomic_store(A: &RingBufferAddress, V: 0, MO: memory_order_release);
1879 }
1880};
1881
1882} // namespace scudo
1883
1884#endif // SCUDO_COMBINED_H_
1885