1//===-- combined.h ----------------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#ifndef SCUDO_COMBINED_H_
10#define SCUDO_COMBINED_H_
11
12#include "allocator_config_wrapper.h"
13#include "atomic_helpers.h"
14#include "chunk.h"
15#include "common.h"
16#include "flags.h"
17#include "flags_parser.h"
18#include "mem_map.h"
19#include "memtag.h"
20#include "mutex.h"
21#include "options.h"
22#include "quarantine.h"
23#include "report.h"
24#include "secondary.h"
25#include "size_class_allocator.h"
26#include "stack_depot.h"
27#include "string_utils.h"
28#include "tracing.h"
29#include "tsd.h"
30
31#include "scudo/interface.h"
32
33#ifdef GWP_ASAN_HOOKS
34#include "gwp_asan/guarded_pool_allocator.h"
35#include "gwp_asan/optional/backtrace.h"
36#include "gwp_asan/optional/segv_handler.h"
37#endif // GWP_ASAN_HOOKS
38
39extern "C" inline void EmptyCallback() {}
40
41#ifdef HAVE_ANDROID_UNSAFE_FRAME_POINTER_CHASE
42// This function is not part of the NDK so it does not appear in any public
43// header files. We only declare/use it when targeting the platform.
44extern "C" size_t android_unsafe_frame_pointer_chase(scudo::uptr *buf,
45 size_t num_entries);
46#endif
47
48namespace scudo {
49
50template <class Config, void (*PostInitCallback)(void) = EmptyCallback>
51class Allocator {
52public:
53 using AllocatorConfig = BaseConfig<Config>;
54 using PrimaryT =
55 typename AllocatorConfig::template PrimaryT<PrimaryConfig<Config>>;
56 using SecondaryT =
57 typename AllocatorConfig::template SecondaryT<SecondaryConfig<Config>>;
58 using SizeClassAllocatorT = typename PrimaryT::SizeClassAllocatorT;
59 typedef Allocator<Config, PostInitCallback> ThisT;
60 typedef typename AllocatorConfig::template TSDRegistryT<ThisT> TSDRegistryT;
61
62 void callPostInitCallback() {
63 pthread_once(once_control: &PostInitNonce, init_routine: PostInitCallback);
64 }
65
66 struct QuarantineCallback {
67 explicit QuarantineCallback(ThisT &Instance,
68 SizeClassAllocatorT &SizeClassAllocator)
69 : Allocator(Instance), SizeClassAllocator(SizeClassAllocator) {}
70
71 // Chunk recycling function, returns a quarantined chunk to the backend,
72 // first making sure it hasn't been tampered with.
73 void recycle(void *Ptr) {
74 Chunk::UnpackedHeader Header;
75 Chunk::loadHeader(Cookie: Allocator.Cookie, Ptr, NewUnpackedHeader: &Header);
76 if (UNLIKELY(Header.State != Chunk::State::Quarantined))
77 reportInvalidChunkState(Action: AllocatorAction::Recycling, Ptr);
78
79 Header.State = Chunk::State::Available;
80 Chunk::storeHeader(Cookie: Allocator.Cookie, Ptr, NewUnpackedHeader: &Header);
81
82 if (allocatorSupportsMemoryTagging<AllocatorConfig>())
83 Ptr = untagPointer(Ptr);
84 void *BlockBegin = Allocator::getBlockBegin(Ptr, Header: &Header);
85 SizeClassAllocator.deallocate(Header.ClassId, BlockBegin);
86 }
87
88 // We take a shortcut when allocating a quarantine batch by working with the
89 // appropriate class ID instead of using Size. The compiler should optimize
90 // the class ID computation and work with the associated cache directly.
91 void *allocate(UNUSED uptr Size) {
92 const uptr QuarantineClassId = SizeClassMap::getClassIdBySize(
93 sizeof(QuarantineBatch) + Chunk::getHeaderSize());
94 void *Ptr = SizeClassAllocator.allocate(QuarantineClassId);
95 // Quarantine batch allocation failure is fatal.
96 if (UNLIKELY(!Ptr))
97 reportOutOfMemory(SizeClassMap::getSizeByClassId(QuarantineClassId));
98
99 Ptr = reinterpret_cast<void *>(reinterpret_cast<uptr>(Ptr) +
100 Chunk::getHeaderSize());
101 Chunk::UnpackedHeader Header = {};
102 Header.ClassId = QuarantineClassId & Chunk::ClassIdMask;
103 Header.SizeOrUnusedBytes = sizeof(QuarantineBatch);
104 Header.State = Chunk::State::Quarantined;
105 Chunk::storeHeader(Cookie: Allocator.Cookie, Ptr, NewUnpackedHeader: &Header);
106
107 // Reset tag to 0 as this chunk may have been previously used for a tagged
108 // user allocation.
109 if (UNLIKELY(useMemoryTagging<AllocatorConfig>(
110 Allocator.Primary.Options.load())))
111 storeTags(Begin: reinterpret_cast<uptr>(Ptr),
112 End: reinterpret_cast<uptr>(Ptr) + sizeof(QuarantineBatch));
113
114 return Ptr;
115 }
116
117 void deallocate(void *Ptr) {
118 const uptr QuarantineClassId = SizeClassMap::getClassIdBySize(
119 sizeof(QuarantineBatch) + Chunk::getHeaderSize());
120 Chunk::UnpackedHeader Header;
121 Chunk::loadHeader(Cookie: Allocator.Cookie, Ptr, NewUnpackedHeader: &Header);
122
123 if (UNLIKELY(Header.State != Chunk::State::Quarantined))
124 reportInvalidChunkState(Action: AllocatorAction::Deallocating, Ptr);
125 DCHECK_EQ(Header.ClassId, QuarantineClassId);
126 DCHECK_EQ(Header.Offset, 0);
127 DCHECK_EQ(Header.SizeOrUnusedBytes, sizeof(QuarantineBatch));
128
129 Header.State = Chunk::State::Available;
130 Chunk::storeHeader(Cookie: Allocator.Cookie, Ptr, NewUnpackedHeader: &Header);
131 SizeClassAllocator.deallocate(
132 QuarantineClassId,
133 reinterpret_cast<void *>(reinterpret_cast<uptr>(Ptr) -
134 Chunk::getHeaderSize()));
135 }
136
137 private:
138 ThisT &Allocator;
139 SizeClassAllocatorT &SizeClassAllocator;
140 };
141
142 typedef GlobalQuarantine<QuarantineCallback, void> QuarantineT;
143 typedef typename QuarantineT::CacheT QuarantineCacheT;
144
145 void init() {
146 // Make sure that the page size is initialized if it's not a constant.
147 CHECK_NE(getPageSizeCached(), 0U);
148
149 performSanityChecks();
150
151 // Check if hardware CRC32 is supported in the binary and by the platform,
152 // if so, opt for the CRC32 hardware version of the checksum.
153 if (&computeHardwareCRC32 && hasHardwareCRC32())
154 HashAlgorithm = Checksum::HardwareCRC32;
155
156 if (UNLIKELY(!getRandom(&Cookie, sizeof(Cookie)))) {
157 ScopedString Str;
158 Str.append(Format: "Randomness degraded.\n");
159 Str.output();
160 Cookie = static_cast<u32>(getMonotonicTime() ^
161 (reinterpret_cast<uptr>(this) >> 4));
162 }
163
164 initFlags();
165 reportUnrecognizedFlags();
166
167 // Store some flags locally.
168 if (getFlags()->may_return_null)
169 Primary.Options.set(OptionBit::MayReturnNull);
170 if (getFlags()->zero_contents)
171 Primary.Options.setFillContentsMode(ZeroFill);
172 else if (getFlags()->pattern_fill_contents)
173 Primary.Options.setFillContentsMode(PatternOrZeroFill);
174 if (getFlags()->dealloc_type_mismatch)
175 Primary.Options.set(OptionBit::DeallocTypeMismatch);
176 if (getFlags()->dealloc_align_mismatch)
177 Primary.Options.set(OptionBit::DeallocAlignMismatch);
178 if (getFlags()->delete_size_mismatch)
179 Primary.Options.set(OptionBit::DeleteSizeMismatch);
180 if (systemSupportsMemoryTagging())
181 Primary.Options.set(OptionBit::UseMemoryTagging);
182
183 QuarantineMaxChunkSize =
184 static_cast<u32>(getFlags()->quarantine_max_chunk_size);
185#if SCUDO_FUCHSIA
186 ZeroOnDeallocMaxSize =
187 static_cast<u32>(getFlags()->zero_on_dealloc_max_size);
188#endif
189
190 Stats.init();
191 // TODO(chiahungduan): Given that we support setting the default value in
192 // the PrimaryConfig and CacheConfig, consider to deprecate the use of
193 // `release_to_os_interval_ms` flag.
194 const s32 ReleaseToOsIntervalMs = getFlags()->release_to_os_interval_ms;
195 Primary.init(ReleaseToOsIntervalMs);
196 Secondary.init(&Stats, ReleaseToOsIntervalMs);
197 if (!AllocatorConfig::getQuarantineDisabled()) {
198 Quarantine.init(
199 static_cast<uptr>(getFlags()->quarantine_size_kb << 10),
200 static_cast<uptr>(getFlags()->thread_local_quarantine_size_kb << 10));
201 }
202 }
203
204 void enableRingBuffer() NO_THREAD_SAFETY_ANALYSIS {
205 AllocationRingBuffer *RB = getRingBuffer();
206 if (RB)
207 RB->Depot->enable();
208 RingBufferInitLock.unlock();
209 }
210
211 void disableRingBuffer() NO_THREAD_SAFETY_ANALYSIS {
212 RingBufferInitLock.lock();
213 AllocationRingBuffer *RB = getRingBuffer();
214 if (RB)
215 RB->Depot->disable();
216 }
217
218 // Initialize the embedded GWP-ASan instance. Requires the main allocator to
219 // be functional, best called from PostInitCallback.
220 void initGwpAsan() {
221#ifdef GWP_ASAN_HOOKS
222 gwp_asan::options::Options Opt;
223 Opt.Enabled = getFlags()->GWP_ASAN_Enabled;
224 Opt.MaxSimultaneousAllocations =
225 getFlags()->GWP_ASAN_MaxSimultaneousAllocations;
226 Opt.SampleRate = getFlags()->GWP_ASAN_SampleRate;
227 Opt.InstallSignalHandlers = getFlags()->GWP_ASAN_InstallSignalHandlers;
228 Opt.Recoverable = getFlags()->GWP_ASAN_Recoverable;
229 // Embedded GWP-ASan is locked through the Scudo atfork handler (via
230 // Allocator::disable calling GWPASan.disable). Disable GWP-ASan's atfork
231 // handler.
232 Opt.InstallForkHandlers = false;
233 Opt.Backtrace = gwp_asan::backtrace::getBacktraceFunction();
234 GuardedAlloc.init(Opts: Opt);
235
236 if (Opt.InstallSignalHandlers)
237 gwp_asan::segv_handler::installSignalHandlers(
238 GPA: &GuardedAlloc, Printf,
239 PrintBacktrace: gwp_asan::backtrace::getPrintBacktraceFunction(),
240 SegvBacktrace: gwp_asan::backtrace::getSegvBacktraceFunction(),
241 Recoverable: Opt.Recoverable);
242
243 GuardedAllocSlotSize =
244 GuardedAlloc.getAllocatorState()->maximumAllocationSize();
245 Stats.add(I: StatFree, V: static_cast<uptr>(Opt.MaxSimultaneousAllocations) *
246 GuardedAllocSlotSize);
247#endif // GWP_ASAN_HOOKS
248 }
249
250#ifdef GWP_ASAN_HOOKS
251 const gwp_asan::AllocationMetadata *getGwpAsanAllocationMetadata() {
252 return GuardedAlloc.getMetadataRegion();
253 }
254
255 const gwp_asan::AllocatorState *getGwpAsanAllocatorState() {
256 return GuardedAlloc.getAllocatorState();
257 }
258#endif // GWP_ASAN_HOOKS
259
260 ALWAYS_INLINE void initThreadMaybe(bool MinimalInit = false) {
261 TSDRegistry.initThreadMaybe(this, MinimalInit);
262 }
263
264 void unmapTestOnly() {
265 unmapRingBuffer();
266 TSDRegistry.unmapTestOnly(this);
267 Primary.unmapTestOnly();
268 Secondary.unmapTestOnly();
269#ifdef GWP_ASAN_HOOKS
270 if (getFlags()->GWP_ASAN_InstallSignalHandlers)
271 gwp_asan::segv_handler::uninstallSignalHandlers();
272 GuardedAlloc.uninitTestOnly();
273#endif // GWP_ASAN_HOOKS
274 }
275
276 TSDRegistryT *getTSDRegistry() { return &TSDRegistry; }
277 QuarantineT *getQuarantine() { return &Quarantine; }
278
279 // The Cache must be provided zero-initialized.
280 void initAllocator(SizeClassAllocatorT *SizeClassAllocator) {
281 SizeClassAllocator->init(&Stats, &Primary);
282 }
283
284 // Release the resources used by a TSD, which involves:
285 // - draining the local quarantine cache to the global quarantine;
286 // - releasing the cached pointers back to the Primary;
287 // - unlinking the local stats from the global ones (destroying the cache does
288 // the last two items).
289 void commitBack(TSD<ThisT> *TSD) {
290 TSD->assertLocked(/*BypassCheck=*/true);
291 if (!AllocatorConfig::getQuarantineDisabled()) {
292 Quarantine.drain(&TSD->getQuarantineCache(),
293 QuarantineCallback(*this, TSD->getSizeClassAllocator()));
294 }
295 TSD->getSizeClassAllocator().destroy(&Stats);
296 }
297
298 void drainCache(TSD<ThisT> *TSD) {
299 TSD->assertLocked(/*BypassCheck=*/true);
300 if (!AllocatorConfig::getQuarantineDisabled()) {
301 Quarantine.drainAndRecycle(
302 &TSD->getQuarantineCache(),
303 QuarantineCallback(*this, TSD->getSizeClassAllocator()));
304 }
305 TSD->getSizeClassAllocator().drain();
306 }
307 void drainCaches() { TSDRegistry.drainCaches(this); }
308
309 ALWAYS_INLINE void *getHeaderTaggedPointer(void *Ptr) {
310 if (!allocatorSupportsMemoryTagging<AllocatorConfig>())
311 return Ptr;
312 auto UntaggedPtr = untagPointer(Ptr);
313 if (UntaggedPtr != Ptr)
314 return UntaggedPtr;
315 // Secondary, or pointer allocated while memory tagging is unsupported or
316 // disabled. The tag mismatch is okay in the latter case because tags will
317 // not be checked.
318 return addHeaderTag(Ptr);
319 }
320
321 ALWAYS_INLINE uptr addHeaderTag(uptr Ptr) {
322 if (!allocatorSupportsMemoryTagging<AllocatorConfig>())
323 return Ptr;
324 return addFixedTag(Ptr, Tag: 2);
325 }
326
327 ALWAYS_INLINE void *addHeaderTag(void *Ptr) {
328 return reinterpret_cast<void *>(addHeaderTag(reinterpret_cast<uptr>(Ptr)));
329 }
330
331 NOINLINE u32 collectStackTrace(UNUSED StackDepot *Depot) {
332#ifdef HAVE_ANDROID_UNSAFE_FRAME_POINTER_CHASE
333 // Discard collectStackTrace() frame and allocator function frame.
334 constexpr uptr DiscardFrames = 2;
335 uptr Stack[ScudoTraceSize + DiscardFrames];
336 uptr Size = android_unsafe_frame_pointer_chase(Stack, ScudoTraceSize +
337 DiscardFrames);
338 Size = Min<uptr>(Size, ScudoTraceSize + DiscardFrames);
339 return Depot->insert(Stack + Min<uptr>(DiscardFrames, Size), Stack + Size);
340#else
341 return 0;
342#endif
343 }
344
345 uptr computeOddEvenMaskForPointerMaybe(const Options &Options, uptr Ptr,
346 uptr ClassId) {
347 if (!Options.get(Opt: OptionBit::UseOddEvenTags))
348 return 0;
349
350 // If a chunk's tag is odd, we want the tags of the surrounding blocks to be
351 // even, and vice versa. Blocks are laid out Size bytes apart, and adding
352 // Size to Ptr will flip the least significant set bit of Size in Ptr, so
353 // that bit will have the pattern 010101... for consecutive blocks, which we
354 // can use to determine which tag mask to use.
355 return 0x5555U << ((Ptr >> SizeClassMap::getSizeLSBByClassId(ClassId)) & 1);
356 }
357
358 NOINLINE void *allocate(uptr Size, Chunk::Origin Origin,
359 uptr Alignment = MinAlignment,
360 bool ZeroContents = false) NO_THREAD_SAFETY_ANALYSIS {
361 initThreadMaybe();
362
363 const Options Options = Primary.Options.load();
364 if (UNLIKELY(Alignment > MaxAlignment)) {
365 if (Options.get(Opt: OptionBit::MayReturnNull))
366 return nullptr;
367 reportAlignmentTooBig(Alignment, MaxAlignment);
368 }
369 if (Alignment < MinAlignment)
370 Alignment = MinAlignment;
371
372#ifdef GWP_ASAN_HOOKS
373 if (UNLIKELY(GuardedAlloc.shouldSample())) {
374 if (void *Ptr = GuardedAlloc.allocate(Size, Alignment)) {
375 Stats.lock();
376 Stats.add(I: StatAllocated, V: GuardedAllocSlotSize);
377 Stats.sub(I: StatFree, V: GuardedAllocSlotSize);
378 Stats.unlock();
379 return Ptr;
380 }
381 }
382#endif // GWP_ASAN_HOOKS
383
384 const FillContentsMode FillContents = ZeroContents ? ZeroFill
385 : TSDRegistry.getDisableMemInit()
386 ? NoFill
387 : Options.getFillContentsMode();
388
389 // If the requested size happens to be 0 (more common than you might think),
390 // allocate MinAlignment bytes on top of the header. Then add the extra
391 // bytes required to fulfill the alignment requirements: we allocate enough
392 // to be sure that there will be an address in the block that will satisfy
393 // the alignment.
394 const uptr NeededSize =
395 roundUp(X: Size, Boundary: MinAlignment) +
396 ((Alignment > MinAlignment) ? Alignment : Chunk::getHeaderSize());
397
398 // Takes care of extravagantly large sizes as well as integer overflows.
399 static_assert(MaxAllowedMallocSize < UINTPTR_MAX - MaxAlignment, "");
400 if (UNLIKELY(Size >= MaxAllowedMallocSize)) {
401 if (Options.get(Opt: OptionBit::MayReturnNull))
402 return nullptr;
403 reportAllocationSizeTooBig(UserSize: Size, TotalSize: NeededSize, MaxSize: MaxAllowedMallocSize);
404 }
405 DCHECK_LE(Size, NeededSize);
406
407 void *Block = nullptr;
408 uptr ClassId = 0;
409 uptr SecondaryBlockEnd = 0;
410 if (LIKELY(PrimaryT::canAllocate(NeededSize))) {
411 ClassId = SizeClassMap::getClassIdBySize(NeededSize);
412 DCHECK_NE(ClassId, 0U);
413 typename TSDRegistryT::ScopedTSD TSD(TSDRegistry);
414 Block = TSD->getSizeClassAllocator().allocate(ClassId);
415 // If the allocation failed, retry in each successively larger class until
416 // it fits. If it fails to fit in the largest class, fallback to the
417 // Secondary.
418 if (UNLIKELY(!Block)) {
419 while (ClassId < SizeClassMap::LargestClassId && !Block)
420 Block = TSD->getSizeClassAllocator().allocate(++ClassId);
421 if (!Block)
422 ClassId = 0;
423 }
424 }
425 if (UNLIKELY(ClassId == 0)) {
426 Block = Secondary.allocate(Options, Size, Alignment, &SecondaryBlockEnd,
427 FillContents);
428 }
429
430 if (UNLIKELY(!Block)) {
431 if (Options.get(Opt: OptionBit::MayReturnNull))
432 return nullptr;
433 printStats();
434 reportOutOfMemory(RequestedSize: NeededSize);
435 }
436
437 const uptr UserPtr = roundUp(
438 X: reinterpret_cast<uptr>(Block) + Chunk::getHeaderSize(), Boundary: Alignment);
439 const uptr SizeOrUnusedBytes =
440 ClassId ? Size : SecondaryBlockEnd - (UserPtr + Size);
441
442 if (LIKELY(!useMemoryTagging<AllocatorConfig>(Options))) {
443 return initChunk(ClassId, Origin, Block, UserPtr, SizeOrUnusedBytes,
444 FillContents);
445 }
446
447 return initChunkWithMemoryTagging(ClassId, Origin, Block, UserPtr, Size,
448 SizeOrUnusedBytes, FillContents);
449 }
450
451 ALWAYS_INLINE void deallocate(void *Ptr, Chunk::Origin Origin) {
452 deallocate(Ptr, Origin, /*DeleteSize=*/0, /*DeleteAlignment=*/0);
453 }
454
455 ALWAYS_INLINE void deallocateSized(void *Ptr, Chunk::Origin Origin,
456 uptr DeleteSize) {
457 deallocate(Ptr, Origin | Chunk::Origin::Size, DeleteSize,
458 /*DeleteAlignment=*/0);
459 }
460
461 ALWAYS_INLINE void deallocateSizedAligned(void *Ptr, Chunk::Origin Origin,
462 uptr DeleteSize,
463 uptr DeleteAlignment) {
464 deallocate(Ptr, Origin | Chunk::Origin::Size | Chunk::Origin::Align,
465 DeleteSize, DeleteAlignment);
466 }
467
468 ALWAYS_INLINE void deallocateAligned(void *Ptr, Chunk::Origin Origin,
469 uptr DeleteAlignment) {
470 deallocate(Ptr, Origin | Chunk::Origin::Align,
471 /*DeleteSize=*/0, /*DeleteAlignment=*/DeleteAlignment);
472 }
473
474 ALWAYS_INLINE void checkSizeMatch(const void *Ptr,
475 Chunk::UnpackedHeader *Header, uptr Size,
476 uptr DeallocSize) {
477 if (AllocatorConfig::getExactUsableSize()) {
478 if (DeallocSize != Size)
479 reportDeleteSizeMismatch(Ptr, Size: DeallocSize, ExpectedSize: Size);
480 } else if (DeallocSize != Size && DeallocSize != getUsableSize(Ptr, Header))
481 reportDeleteSizeMismatch(Ptr, DeallocSize, Size,
482 getUsableSize(Ptr, Header));
483 }
484
485 ALWAYS_INLINE void checkTypeMatch(AllocatorAction Action, const void *Ptr,
486 u8 AllocOrigin, u8 DeallocOrigin) {
487 if (UNLIKELY(Chunk::originBaseType(AllocOrigin) !=
488 Chunk::originBaseType(DeallocOrigin)))
489 reportDeallocTypeMismatch(Action, Ptr, AllocOrigin, DeallocOrigin);
490
491 // There is no way to store that a new/new [] did an aligned allocate,
492 // so skip that part of the verification.
493 if (UNLIKELY(AllocOrigin == Chunk::Origin::New ||
494 AllocOrigin == Chunk::Origin::NewArray))
495 return;
496
497 if (Chunk::originAligned(Origin: AllocOrigin)) {
498 // Only disallow an aligned allocation and a non-aligned deallocation
499 // if this is a realloc.
500 if (Action == AllocatorAction::Reallocating &&
501 !Chunk::originAligned(Origin: DeallocOrigin))
502 reportDeallocTypeMismatch(Action, Ptr, AllocOrigin, DeallocOrigin);
503 } else if (Chunk::originAligned(Origin: DeallocOrigin)) {
504 // Origin not aligned, dealloc aligned.
505 reportDeallocTypeMismatch(Action, Ptr, AllocOrigin, DeallocOrigin);
506 }
507 }
508
509 NOINLINE void deallocate(void *Ptr, u8 DeallocOrigin, uptr DeleteSize,
510 uptr DeleteAlignment) {
511 if (UNLIKELY(!Ptr))
512 return;
513
514 // For a deallocation, we only ensure minimal initialization, meaning thread
515 // local data will be left uninitialized for now (when using ELF TLS). The
516 // fallback cache will be used instead. This is a workaround for a situation
517 // where the only heap operation performed in a thread would be a free past
518 // the TLS destructors, ending up in initialized thread specific data never
519 // being destroyed properly. Any other heap operation will do a full init.
520 initThreadMaybe(/*MinimalInit=*/MinimalInit: true);
521
522#ifdef GWP_ASAN_HOOKS
523 if (UNLIKELY(GuardedAlloc.pointerIsMine(Ptr))) {
524 GuardedAlloc.deallocate(Ptr);
525 Stats.lock();
526 Stats.add(I: StatFree, V: GuardedAllocSlotSize);
527 Stats.sub(I: StatAllocated, V: GuardedAllocSlotSize);
528 Stats.unlock();
529 return;
530 }
531#endif // GWP_ASAN_HOOKS
532
533 if (UNLIKELY(!isAligned(reinterpret_cast<uptr>(Ptr), MinAlignment)))
534 reportMisalignedPointer(Action: AllocatorAction::Deallocating, Ptr);
535
536 if (UNLIKELY(Chunk::originAligned(DeallocOrigin) &&
537 !isPowerOfTwo(DeleteAlignment)))
538 reportAlignmentNotPowerOfTwo(Alignment: DeleteAlignment);
539
540 void *TaggedPtr = Ptr;
541 Ptr = getHeaderTaggedPointer(Ptr);
542
543 Chunk::UnpackedHeader Header;
544 Chunk::loadHeader(Cookie, Ptr, NewUnpackedHeader: &Header);
545
546 if (UNLIKELY(Header.State != Chunk::State::Allocated))
547 reportInvalidChunkState(Action: AllocatorAction::Deallocating, Ptr);
548
549 const Options Options = Primary.Options.load();
550 const uptr Size = getSize(Ptr, Header: &Header);
551 if (AllocatorConfig::getAbortOnDeallocSizeMismatch() &&
552 Chunk::originSized(Origin: DeallocOrigin) &&
553 Options.get(Opt: OptionBit::DeleteSizeMismatch))
554 checkSizeMatch(Ptr, Header: &Header, Size, DeallocSize: DeleteSize);
555
556 if (AllocatorConfig::getAbortOnDeallocTypeMismatch() &&
557 Options.get(Opt: OptionBit::DeallocTypeMismatch))
558 checkTypeMatch(Action: AllocatorAction::Deallocating, Ptr, AllocOrigin: Header.getOrigin(),
559 DeallocOrigin);
560
561 if (UNLIKELY(AllocatorConfig::getAbortOnDeallocAlignmentMismatch() &&
562 Chunk::originAligned(DeallocOrigin) &&
563 Options.get(OptionBit::DeallocAlignMismatch) &&
564 !isAligned(reinterpret_cast<uptr>(Ptr), DeleteAlignment)))
565 reportDeleteAlignmentMismatch(Ptr, Alignment: DeleteAlignment);
566
567 quarantineOrDeallocateChunk(Options, TaggedPtr, Header: &Header, Size);
568 }
569
570 void *reallocate(void *OldPtr, uptr NewSize, uptr Alignment = MinAlignment) {
571 initThreadMaybe();
572
573 const Options Options = Primary.Options.load();
574 if (UNLIKELY(NewSize >= MaxAllowedMallocSize)) {
575 if (Options.get(Opt: OptionBit::MayReturnNull))
576 return nullptr;
577 reportAllocationSizeTooBig(UserSize: NewSize, TotalSize: 0, MaxSize: MaxAllowedMallocSize);
578 }
579
580 // The following cases are handled by the C wrappers.
581 DCHECK_NE(OldPtr, nullptr);
582 DCHECK_NE(NewSize, 0);
583
584#ifdef GWP_ASAN_HOOKS
585 if (UNLIKELY(GuardedAlloc.pointerIsMine(OldPtr))) {
586 uptr OldSize = GuardedAlloc.getSize(Ptr: OldPtr);
587 void *NewPtr = allocate(Size: NewSize, Origin: Chunk::Origin::Malloc, Alignment);
588 if (NewPtr)
589 memcpy(dest: NewPtr, src: OldPtr, n: (NewSize < OldSize) ? NewSize : OldSize);
590 GuardedAlloc.deallocate(Ptr: OldPtr);
591 Stats.lock();
592 Stats.add(I: StatFree, V: GuardedAllocSlotSize);
593 Stats.sub(I: StatAllocated, V: GuardedAllocSlotSize);
594 Stats.unlock();
595 return NewPtr;
596 }
597#endif // GWP_ASAN_HOOKS
598
599 void *OldTaggedPtr = OldPtr;
600 OldPtr = getHeaderTaggedPointer(Ptr: OldPtr);
601
602 if (UNLIKELY(!isAligned(reinterpret_cast<uptr>(OldPtr), MinAlignment)))
603 reportMisalignedPointer(Action: AllocatorAction::Reallocating, Ptr: OldPtr);
604
605 Chunk::UnpackedHeader Header;
606 Chunk::loadHeader(Cookie, Ptr: OldPtr, NewUnpackedHeader: &Header);
607
608 if (UNLIKELY(Header.State != Chunk::State::Allocated))
609 reportInvalidChunkState(Action: AllocatorAction::Reallocating, Ptr: OldPtr);
610
611 // Pointer has to be allocated with a malloc-type function. Some
612 // applications think that it is OK to realloc a memalign'ed pointer, which
613 // will trigger this check. It really isn't.
614 if (AllocatorConfig::getAbortOnDeallocTypeMismatch() &&
615 Options.get(Opt: OptionBit::DeallocTypeMismatch))
616 checkTypeMatch(Action: AllocatorAction::Reallocating, Ptr: OldPtr, AllocOrigin: Header.getOrigin(),
617 DeallocOrigin: Chunk::Origin::Malloc);
618
619 void *BlockBegin = getBlockBegin(Ptr: OldTaggedPtr, Header: &Header);
620 uptr BlockEnd;
621 const uptr ClassId = Header.ClassId;
622 uptr OldSize;
623 uptr UsableSize;
624 if (LIKELY(ClassId)) {
625 BlockEnd = reinterpret_cast<uptr>(BlockBegin) +
626 SizeClassMap::getSizeByClassId(ClassId);
627 OldSize = Header.SizeOrUnusedBytes;
628 UsableSize = BlockEnd - reinterpret_cast<uptr>(OldTaggedPtr);
629 } else {
630 BlockEnd = SecondaryT::getBlockEnd(BlockBegin);
631 UsableSize = BlockEnd - reinterpret_cast<uptr>(OldTaggedPtr);
632 OldSize = UsableSize - Header.SizeOrUnusedBytes;
633 }
634
635 // If the new chunk fits in the previously allocated block, do nothing
636 // but update the header and return immediately.
637 if (NewSize <= UsableSize) {
638 Header.SizeOrUnusedBytes =
639 (ClassId
640 ? NewSize
641 : BlockEnd - (reinterpret_cast<uptr>(OldTaggedPtr) + NewSize)) &
642 Chunk::SizeOrUnusedBytesMask;
643 Chunk::storeHeader(Cookie, Ptr: OldPtr, NewUnpackedHeader: &Header);
644 if (UNLIKELY(useMemoryTagging<AllocatorConfig>(Options))) {
645 if (ClassId) {
646 resizeTaggedChunk</*ClearAllTags*/ true>(
647 reinterpret_cast<uptr>(OldTaggedPtr) + OldSize,
648 reinterpret_cast<uptr>(OldTaggedPtr) + NewSize, NewSize,
649 untagPointer(Ptr: BlockEnd));
650 storePrimaryAllocationStackMaybe(Options, Ptr: OldPtr);
651 } else {
652 storeSecondaryAllocationStackMaybe(Options, Ptr: OldPtr, Size: NewSize);
653 }
654 }
655 return OldTaggedPtr;
656 }
657
658 // Otherwise we allocate a new one, and deallocate the old one. Some
659 // allocators will allocate an even larger chunk (by a fixed factor) to
660 // allow for potential further in-place realloc. The gains of such a trick
661 // are currently unclear.
662 void *NewPtr = allocate(Size: NewSize, Origin: Chunk::Origin::Malloc, Alignment);
663 if (LIKELY(NewPtr)) {
664 bool ExactSize = AllocatorConfig::getExactUsableSize() ||
665 useMemoryTagging<AllocatorConfig>(Options);
666 memcpy(dest: NewPtr, src: OldTaggedPtr,
667 n: Min(A: NewSize, B: ExactSize ? OldSize : UsableSize));
668 quarantineOrDeallocateChunk(Options, TaggedPtr: OldTaggedPtr, Header: &Header, Size: OldSize);
669 }
670 return NewPtr;
671 }
672
673 // TODO(kostyak): disable() is currently best-effort. There are some small
674 // windows of time when an allocation could still succeed after
675 // this function finishes. We will revisit that later.
676 void disable() NO_THREAD_SAFETY_ANALYSIS {
677 initThreadMaybe();
678#ifdef GWP_ASAN_HOOKS
679 GuardedAlloc.disable();
680#endif
681 TSDRegistry.disable();
682 Stats.disable();
683 if (!AllocatorConfig::getQuarantineDisabled())
684 Quarantine.disable();
685 Primary.disable();
686 Secondary.disable();
687 disableRingBuffer();
688 }
689
690 void enable() NO_THREAD_SAFETY_ANALYSIS {
691 initThreadMaybe();
692 enableRingBuffer();
693 Secondary.enable();
694 Primary.enable();
695 if (!AllocatorConfig::getQuarantineDisabled())
696 Quarantine.enable();
697 Stats.enable();
698 TSDRegistry.enable();
699#ifdef GWP_ASAN_HOOKS
700 GuardedAlloc.enable();
701#endif
702 }
703
704 // The function returns the amount of bytes required to store the statistics,
705 // which might be larger than the amount of bytes provided. Note that the
706 // statistics buffer is not necessarily constant between calls to this
707 // function. This can be called with a null buffer or zero size for buffer
708 // sizing purposes.
709 uptr getStats(char *Buffer, uptr Size) {
710 ScopedString Str;
711 const uptr Length = getStats(&Str) + 1;
712 if (Length < Size)
713 Size = Length;
714 if (Buffer && Size) {
715 memcpy(dest: Buffer, src: Str.data(), n: Size);
716 Buffer[Size - 1] = '\0';
717 }
718 return Length;
719 }
720
721 void printStats() {
722 ScopedString Str;
723 getStats(&Str);
724 Str.output();
725 }
726
727 void printFragmentationInfo() {
728 ScopedString Str;
729 Primary.getFragmentationInfo(&Str);
730 // Secondary allocator dumps the fragmentation data in getStats().
731 Str.output();
732 }
733
734 void releaseToOS(ReleaseToOS ReleaseType) {
735 initThreadMaybe();
736 SCUDO_SCOPED_TRACE(GetReleaseToOSTraceName(ReleaseType));
737 if (ReleaseType == ReleaseToOS::ForceAll)
738 drainCaches();
739 Primary.releaseToOS(ReleaseType);
740 Secondary.releaseToOS(ReleaseType);
741 }
742
743 // Iterate over all chunks and call a callback for all busy chunks located
744 // within the provided memory range. Said callback must not use this allocator
745 // or a deadlock can ensue. This fits Android's malloc_iterate() needs.
746 void iterateOverChunks(uptr Base, uptr Size, iterate_callback Callback,
747 void *Arg) {
748 initThreadMaybe();
749 if (archSupportsMemoryTagging())
750 Base = untagPointer(Ptr: Base);
751 const uptr From = Base;
752 const uptr To = Base + Size;
753 const Options Options = Primary.Options.load();
754 bool MayHaveTaggedPrimary = useMemoryTagging<AllocatorConfig>(Options);
755 auto Lambda = [this, From, To, MayHaveTaggedPrimary, Callback,
756 Arg](uptr Block) {
757 if (Block < From || Block >= To)
758 return;
759 uptr Chunk;
760 Chunk::UnpackedHeader Header;
761 if (UNLIKELY(MayHaveTaggedPrimary)) {
762 // A chunk header can either have a zero tag (tagged primary) or the
763 // header tag (secondary, or untagged primary). We don't know which so
764 // try both.
765 ScopedDisableMemoryTagChecks x;
766 if (!getChunkFromBlock(Block, Chunk: &Chunk, Header: &Header) &&
767 !getChunkFromBlock(Block: addHeaderTag(Block), Chunk: &Chunk, Header: &Header))
768 return;
769 } else if (!getChunkFromBlock(Block: addHeaderTag(Block), Chunk: &Chunk, Header: &Header)) {
770 return;
771 }
772
773 if (Header.State != Chunk::State::Allocated)
774 return;
775
776 uptr TaggedChunk = Chunk;
777 if (allocatorSupportsMemoryTagging<AllocatorConfig>())
778 TaggedChunk = untagPointer(Ptr: TaggedChunk);
779 uptr Size;
780 if (UNLIKELY(useMemoryTagging<AllocatorConfig>(Primary.Options.load()))) {
781 TaggedChunk = loadTag(Ptr: Chunk);
782 Size = getSize(Ptr: reinterpret_cast<void *>(Chunk), Header: &Header);
783 } else if (AllocatorConfig::getExactUsableSize()) {
784 Size = getSize(Ptr: reinterpret_cast<void *>(Chunk), Header: &Header);
785 } else {
786 Size = getUsableSize(reinterpret_cast<void *>(Chunk), &Header);
787 }
788 Callback(TaggedChunk, Size, Arg);
789 };
790 Primary.iterateOverBlocks(Lambda);
791 Secondary.iterateOverBlocks(Lambda);
792#ifdef GWP_ASAN_HOOKS
793 GuardedAlloc.iterate(Base: reinterpret_cast<void *>(Base), Size, Cb: Callback, Arg);
794#endif
795 }
796
797 bool canReturnNull() {
798 initThreadMaybe();
799 return Primary.Options.load().get(OptionBit::MayReturnNull);
800 }
801
802 bool setOption(Option O, sptr Value) {
803 initThreadMaybe();
804 if (O == Option::MemtagTuning) {
805 // Enabling odd/even tags involves a tradeoff between use-after-free
806 // detection and buffer overflow detection. Odd/even tags make it more
807 // likely for buffer overflows to be detected by increasing the size of
808 // the guaranteed "red zone" around the allocation, but on the other hand
809 // use-after-free is less likely to be detected because the tag space for
810 // any particular chunk is cut in half. Therefore we use this tuning
811 // setting to control whether odd/even tags are enabled.
812 if (Value == M_MEMTAG_TUNING_BUFFER_OVERFLOW)
813 Primary.Options.set(OptionBit::UseOddEvenTags);
814 else if (Value == M_MEMTAG_TUNING_UAF)
815 Primary.Options.clear(OptionBit::UseOddEvenTags);
816 return true;
817 } else {
818 // We leave it to the various sub-components to decide whether or not they
819 // want to handle the option, but we do not want to short-circuit
820 // execution if one of the setOption was to return false.
821 const bool PrimaryResult = Primary.setOption(O, Value);
822 const bool SecondaryResult = Secondary.setOption(O, Value);
823 const bool RegistryResult = TSDRegistry.setOption(O, Value);
824 return PrimaryResult && SecondaryResult && RegistryResult;
825 }
826 return false;
827 }
828
829 ALWAYS_INLINE uptr getUsableSize(const void *Ptr,
830 Chunk::UnpackedHeader *Header) {
831 void *BlockBegin = getBlockBegin(Ptr, Header);
832 if (LIKELY(Header->ClassId)) {
833 return SizeClassMap::getSizeByClassId(Header->ClassId) -
834 (reinterpret_cast<uptr>(Ptr) - reinterpret_cast<uptr>(BlockBegin));
835 }
836
837 uptr UntaggedPtr = reinterpret_cast<uptr>(Ptr);
838 if (allocatorSupportsMemoryTagging<AllocatorConfig>()) {
839 UntaggedPtr = untagPointer(Ptr: UntaggedPtr);
840 BlockBegin = untagPointer(Ptr: BlockBegin);
841 }
842 return SecondaryT::getBlockEnd(BlockBegin) - UntaggedPtr;
843 }
844
845 // Return the usable size for a given chunk. If MTE is enabled or if the
846 // ExactUsableSize config parameter is true, we report the exact size of
847 // the original allocation size. Otherwise, we will return the total
848 // actual usable size.
849 uptr getUsableSize(const void *Ptr) {
850 if (UNLIKELY(!Ptr))
851 return 0;
852
853 if (AllocatorConfig::getExactUsableSize() ||
854 UNLIKELY(useMemoryTagging<AllocatorConfig>(Primary.Options.load())))
855 return getAllocSize(Ptr);
856
857 initThreadMaybe();
858
859#ifdef GWP_ASAN_HOOKS
860 if (UNLIKELY(GuardedAlloc.pointerIsMine(Ptr)))
861 return GuardedAlloc.getSize(Ptr);
862#endif // GWP_ASAN_HOOKS
863
864 Ptr = getHeaderTaggedPointer(Ptr: const_cast<void *>(Ptr));
865 Chunk::UnpackedHeader Header;
866 Chunk::loadHeader(Cookie, Ptr, NewUnpackedHeader: &Header);
867
868 // Getting the alloc size of a chunk only makes sense if it's allocated.
869 if (UNLIKELY(Header.State != Chunk::State::Allocated))
870 reportInvalidChunkState(Action: AllocatorAction::Sizing, Ptr);
871
872 return getUsableSize(Ptr, &Header);
873 }
874
875 uptr getAllocSize(const void *Ptr) {
876 initThreadMaybe();
877
878#ifdef GWP_ASAN_HOOKS
879 if (UNLIKELY(GuardedAlloc.pointerIsMine(Ptr)))
880 return GuardedAlloc.getSize(Ptr);
881#endif // GWP_ASAN_HOOKS
882
883 Ptr = getHeaderTaggedPointer(Ptr: const_cast<void *>(Ptr));
884 Chunk::UnpackedHeader Header;
885 Chunk::loadHeader(Cookie, Ptr, NewUnpackedHeader: &Header);
886
887 // Getting the alloc size of a chunk only makes sense if it's allocated.
888 if (UNLIKELY(Header.State != Chunk::State::Allocated))
889 reportInvalidChunkState(Action: AllocatorAction::Sizing, Ptr);
890
891 return getSize(Ptr, Header: &Header);
892 }
893
894 void getStats(StatCounters S) {
895 initThreadMaybe();
896 Stats.get(S);
897 }
898
899 // Returns true if the pointer provided was allocated by the current
900 // allocator instance, which is compliant with tcmalloc's ownership concept.
901 // A corrupted chunk will not be reported as owned, which is WAI.
902 bool isOwned(const void *Ptr) {
903 initThreadMaybe();
904 // If the allocation is not owned, the tags could be wrong.
905 ScopedDisableMemoryTagChecks x(
906 useMemoryTagging<AllocatorConfig>(Primary.Options.load()));
907#ifdef GWP_ASAN_HOOKS
908 if (GuardedAlloc.pointerIsMine(Ptr))
909 return true;
910#endif // GWP_ASAN_HOOKS
911 if (!Ptr || !isAligned(X: reinterpret_cast<uptr>(Ptr), Alignment: MinAlignment))
912 return false;
913 Ptr = getHeaderTaggedPointer(Ptr: const_cast<void *>(Ptr));
914 Chunk::UnpackedHeader Header;
915 return Chunk::isValid(Cookie, Ptr, NewUnpackedHeader: &Header) &&
916 Header.State == Chunk::State::Allocated;
917 }
918
919 bool useMemoryTaggingTestOnly() const {
920 return useMemoryTagging<AllocatorConfig>(Primary.Options.load());
921 }
922 void disableMemoryTagging() {
923 // If we haven't been initialized yet, we need to initialize now in order to
924 // prevent a future call to initThreadMaybe() from enabling memory tagging
925 // based on feature detection. But don't call initThreadMaybe() because it
926 // may end up calling the allocator (via pthread_atfork, via the post-init
927 // callback), which may cause mappings to be created with memory tagging
928 // enabled.
929 TSDRegistry.initOnceMaybe(this);
930 if (allocatorSupportsMemoryTagging<AllocatorConfig>()) {
931 Secondary.disableMemoryTagging();
932 Primary.Options.clear(OptionBit::UseMemoryTagging);
933 }
934 }
935
936 void setTrackAllocationStacks(bool Track) {
937 initThreadMaybe();
938 if (getFlags()->allocation_ring_buffer_size <= 0) {
939 DCHECK(!Primary.Options.load().get(OptionBit::TrackAllocationStacks));
940 return;
941 }
942
943 if (Track) {
944 initRingBufferMaybe();
945 Primary.Options.set(OptionBit::TrackAllocationStacks);
946 } else
947 Primary.Options.clear(OptionBit::TrackAllocationStacks);
948 }
949
950 void setFillContents(FillContentsMode FillContents) {
951 initThreadMaybe();
952 Primary.Options.setFillContentsMode(FillContents);
953 }
954
955 void setAddLargeAllocationSlack(bool AddSlack) {
956 initThreadMaybe();
957 if (AddSlack)
958 Primary.Options.set(OptionBit::AddLargeAllocationSlack);
959 else
960 Primary.Options.clear(OptionBit::AddLargeAllocationSlack);
961 }
962
963 const char *getStackDepotAddress() {
964 initThreadMaybe();
965 AllocationRingBuffer *RB = getRingBuffer();
966 return RB ? reinterpret_cast<char *>(RB->Depot) : nullptr;
967 }
968
969 uptr getStackDepotSize() {
970 initThreadMaybe();
971 AllocationRingBuffer *RB = getRingBuffer();
972 return RB ? RB->StackDepotSize : 0;
973 }
974
975 const char *getRegionInfoArrayAddress() const {
976 return Primary.getRegionInfoArrayAddress();
977 }
978
979 static uptr getRegionInfoArraySize() {
980 return PrimaryT::getRegionInfoArraySize();
981 }
982
983 const char *getRingBufferAddress() {
984 initThreadMaybe();
985 return reinterpret_cast<char *>(getRingBuffer());
986 }
987
988 uptr getRingBufferSize() {
989 initThreadMaybe();
990 AllocationRingBuffer *RB = getRingBuffer();
991 return RB && RB->RingBufferElements
992 ? ringBufferSizeInBytes(RingBufferElements: RB->RingBufferElements)
993 : 0;
994 }
995
996 void getErrorInfo(uptr FaultAddr, size_t MinDistance, size_t MaxDistance,
997 scudo_error_report *Reports, size_t &ReportIndex) {
998 auto *RingBuffer = getRingBuffer();
999 if (RingBuffer == nullptr)
1000 return;
1001
1002 // No more room for any more error reports.
1003 if (ReportIndex == ScudoNumErrorReports)
1004 return;
1005
1006 uptr UntaggedFaultAddr = untagPointer(Ptr: FaultAddr);
1007 BlockInfo Info = Primary.findNearestBlock(UntaggedFaultAddr);
1008
1009 auto GetBlockInfo = [&](uptr Addr, Chunk::UnpackedHeader *Header,
1010 uptr &ChunkAddr) {
1011 uptr ChunkOffset = getChunkOffsetFromBlock(
1012 Block: reinterpret_cast<const char *>(loadTagUnaligned(Ptr: Addr)));
1013 ChunkAddr = loadTagUnaligned(Ptr: Addr + ChunkOffset);
1014 uptr HeaderAddr = loadTag(Ptr: Addr + ChunkOffset - Chunk::getHeaderSize());
1015 *Header = *reinterpret_cast<const Chunk::UnpackedHeader *>(HeaderAddr);
1016 };
1017
1018 auto CheckOOB = [&](uptr BlockAddr) {
1019 if (BlockAddr < Info.RegionBegin || BlockAddr >= Info.RegionEnd)
1020 return false;
1021
1022 uptr ChunkAddr;
1023 Chunk::UnpackedHeader Header;
1024 GetBlockInfo(BlockAddr, &Header, ChunkAddr);
1025 if (Header.State != Chunk::State::Allocated)
1026 return false;
1027 const u8 ChunkTag = extractTag(Ptr: ChunkAddr);
1028 if (extractTag(Ptr: FaultAddr) != ChunkTag) {
1029 // If the allocated size is zero, then there isn't a tag on this
1030 // pointer, so allow a tag mismatch.
1031 if (ChunkTag != 0 || Header.SizeOrUnusedBytes != 0)
1032 return false;
1033 }
1034 auto *Report = &Reports[ReportIndex++];
1035 uptr UntaggedChunkAddr = untagPointer(Ptr: ChunkAddr);
1036 const u32 *ChunkData = reinterpret_cast<const u32 *>(UntaggedChunkAddr);
1037 Report->error_type = UntaggedFaultAddr < UntaggedChunkAddr
1038 ? BUFFER_UNDERFLOW
1039 : BUFFER_OVERFLOW;
1040 Report->allocation_address = UntaggedChunkAddr;
1041 Report->allocation_size = Header.SizeOrUnusedBytes;
1042 if (RingBuffer->Depot) {
1043 const u32 *TracePtr = reinterpret_cast<const u32 *>(loadTagUnaligned(
1044 Ptr: reinterpret_cast<uptr>(&ChunkData[MemTagAllocationTraceIndex])));
1045 collectTraceMaybe(Depot: RingBuffer->Depot, Trace&: Report->allocation_trace,
1046 Hash: *TracePtr);
1047 }
1048 const u32 *TidPtr = reinterpret_cast<const u32 *>(loadTagUnaligned(
1049 Ptr: reinterpret_cast<uptr>(&ChunkData[MemTagAllocationTidIndex])));
1050 Report->allocation_tid = *TidPtr;
1051 return ReportIndex == ScudoNumErrorReports;
1052 };
1053
1054 if (MinDistance == 0 && CheckOOB(Info.BlockBegin))
1055 return;
1056
1057 for (size_t I = Max<size_t>(A: MinDistance, B: 1); I != MaxDistance; ++I)
1058 if (CheckOOB(Info.BlockBegin + I * Info.BlockSize) ||
1059 CheckOOB(Info.BlockBegin - I * Info.BlockSize))
1060 return;
1061 }
1062
1063 void getRingBufferErrorInfo(uintptr_t FaultAddr, scudo_error_report *Reports,
1064 size_t &ReportIndex) {
1065 auto *RingBuffer = getRingBuffer();
1066 if (RingBuffer == nullptr)
1067 return;
1068
1069 // No more room for any more error reports.
1070 if (ReportIndex == ScudoNumErrorReports)
1071 return;
1072
1073 uptr Pos = atomic_load_relaxed(&RingBuffer->Pos);
1074 if (Pos == 0)
1075 return;
1076
1077 const uptr RingBufferElements = RingBuffer->RingBufferElements;
1078
1079 // Pos is a value that is always increasing.
1080 uptr LastPos;
1081 if (Pos < RingBufferElements) {
1082 LastPos = 0;
1083 } else {
1084 LastPos = Pos - RingBufferElements;
1085 }
1086 for (uptr I = Pos; I > LastPos; --I) {
1087 auto *Entry =
1088 getRingBufferEntry(RingBuffer, (I - 1) % RingBufferElements);
1089 uptr EntryPtr = atomic_load_relaxed(&Entry->Ptr);
1090 if (!EntryPtr)
1091 continue;
1092
1093 uptr UntaggedEntryPtr = untagPointer(Ptr: EntryPtr);
1094 uptr EntrySize = atomic_load_relaxed(&Entry->AllocationSize);
1095 u32 AllocationTrace = atomic_load_relaxed(&Entry->AllocationTrace);
1096 u32 AllocationTid = atomic_load_relaxed(&Entry->AllocationTid);
1097 u32 DeallocationTrace = atomic_load_relaxed(&Entry->DeallocationTrace);
1098 u32 DeallocationTid = atomic_load_relaxed(&Entry->DeallocationTid);
1099 if (DeallocationTid) {
1100 // For UAF we only consider in-bounds fault addresses because
1101 // out-of-bounds UAF is rare and attempting to detect it is very likely
1102 // to result in false positives.
1103 if (FaultAddr < EntryPtr || FaultAddr >= EntryPtr + EntrySize)
1104 continue;
1105 } else {
1106 // Ring buffer OOB is only possible with secondary allocations. In this
1107 // case we are guaranteed a guard region of at least a page on either
1108 // side of the allocation (guard page on the right, guard page + tagged
1109 // region on the left), so ignore any faults outside of that range.
1110 if (FaultAddr < EntryPtr - getPageSizeCached() ||
1111 FaultAddr >= EntryPtr + EntrySize + getPageSizeCached())
1112 continue;
1113
1114 // For UAF the ring buffer will contain two entries, one for the
1115 // allocation and another for the deallocation. Don't report buffer
1116 // overflow/underflow using the allocation entry if we have already
1117 // collected a report from the deallocation entry.
1118 bool Found = false;
1119 for (uptr J = 0; J != ReportIndex; ++J) {
1120 if (Reports[J].allocation_address == UntaggedEntryPtr) {
1121 Found = true;
1122 break;
1123 }
1124 }
1125 if (Found)
1126 continue;
1127 }
1128
1129 auto *Report = &Reports[ReportIndex++];
1130 if (DeallocationTid)
1131 Report->error_type = USE_AFTER_FREE;
1132 else if (FaultAddr < EntryPtr)
1133 Report->error_type = BUFFER_UNDERFLOW;
1134 else
1135 Report->error_type = BUFFER_OVERFLOW;
1136
1137 Report->allocation_address = UntaggedEntryPtr;
1138 Report->allocation_size = EntrySize;
1139 collectTraceMaybe(Depot: RingBuffer->Depot, Trace&: Report->allocation_trace,
1140 Hash: AllocationTrace);
1141 Report->allocation_tid = AllocationTid;
1142 collectTraceMaybe(Depot: RingBuffer->Depot, Trace&: Report->deallocation_trace,
1143 Hash: DeallocationTrace);
1144 Report->deallocation_tid = DeallocationTid;
1145 if (ReportIndex == ScudoNumErrorReports) {
1146 // No more report entries.
1147 return;
1148 }
1149 }
1150 }
1151
1152 void getErrorInfo(uintptr_t FaultAddr, struct scudo_error_info *ErrorInfo) {
1153 const Options Options = Primary.Options.load();
1154 if (!useMemoryTagging<AllocatorConfig>(Options))
1155 return;
1156
1157 bool TagExists = extractTag(Ptr: FaultAddr) != 0;
1158 size_t ReportIndex = 0;
1159 if (TagExists) {
1160 // Check for OOB in the current block and the two surrounding blocks.
1161 // Beyond that, UAF is more likely.
1162 getErrorInfo(FaultAddr, 0, 2, &ErrorInfo->reports[0], ReportIndex);
1163 }
1164
1165 // Check the ring buffer. For primary allocations this will only find UAF;
1166 // for secondary allocations we can find either UAF or OOB.
1167 getRingBufferErrorInfo(FaultAddr, Reports: &ErrorInfo->reports[0], ReportIndex);
1168
1169 if (TagExists) {
1170 // Check for OOB in the 28 blocks surrounding the 3 we checked earlier.
1171 // Beyond that we are likely to hit false positives.
1172 getErrorInfo(FaultAddr, 2, 16, &ErrorInfo->reports[0], ReportIndex);
1173 }
1174 }
1175
1176 static void collectTraceMaybe(const StackDepot *Depot,
1177 uintptr_t (&Trace)[ScudoTraceSize], u32 Hash) {
1178 uptr RingPos, Size;
1179 if (!Depot->find(Hash, RingPosPtr: &RingPos, SizePtr: &Size))
1180 return;
1181 for (unsigned I = 0; I != Size && I != ScudoTraceSize; ++I)
1182 Trace[I] = static_cast<uintptr_t>(Depot->at(RingPos: RingPos + I));
1183 }
1184
1185 uptr getBlockBeginTestOnly(const void *Ptr) {
1186 Chunk::UnpackedHeader Header;
1187 Chunk::loadHeader(Cookie, Ptr, NewUnpackedHeader: &Header);
1188 DCHECK(Header.State == Chunk::State::Allocated);
1189
1190 if (allocatorSupportsMemoryTagging<AllocatorConfig>())
1191 Ptr = untagPointer(Ptr: const_cast<void *>(Ptr));
1192 void *Begin = getBlockBegin(Ptr, Header: &Header);
1193 if (allocatorSupportsMemoryTagging<AllocatorConfig>())
1194 Begin = untagPointer(Ptr: Begin);
1195 return reinterpret_cast<uptr>(Begin);
1196 }
1197
1198private:
1199 typedef typename PrimaryT::SizeClassMap SizeClassMap;
1200
1201 static const uptr MinAlignmentLog = SCUDO_MIN_ALIGNMENT_LOG;
1202 static const uptr MaxAlignmentLog = 24U; // 16 MB seems reasonable.
1203 static const uptr MinAlignment = 1UL << MinAlignmentLog;
1204 static const uptr MaxAlignment = 1UL << MaxAlignmentLog;
1205 static const uptr MaxAllowedMallocSize =
1206 FIRST_32_SECOND_64(1UL << 31, 1ULL << 40);
1207
1208 static_assert(MinAlignment >= sizeof(Chunk::PackedHeader),
1209 "Minimal alignment must at least cover a chunk header.");
1210 static_assert(!allocatorSupportsMemoryTagging<AllocatorConfig>() ||
1211 MinAlignment >= archMemoryTagGranuleSize(),
1212 "");
1213
1214 static const u32 BlockMarker = 0x44554353U;
1215
1216 // These are indexes into an "array" of 32-bit values that store information
1217 // inline with a chunk that is relevant to diagnosing memory tag faults, where
1218 // 0 corresponds to the address of the user memory. This means that only
1219 // negative indexes may be used. The smallest index that may be used is -2,
1220 // which corresponds to 8 bytes before the user memory, because the chunk
1221 // header size is 8 bytes and in allocators that support memory tagging the
1222 // minimum alignment is at least the tag granule size (16 on aarch64).
1223 static const sptr MemTagAllocationTraceIndex = -2;
1224 static const sptr MemTagAllocationTidIndex = -1;
1225
1226 u32 Cookie = 0;
1227 u32 QuarantineMaxChunkSize = 0;
1228#if SCUDO_FUCHSIA
1229 u32 ZeroOnDeallocMaxSize = 0;
1230#endif
1231
1232 GlobalStats Stats;
1233 PrimaryT Primary;
1234 SecondaryT Secondary;
1235 QuarantineT Quarantine;
1236 TSDRegistryT TSDRegistry;
1237 pthread_once_t PostInitNonce = PTHREAD_ONCE_INIT;
1238
1239#ifdef GWP_ASAN_HOOKS
1240 gwp_asan::GuardedPoolAllocator GuardedAlloc;
1241 uptr GuardedAllocSlotSize = 0;
1242#endif // GWP_ASAN_HOOKS
1243
1244 struct AllocationRingBuffer {
1245 struct Entry {
1246 atomic_uptr Ptr;
1247 atomic_uptr AllocationSize;
1248 atomic_u32 AllocationTrace;
1249 atomic_u32 AllocationTid;
1250 atomic_u32 DeallocationTrace;
1251 atomic_u32 DeallocationTid;
1252 };
1253 StackDepot *Depot = nullptr;
1254 uptr StackDepotSize = 0;
1255 MemMapT RawRingBufferMap;
1256 MemMapT RawStackDepotMap;
1257 u32 RingBufferElements = 0;
1258 atomic_uptr Pos;
1259 // An array of Size (at least one) elements of type Entry is immediately
1260 // following to this struct.
1261 };
1262 static_assert(sizeof(AllocationRingBuffer) %
1263 alignof(typename AllocationRingBuffer::Entry) ==
1264 0,
1265 "invalid alignment");
1266
1267 // Lock to initialize the RingBuffer
1268 HybridMutex RingBufferInitLock;
1269
1270 // Pointer to memory mapped area starting with AllocationRingBuffer struct,
1271 // and immediately followed by Size elements of type Entry.
1272 atomic_uptr RingBufferAddress = {};
1273
1274 AllocationRingBuffer *getRingBuffer() {
1275 return reinterpret_cast<AllocationRingBuffer *>(
1276 atomic_load(A: &RingBufferAddress, MO: memory_order_acquire));
1277 }
1278
1279 // The following might get optimized out by the compiler.
1280 NOINLINE void performSanityChecks() {
1281 // Verify that the header offset field can hold the maximum offset. In the
1282 // case of the Secondary allocator, it takes care of alignment and the
1283 // offset will always be small. In the case of the Primary, the worst case
1284 // scenario happens in the last size class, when the backend allocation
1285 // would already be aligned on the requested alignment, which would happen
1286 // to be the maximum alignment that would fit in that size class. As a
1287 // result, the maximum offset will be at most the maximum alignment for the
1288 // last size class minus the header size, in multiples of MinAlignment.
1289 Chunk::UnpackedHeader Header = {};
1290 const uptr MaxPrimaryAlignment = 1UL << getMostSignificantSetBitIndex(
1291 SizeClassMap::MaxSize - MinAlignment);
1292 const uptr MaxOffset =
1293 (MaxPrimaryAlignment - Chunk::getHeaderSize()) >> MinAlignmentLog;
1294 Header.Offset = MaxOffset & Chunk::OffsetMask;
1295 if (UNLIKELY(Header.Offset != MaxOffset))
1296 reportSanityCheckError(Field: "offset");
1297
1298 // Verify that we can fit the maximum size or amount of unused bytes in the
1299 // header. Given that the Secondary fits the allocation to a page, the worst
1300 // case scenario happens in the Primary. It will depend on the second to
1301 // last and last class sizes, as well as the dynamic base for the Primary.
1302 // The following is an over-approximation that works for our needs.
1303 const uptr MaxSizeOrUnusedBytes = SizeClassMap::MaxSize - 1;
1304 Header.SizeOrUnusedBytes = MaxSizeOrUnusedBytes;
1305 if (UNLIKELY(Header.SizeOrUnusedBytes != MaxSizeOrUnusedBytes))
1306 reportSanityCheckError(Field: "size (or unused bytes)");
1307
1308 const uptr LargestClassId = SizeClassMap::LargestClassId;
1309 Header.ClassId = LargestClassId;
1310 if (UNLIKELY(Header.ClassId != LargestClassId))
1311 reportSanityCheckError(Field: "class ID");
1312 }
1313
1314 static inline void *getBlockBegin(const void *Ptr,
1315 Chunk::UnpackedHeader *Header) {
1316 return reinterpret_cast<void *>(
1317 reinterpret_cast<uptr>(Ptr) - Chunk::getHeaderSize() -
1318 (static_cast<uptr>(Header->Offset) << MinAlignmentLog));
1319 }
1320
1321 // Return the size of a chunk as requested during its allocation.
1322 inline uptr getSize(const void *Ptr, Chunk::UnpackedHeader *Header) {
1323 const uptr SizeOrUnusedBytes = Header->SizeOrUnusedBytes;
1324 if (LIKELY(Header->ClassId))
1325 return SizeOrUnusedBytes;
1326 if (allocatorSupportsMemoryTagging<AllocatorConfig>())
1327 Ptr = untagPointer(Ptr: const_cast<void *>(Ptr));
1328 return SecondaryT::getBlockEnd(getBlockBegin(Ptr, Header)) -
1329 reinterpret_cast<uptr>(Ptr) - SizeOrUnusedBytes;
1330 }
1331
1332 ALWAYS_INLINE void *initChunk(const uptr ClassId, const Chunk::Origin Origin,
1333 void *Block, const uptr UserPtr,
1334 const uptr SizeOrUnusedBytes,
1335 const FillContentsMode FillContents) {
1336 // Compute the default pointer before adding the header tag
1337 const uptr DefaultAlignedPtr =
1338 reinterpret_cast<uptr>(Block) + Chunk::getHeaderSize();
1339
1340 Block = addHeaderTag(Block);
1341 // Only do content fill when it's from primary allocator because secondary
1342 // allocator has filled the content.
1343 if (ClassId != 0 && UNLIKELY(FillContents != NoFill)) {
1344 // This condition is not necessarily unlikely, but since memset is
1345 // costly, we might as well mark it as such.
1346 memset(Block, FillContents == ZeroFill ? 0 : PatternFillByte,
1347 PrimaryT::getSizeByClassId(ClassId));
1348 }
1349
1350 Chunk::UnpackedHeader Header = {};
1351
1352 if (UNLIKELY(DefaultAlignedPtr != UserPtr)) {
1353 const uptr Offset = UserPtr - DefaultAlignedPtr;
1354 DCHECK_GE(Offset, 2 * sizeof(u32));
1355 // The BlockMarker has no security purpose, but is specifically meant for
1356 // the chunk iteration function that can be used in debugging situations.
1357 // It is the only situation where we have to locate the start of a chunk
1358 // based on its block address.
1359 reinterpret_cast<u32 *>(Block)[0] = BlockMarker;
1360 reinterpret_cast<u32 *>(Block)[1] = static_cast<u32>(Offset);
1361 Header.Offset = (Offset >> MinAlignmentLog) & Chunk::OffsetMask;
1362 }
1363
1364 Header.ClassId = ClassId & Chunk::ClassIdMask;
1365 Header.State = Chunk::State::Allocated;
1366 Header.setOrigin(Origin);
1367 Header.SizeOrUnusedBytes = SizeOrUnusedBytes & Chunk::SizeOrUnusedBytesMask;
1368 Chunk::storeHeader(Cookie, Ptr: reinterpret_cast<void *>(addHeaderTag(UserPtr)),
1369 NewUnpackedHeader: &Header);
1370
1371 return reinterpret_cast<void *>(UserPtr);
1372 }
1373
1374 NOINLINE void *
1375 initChunkWithMemoryTagging(const uptr ClassId, const Chunk::Origin Origin,
1376 void *Block, const uptr UserPtr, const uptr Size,
1377 const uptr SizeOrUnusedBytes,
1378 const FillContentsMode FillContents) {
1379 const Options Options = Primary.Options.load();
1380 DCHECK(useMemoryTagging<AllocatorConfig>(Options));
1381
1382 // Compute the default pointer before adding the header tag
1383 const uptr DefaultAlignedPtr =
1384 reinterpret_cast<uptr>(Block) + Chunk::getHeaderSize();
1385
1386 void *Ptr = reinterpret_cast<void *>(UserPtr);
1387 void *TaggedPtr = Ptr;
1388
1389 if (LIKELY(ClassId)) {
1390 // Init the primary chunk.
1391 //
1392 // We only need to zero or tag the contents for Primary backed
1393 // allocations. We only set tags for primary allocations in order to avoid
1394 // faulting potentially large numbers of pages for large secondary
1395 // allocations. We assume that guard pages are enough to protect these
1396 // allocations.
1397 //
1398 // FIXME: When the kernel provides a way to set the background tag of a
1399 // mapping, we should be able to tag secondary allocations as well.
1400 //
1401 // When memory tagging is enabled, zeroing the contents is done as part of
1402 // setting the tag.
1403
1404 Chunk::UnpackedHeader Header;
1405 const uptr BlockSize = PrimaryT::getSizeByClassId(ClassId);
1406 const uptr BlockUptr = reinterpret_cast<uptr>(Block);
1407 const uptr BlockEnd = BlockUptr + BlockSize;
1408 // If possible, try to reuse the UAF tag that was set by deallocate().
1409 // For simplicity, only reuse tags if we have the same start address as
1410 // the previous allocation. This handles the majority of cases since
1411 // most allocations will not be more aligned than the minimum alignment.
1412 //
1413 // We need to handle situations involving reclaimed chunks, and retag
1414 // the reclaimed portions if necessary. In the case where the chunk is
1415 // fully reclaimed, the chunk's header will be zero, which will trigger
1416 // the code path for new mappings and invalid chunks that prepares the
1417 // chunk from scratch. There are three possibilities for partial
1418 // reclaiming:
1419 //
1420 // (1) Header was reclaimed, data was partially reclaimed.
1421 // (2) Header was not reclaimed, all data was reclaimed (e.g. because
1422 // data started on a page boundary).
1423 // (3) Header was not reclaimed, data was partially reclaimed.
1424 //
1425 // Case (1) will be handled in the same way as for full reclaiming,
1426 // since the header will be zero.
1427 //
1428 // We can detect case (2) by loading the tag from the start
1429 // of the chunk. If it is zero, it means that either all data was
1430 // reclaimed (since we never use zero as the chunk tag), or that the
1431 // previous allocation was of size zero. Either way, we need to prepare
1432 // a new chunk from scratch.
1433 //
1434 // We can detect case (3) by moving to the next page (if covered by the
1435 // chunk) and loading the tag of its first granule. If it is zero, it
1436 // means that all following pages may need to be retagged. On the other
1437 // hand, if it is nonzero, we can assume that all following pages are
1438 // still tagged, according to the logic that if any of the pages
1439 // following the next page were reclaimed, the next page would have been
1440 // reclaimed as well.
1441 uptr TaggedUserPtr;
1442 uptr PrevUserPtr;
1443 if (getChunkFromBlock(Block: BlockUptr, Chunk: &PrevUserPtr, Header: &Header) &&
1444 PrevUserPtr == UserPtr &&
1445 (TaggedUserPtr = loadTag(Ptr: UserPtr)) != UserPtr) {
1446 uptr PrevEnd = TaggedUserPtr + Header.SizeOrUnusedBytes;
1447 const uptr NextPage = roundUp(X: TaggedUserPtr, Boundary: getPageSizeCached());
1448 if (NextPage < PrevEnd && loadTag(Ptr: NextPage) != NextPage)
1449 PrevEnd = NextPage;
1450 TaggedPtr = reinterpret_cast<void *>(TaggedUserPtr);
1451 resizeTaggedChunk</*ClearAllTags*/ false>(PrevEnd, TaggedUserPtr + Size,
1452 Size, BlockEnd);
1453 if (UNLIKELY(FillContents != NoFill && !Header.OriginOrWasZeroed)) {
1454 // If an allocation needs to be zeroed (i.e. calloc) we can normally
1455 // avoid zeroing the memory now since we can rely on memory having
1456 // been zeroed on free, as this is normally done while setting the
1457 // UAF tag. But if tagging was disabled per-thread when the memory
1458 // was freed, it would not have been retagged and thus zeroed, and
1459 // therefore it needs to be zeroed now.
1460 memset(s: TaggedPtr, c: 0,
1461 n: Min(A: Size, B: roundUp(X: PrevEnd - TaggedUserPtr,
1462 Boundary: archMemoryTagGranuleSize())));
1463 } else if (Size) {
1464 // Clear any stack metadata that may have previously been stored in
1465 // the chunk data.
1466 memset(s: TaggedPtr, c: 0, n: archMemoryTagGranuleSize());
1467 }
1468 } else {
1469 const uptr OddEvenMask =
1470 computeOddEvenMaskForPointerMaybe(Options, Ptr: BlockUptr, ClassId);
1471 TaggedPtr = prepareTaggedChunk(Ptr, Size, ExcludeMask: OddEvenMask, BlockEnd);
1472 }
1473 storePrimaryAllocationStackMaybe(Options, Ptr);
1474 } else {
1475 // Init the secondary chunk.
1476
1477 Block = addHeaderTag(Block);
1478 Ptr = addHeaderTag(Ptr);
1479 storeTags(Begin: reinterpret_cast<uptr>(Block), End: reinterpret_cast<uptr>(Ptr));
1480 storeSecondaryAllocationStackMaybe(Options, Ptr, Size);
1481 }
1482
1483 Chunk::UnpackedHeader Header = {};
1484
1485 if (UNLIKELY(DefaultAlignedPtr != UserPtr)) {
1486 const uptr Offset = UserPtr - DefaultAlignedPtr;
1487 DCHECK_GE(Offset, 2 * sizeof(u32));
1488 // The BlockMarker has no security purpose, but is specifically meant for
1489 // the chunk iteration function that can be used in debugging situations.
1490 // It is the only situation where we have to locate the start of a chunk
1491 // based on its block address.
1492 reinterpret_cast<u32 *>(Block)[0] = BlockMarker;
1493 reinterpret_cast<u32 *>(Block)[1] = static_cast<u32>(Offset);
1494 Header.Offset = (Offset >> MinAlignmentLog) & Chunk::OffsetMask;
1495 }
1496
1497 Header.ClassId = ClassId & Chunk::ClassIdMask;
1498 Header.State = Chunk::State::Allocated;
1499 Header.setOrigin(Origin);
1500 Header.SizeOrUnusedBytes = SizeOrUnusedBytes & Chunk::SizeOrUnusedBytesMask;
1501 Chunk::storeHeader(Cookie, Ptr, NewUnpackedHeader: &Header);
1502
1503 return TaggedPtr;
1504 }
1505
1506 void quarantineOrDeallocateChunk(const Options &Options, void *TaggedPtr,
1507 Chunk::UnpackedHeader *Header,
1508 uptr Size) NO_THREAD_SAFETY_ANALYSIS {
1509 void *Ptr = getHeaderTaggedPointer(Ptr: TaggedPtr);
1510 // If the quarantine is disabled, the actual size of a chunk is 0 or larger
1511 // than the maximum allowed, we return a chunk directly to the backend.
1512 // This purposefully underflows for Size == 0.
1513 const bool BypassQuarantine = AllocatorConfig::getQuarantineDisabled() ||
1514 !Quarantine.getCacheSize() ||
1515 ((Size - 1) >= QuarantineMaxChunkSize) ||
1516 !Header->ClassId;
1517 if (BypassQuarantine)
1518 Header->State = Chunk::State::Available;
1519 else
1520 Header->State = Chunk::State::Quarantined;
1521
1522 if (LIKELY(!useMemoryTagging<AllocatorConfig>(Options)))
1523 Header->OriginOrWasZeroed = 0U;
1524 else {
1525 Header->OriginOrWasZeroed =
1526 Header->ClassId && !TSDRegistry.getDisableMemInit();
1527 }
1528
1529 Chunk::storeHeader(Cookie, Ptr, NewUnpackedHeader: Header);
1530
1531 if (BypassQuarantine) {
1532 void *BlockBegin;
1533 if (LIKELY(!useMemoryTagging<AllocatorConfig>(Options))) {
1534 // Must do this after storeHeader because loadHeader uses a tagged ptr.
1535 if (allocatorSupportsMemoryTagging<AllocatorConfig>())
1536 Ptr = untagPointer(Ptr);
1537 BlockBegin = getBlockBegin(Ptr, Header);
1538 } else {
1539 BlockBegin = retagBlock(Options, TaggedPtr, Ptr, Header, Size, BypassQuarantine: true);
1540 }
1541
1542#if SCUDO_FUCHSIA
1543 if (AllocatorConfig::getEnableZeroOnDealloc()) {
1544 // Clearing the header is incompatible with quarantine and tagging.
1545 // Hence, it is fine to implement it only when quarantine is bypassed.
1546 DCHECK(!useMemoryTagging<AllocatorConfig>(Options));
1547 uptr length = reinterpret_cast<uptr>(Ptr) + Size -
1548 reinterpret_cast<uptr>(BlockBegin);
1549 if (length <= ZeroOnDeallocMaxSize)
1550 memset(BlockBegin, 0, length);
1551 }
1552#endif // SCUDO_FUCHSIA
1553
1554 const uptr ClassId = Header->ClassId;
1555 if (LIKELY(ClassId)) {
1556 bool CacheDrained;
1557 {
1558 typename TSDRegistryT::ScopedTSD TSD(TSDRegistry);
1559 CacheDrained =
1560 TSD->getSizeClassAllocator().deallocate(ClassId, BlockBegin);
1561 }
1562 // When we have drained some blocks back to the Primary from TSD, that
1563 // implies that we may have the chance to release some pages as well.
1564 // Note that in order not to block other thread's accessing the TSD,
1565 // release the TSD first then try the page release.
1566 if (CacheDrained)
1567 Primary.tryReleaseToOS(ClassId, ReleaseToOS::Normal);
1568 } else {
1569 Secondary.deallocate(Options, BlockBegin);
1570 }
1571 } else {
1572 if (UNLIKELY(useMemoryTagging<AllocatorConfig>(Options)))
1573 retagBlock(Options, TaggedPtr, Ptr, Header, Size, BypassQuarantine: false);
1574 typename TSDRegistryT::ScopedTSD TSD(TSDRegistry);
1575 Quarantine.put(&TSD->getQuarantineCache(),
1576 QuarantineCallback(*this, TSD->getSizeClassAllocator()),
1577 Ptr, Size);
1578 }
1579 }
1580
1581 NOINLINE void *retagBlock(const Options &Options, void *TaggedPtr, void *&Ptr,
1582 Chunk::UnpackedHeader *Header, const uptr Size,
1583 bool BypassQuarantine) {
1584 DCHECK(useMemoryTagging<AllocatorConfig>(Options));
1585
1586 const u8 PrevTag = extractTag(Ptr: reinterpret_cast<uptr>(TaggedPtr));
1587 storeDeallocationStackMaybe(Options, Ptr, PrevTag, Size);
1588 if (Header->ClassId && !TSDRegistry.getDisableMemInit()) {
1589 uptr TaggedBegin, TaggedEnd;
1590 const uptr OddEvenMask = computeOddEvenMaskForPointerMaybe(
1591 Options, Ptr: reinterpret_cast<uptr>(getBlockBegin(Ptr, Header)),
1592 ClassId: Header->ClassId);
1593 // Exclude the previous tag so that immediate use after free is
1594 // detected 100% of the time.
1595 setRandomTag(Ptr, Size, ExcludeMask: OddEvenMask | (1UL << PrevTag), TaggedBegin: &TaggedBegin,
1596 TaggedEnd: &TaggedEnd);
1597 }
1598
1599 Ptr = untagPointer(Ptr);
1600 void *BlockBegin = getBlockBegin(Ptr, Header);
1601 if (BypassQuarantine && !Header->ClassId) {
1602 storeTags(Begin: reinterpret_cast<uptr>(BlockBegin),
1603 End: reinterpret_cast<uptr>(Ptr));
1604 }
1605
1606 return BlockBegin;
1607 }
1608
1609 bool getChunkFromBlock(uptr Block, uptr *Chunk,
1610 Chunk::UnpackedHeader *Header) {
1611 *Chunk =
1612 Block + getChunkOffsetFromBlock(Block: reinterpret_cast<const char *>(Block));
1613 return Chunk::isValid(Cookie, Ptr: reinterpret_cast<void *>(*Chunk), NewUnpackedHeader: Header);
1614 }
1615
1616 static uptr getChunkOffsetFromBlock(const char *Block) {
1617 u32 Offset = 0;
1618 if (reinterpret_cast<const u32 *>(Block)[0] == BlockMarker)
1619 Offset = reinterpret_cast<const u32 *>(Block)[1];
1620 return Offset + Chunk::getHeaderSize();
1621 }
1622
1623 // Set the tag of the granule past the end of the allocation to 0, to catch
1624 // linear overflows even if a previous larger allocation used the same block
1625 // and tag. Only do this if the granule past the end is in our block, because
1626 // this would otherwise lead to a SEGV if the allocation covers the entire
1627 // block and our block is at the end of a mapping. The tag of the next block's
1628 // header granule will be set to 0, so it will serve the purpose of catching
1629 // linear overflows in this case.
1630 //
1631 // For allocations of size 0 we do not end up storing the address tag to the
1632 // memory tag space, which getInlineErrorInfo() normally relies on to match
1633 // address tags against chunks. To allow matching in this case we store the
1634 // address tag in the first byte of the chunk.
1635 void storeEndMarker(uptr End, uptr Size, uptr BlockEnd) {
1636 DCHECK_EQ(BlockEnd, untagPointer(BlockEnd));
1637 uptr UntaggedEnd = untagPointer(Ptr: End);
1638 if (UntaggedEnd != BlockEnd) {
1639 storeTag(Ptr: UntaggedEnd);
1640 if (Size == 0)
1641 *reinterpret_cast<u8 *>(UntaggedEnd) = extractTag(Ptr: End);
1642 }
1643 }
1644
1645 void *prepareTaggedChunk(void *Ptr, uptr Size, uptr ExcludeMask,
1646 uptr BlockEnd) {
1647 // Prepare the granule before the chunk to store the chunk header by setting
1648 // its tag to 0. Normally its tag will already be 0, but in the case where a
1649 // chunk holding a low alignment allocation is reused for a higher alignment
1650 // allocation, the chunk may already have a non-zero tag from the previous
1651 // allocation.
1652 storeTag(Ptr: reinterpret_cast<uptr>(Ptr) - archMemoryTagGranuleSize());
1653
1654 uptr TaggedBegin, TaggedEnd;
1655 setRandomTag(Ptr, Size, ExcludeMask, TaggedBegin: &TaggedBegin, TaggedEnd: &TaggedEnd);
1656
1657 storeEndMarker(End: TaggedEnd, Size, BlockEnd);
1658 return reinterpret_cast<void *>(TaggedBegin);
1659 }
1660
1661 template <bool ClearAllTags>
1662 void resizeTaggedChunk(uptr OldPtr, uptr NewPtr, uptr NewSize,
1663 uptr BlockEnd) {
1664 uptr RoundOldPtr = roundUp(X: OldPtr, Boundary: archMemoryTagGranuleSize());
1665 uptr RoundNewPtr;
1666 if (RoundOldPtr >= NewPtr) {
1667 RoundNewPtr = roundUp(X: NewPtr, Boundary: archMemoryTagGranuleSize());
1668 if (ClearAllTags) {
1669 // Shrinking resize, set the tag to zero for the rest of the allocation.
1670 // This prevents use after realloc of the excess memory.
1671 storeTags(Begin: untagPointer(Ptr: RoundNewPtr), End: untagPointer(Ptr: RoundOldPtr));
1672 } else {
1673 // Add a zero tag right after the pointer to prevent overflow.
1674 storeEndMarker(End: RoundNewPtr, Size: NewSize, BlockEnd);
1675 }
1676 } else {
1677 // Set the memory tag of the region
1678 // [RoundOldPtr, roundUp(NewPtr, archMemoryTagGranuleSize()))
1679 // to the pointer tag stored in OldPtr.
1680 RoundNewPtr = storeTags(Begin: RoundOldPtr, End: NewPtr);
1681 // Add a zero tag right after the pointer to prevent overflow.
1682 storeEndMarker(End: RoundNewPtr, Size: NewSize, BlockEnd);
1683 }
1684 }
1685
1686 void storePrimaryAllocationStackMaybe(const Options &Options, void *Ptr) {
1687 if (!UNLIKELY(Options.get(OptionBit::TrackAllocationStacks)))
1688 return;
1689 AllocationRingBuffer *RB = getRingBuffer();
1690 if (!RB)
1691 return;
1692 auto *Ptr32 = reinterpret_cast<u32 *>(Ptr);
1693 Ptr32[MemTagAllocationTraceIndex] = collectStackTrace(Depot: RB->Depot);
1694 Ptr32[MemTagAllocationTidIndex] = getThreadID();
1695 }
1696
1697 void storeRingBufferEntry(AllocationRingBuffer *RB, void *Ptr,
1698 u32 AllocationTrace, u32 AllocationTid,
1699 uptr AllocationSize, u32 DeallocationTrace,
1700 u32 DeallocationTid) {
1701 uptr Pos = atomic_fetch_add(&RB->Pos, 1, memory_order_relaxed);
1702 typename AllocationRingBuffer::Entry *Entry =
1703 getRingBufferEntry(RB, Pos % RB->RingBufferElements);
1704
1705 // First invalidate our entry so that we don't attempt to interpret a
1706 // partially written state in getSecondaryErrorInfo(). The fences below
1707 // ensure that the compiler does not move the stores to Ptr in between the
1708 // stores to the other fields.
1709 atomic_store_relaxed(&Entry->Ptr, 0);
1710
1711 __atomic_signal_fence(__ATOMIC_SEQ_CST);
1712 atomic_store_relaxed(&Entry->AllocationTrace, AllocationTrace);
1713 atomic_store_relaxed(&Entry->AllocationTid, AllocationTid);
1714 atomic_store_relaxed(&Entry->AllocationSize, AllocationSize);
1715 atomic_store_relaxed(&Entry->DeallocationTrace, DeallocationTrace);
1716 atomic_store_relaxed(&Entry->DeallocationTid, DeallocationTid);
1717 __atomic_signal_fence(__ATOMIC_SEQ_CST);
1718
1719 atomic_store_relaxed(&Entry->Ptr, reinterpret_cast<uptr>(Ptr));
1720 }
1721
1722 void storeSecondaryAllocationStackMaybe(const Options &Options, void *Ptr,
1723 uptr Size) {
1724 if (!UNLIKELY(Options.get(OptionBit::TrackAllocationStacks)))
1725 return;
1726 AllocationRingBuffer *RB = getRingBuffer();
1727 if (!RB)
1728 return;
1729 u32 Trace = collectStackTrace(Depot: RB->Depot);
1730 u32 Tid = getThreadID();
1731
1732 auto *Ptr32 = reinterpret_cast<u32 *>(Ptr);
1733 Ptr32[MemTagAllocationTraceIndex] = Trace;
1734 Ptr32[MemTagAllocationTidIndex] = Tid;
1735
1736 storeRingBufferEntry(RB, Ptr: untagPointer(Ptr), AllocationTrace: Trace, AllocationTid: Tid, AllocationSize: Size, DeallocationTrace: 0, DeallocationTid: 0);
1737 }
1738
1739 void storeDeallocationStackMaybe(const Options &Options, void *Ptr,
1740 u8 PrevTag, uptr Size) {
1741 if (!UNLIKELY(Options.get(OptionBit::TrackAllocationStacks)))
1742 return;
1743 AllocationRingBuffer *RB = getRingBuffer();
1744 if (!RB)
1745 return;
1746 auto *Ptr32 = reinterpret_cast<u32 *>(Ptr);
1747 u32 AllocationTrace = Ptr32[MemTagAllocationTraceIndex];
1748 u32 AllocationTid = Ptr32[MemTagAllocationTidIndex];
1749
1750 u32 DeallocationTrace = collectStackTrace(Depot: RB->Depot);
1751 u32 DeallocationTid = getThreadID();
1752
1753 storeRingBufferEntry(RB, Ptr: addFixedTag(Ptr: untagPointer(Ptr), Tag: PrevTag),
1754 AllocationTrace, AllocationTid, AllocationSize: Size,
1755 DeallocationTrace, DeallocationTid);
1756 }
1757
1758 uptr getStats(ScopedString *Str) {
1759 Str->append(Format: "Config Stats Base: ");
1760 AllocatorConfig::getConfigValues(Str);
1761 Primary.getStats(Str);
1762 Secondary.getStats(Str);
1763 if (!AllocatorConfig::getQuarantineDisabled())
1764 Quarantine.getStats(Str);
1765 TSDRegistry.getStats(Str);
1766 return Str->length();
1767 }
1768
1769 static typename AllocationRingBuffer::Entry *
1770 getRingBufferEntry(AllocationRingBuffer *RB, uptr N) {
1771 char *RBEntryStart =
1772 &reinterpret_cast<char *>(RB)[sizeof(AllocationRingBuffer)];
1773 return &reinterpret_cast<typename AllocationRingBuffer::Entry *>(
1774 RBEntryStart)[N];
1775 }
1776 static const typename AllocationRingBuffer::Entry *
1777 getRingBufferEntry(const AllocationRingBuffer *RB, uptr N) {
1778 const char *RBEntryStart =
1779 &reinterpret_cast<const char *>(RB)[sizeof(AllocationRingBuffer)];
1780 return &reinterpret_cast<const typename AllocationRingBuffer::Entry *>(
1781 RBEntryStart)[N];
1782 }
1783
1784 void initRingBufferMaybe() {
1785 ScopedLock L(RingBufferInitLock);
1786 if (getRingBuffer() != nullptr)
1787 return;
1788
1789 int ring_buffer_size = getFlags()->allocation_ring_buffer_size;
1790 if (ring_buffer_size <= 0)
1791 return;
1792
1793 u32 AllocationRingBufferSize = static_cast<u32>(ring_buffer_size);
1794
1795 // We store alloc and free stacks for each entry.
1796 constexpr u32 kStacksPerRingBufferEntry = 2;
1797 constexpr u32 kMaxU32Pow2 = ~(UINT32_MAX >> 1);
1798 static_assert(isPowerOfTwo(X: kMaxU32Pow2));
1799 // On Android we always have 3 frames at the bottom: __start_main,
1800 // __libc_init, main, and 3 at the top: malloc, scudo_malloc and
1801 // Allocator::allocate. This leaves 10 frames for the user app. The next
1802 // smallest power of two (8) would only leave 2, which is clearly too
1803 // little.
1804 constexpr u32 kFramesPerStack = 16;
1805 static_assert(isPowerOfTwo(X: kFramesPerStack));
1806
1807 if (AllocationRingBufferSize > kMaxU32Pow2 / kStacksPerRingBufferEntry)
1808 return;
1809 u32 TabSize = static_cast<u32>(roundUpPowerOfTwo(Size: kStacksPerRingBufferEntry *
1810 AllocationRingBufferSize));
1811 if (TabSize > UINT32_MAX / kFramesPerStack)
1812 return;
1813 u32 RingSize = static_cast<u32>(TabSize * kFramesPerStack);
1814
1815 uptr StackDepotSize = sizeof(StackDepot) + sizeof(atomic_u64) * RingSize +
1816 sizeof(atomic_u32) * TabSize;
1817 MemMapT DepotMap;
1818 DepotMap.map(
1819 /*Addr=*/Addr: 0U, Size: roundUp(X: StackDepotSize, Boundary: getPageSizeCached()),
1820 Name: "scudo:stack_depot");
1821 auto *Depot = reinterpret_cast<StackDepot *>(DepotMap.getBase());
1822 Depot->init(RingSz: RingSize, TabSz: TabSize);
1823
1824 MemMapT MemMap;
1825 MemMap.map(
1826 /*Addr=*/Addr: 0U,
1827 Size: roundUp(X: ringBufferSizeInBytes(RingBufferElements: AllocationRingBufferSize),
1828 Boundary: getPageSizeCached()),
1829 Name: "scudo:ring_buffer");
1830 auto *RB = reinterpret_cast<AllocationRingBuffer *>(MemMap.getBase());
1831 RB->RawRingBufferMap = MemMap;
1832 RB->RingBufferElements = AllocationRingBufferSize;
1833 RB->Depot = Depot;
1834 RB->StackDepotSize = StackDepotSize;
1835 RB->RawStackDepotMap = DepotMap;
1836
1837 atomic_store(A: &RingBufferAddress, V: reinterpret_cast<uptr>(RB),
1838 MO: memory_order_release);
1839 }
1840
1841 void unmapRingBuffer() {
1842 AllocationRingBuffer *RB = getRingBuffer();
1843 if (RB == nullptr)
1844 return;
1845 // N.B. because RawStackDepotMap is part of RawRingBufferMap, the order
1846 // is very important.
1847 RB->RawStackDepotMap.unmap();
1848 // Note that the `RB->RawRingBufferMap` is stored on the pages managed by
1849 // itself. Take over the ownership before calling unmap() so that any
1850 // operation along with unmap() won't touch inaccessible pages.
1851 MemMapT RawRingBufferMap = RB->RawRingBufferMap;
1852 RawRingBufferMap.unmap();
1853 atomic_store(A: &RingBufferAddress, V: 0, MO: memory_order_release);
1854 }
1855
1856 static constexpr size_t ringBufferSizeInBytes(u32 RingBufferElements) {
1857 return sizeof(AllocationRingBuffer) +
1858 RingBufferElements * sizeof(typename AllocationRingBuffer::Entry);
1859 }
1860
1861 static constexpr size_t ringBufferElementsFromBytes(size_t Bytes) {
1862 if (Bytes < sizeof(AllocationRingBuffer)) {
1863 return 0;
1864 }
1865 return (Bytes - sizeof(AllocationRingBuffer)) /
1866 sizeof(typename AllocationRingBuffer::Entry);
1867 }
1868};
1869
1870} // namespace scudo
1871
1872#endif // SCUDO_COMBINED_H_
1873