1//===-- combined.h ----------------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#ifndef SCUDO_COMBINED_H_
10#define SCUDO_COMBINED_H_
11
12#include "allocator_config_wrapper.h"
13#include "atomic_helpers.h"
14#include "chunk.h"
15#include "common.h"
16#include "flags.h"
17#include "flags_parser.h"
18#include "mem_map.h"
19#include "memtag.h"
20#include "mutex.h"
21#include "options.h"
22#include "quarantine.h"
23#include "report.h"
24#include "secondary.h"
25#include "size_class_allocator.h"
26#include "stack_depot.h"
27#include "string_utils.h"
28#include "tracing.h"
29#include "tsd.h"
30
31#include "scudo/interface.h"
32
33#ifdef GWP_ASAN_HOOKS
34#include "gwp_asan/guarded_pool_allocator.h"
35#include "gwp_asan/optional/backtrace.h"
36#include "gwp_asan/optional/segv_handler.h"
37#endif // GWP_ASAN_HOOKS
38
39extern "C" inline void EmptyCallback() {}
40
41#ifdef HAVE_ANDROID_UNSAFE_FRAME_POINTER_CHASE
42// This function is not part of the NDK so it does not appear in any public
43// header files. We only declare/use it when targeting the platform.
44extern "C" size_t android_unsafe_frame_pointer_chase(scudo::uptr *buf,
45 size_t num_entries);
46#endif
47
48namespace scudo {
49
50template <class Config, void (*PostInitCallback)(void) = EmptyCallback>
51class Allocator {
52public:
53 using AllocatorConfig = BaseConfig<Config>;
54 using PrimaryT =
55 typename AllocatorConfig::template PrimaryT<PrimaryConfig<Config>>;
56 using SecondaryT =
57 typename AllocatorConfig::template SecondaryT<SecondaryConfig<Config>>;
58 using SizeClassAllocatorT = typename PrimaryT::SizeClassAllocatorT;
59 typedef Allocator<Config, PostInitCallback> ThisT;
60 typedef typename AllocatorConfig::template TSDRegistryT<ThisT> TSDRegistryT;
61
62 void callPostInitCallback() {
63 pthread_once(once_control: &PostInitNonce, init_routine: PostInitCallback);
64 }
65
66 struct QuarantineCallback {
67 explicit QuarantineCallback(ThisT &Instance,
68 SizeClassAllocatorT &SizeClassAllocator)
69 : Allocator(Instance), SizeClassAllocator(SizeClassAllocator) {}
70
71 // Chunk recycling function, returns a quarantined chunk to the backend,
72 // first making sure it hasn't been tampered with.
73 void recycle(void *Ptr) {
74 Chunk::UnpackedHeader Header;
75 Chunk::loadHeader(Cookie: Allocator.Cookie, Ptr, NewUnpackedHeader: &Header);
76 if (UNLIKELY(Header.State != Chunk::State::Quarantined))
77 reportInvalidChunkState(Action: AllocatorAction::Recycling, Ptr);
78
79 Header.State = Chunk::State::Available;
80 Chunk::storeHeader(Cookie: Allocator.Cookie, Ptr, NewUnpackedHeader: &Header);
81
82 if (allocatorSupportsMemoryTagging<AllocatorConfig>())
83 Ptr = untagPointer(Ptr);
84 void *BlockBegin = Allocator::getBlockBegin(Ptr, Header: &Header);
85 SizeClassAllocator.deallocate(Header.ClassId, BlockBegin);
86 }
87
88 // We take a shortcut when allocating a quarantine batch by working with the
89 // appropriate class ID instead of using Size. The compiler should optimize
90 // the class ID computation and work with the associated cache directly.
91 void *allocate(UNUSED uptr Size) {
92 const uptr QuarantineClassId = SizeClassMap::getClassIdBySize(
93 sizeof(QuarantineBatch) + Chunk::getHeaderSize());
94 void *Ptr = SizeClassAllocator.allocate(QuarantineClassId);
95 // Quarantine batch allocation failure is fatal.
96 if (UNLIKELY(!Ptr))
97 reportOutOfMemory(SizeClassMap::getSizeByClassId(QuarantineClassId));
98
99 Ptr = reinterpret_cast<void *>(reinterpret_cast<uptr>(Ptr) +
100 Chunk::getHeaderSize());
101 Chunk::UnpackedHeader Header = {};
102 Header.ClassId = QuarantineClassId & Chunk::ClassIdMask;
103 Header.SizeOrUnusedBytes = sizeof(QuarantineBatch);
104 Header.State = Chunk::State::Quarantined;
105 Chunk::storeHeader(Cookie: Allocator.Cookie, Ptr, NewUnpackedHeader: &Header);
106
107 // Reset tag to 0 as this chunk may have been previously used for a tagged
108 // user allocation.
109 if (UNLIKELY(useMemoryTagging<AllocatorConfig>(
110 Allocator.Primary.Options.load())))
111 storeTags(Begin: reinterpret_cast<uptr>(Ptr),
112 End: reinterpret_cast<uptr>(Ptr) + sizeof(QuarantineBatch));
113
114 return Ptr;
115 }
116
117 void deallocate(void *Ptr) {
118 const uptr QuarantineClassId = SizeClassMap::getClassIdBySize(
119 sizeof(QuarantineBatch) + Chunk::getHeaderSize());
120 Chunk::UnpackedHeader Header;
121 Chunk::loadHeader(Cookie: Allocator.Cookie, Ptr, NewUnpackedHeader: &Header);
122
123 if (UNLIKELY(Header.State != Chunk::State::Quarantined))
124 reportInvalidChunkState(Action: AllocatorAction::Deallocating, Ptr);
125 DCHECK_EQ(Header.ClassId, QuarantineClassId);
126 DCHECK_EQ(Header.Offset, 0);
127 DCHECK_EQ(Header.SizeOrUnusedBytes, sizeof(QuarantineBatch));
128
129 Header.State = Chunk::State::Available;
130 Chunk::storeHeader(Cookie: Allocator.Cookie, Ptr, NewUnpackedHeader: &Header);
131 SizeClassAllocator.deallocate(
132 QuarantineClassId,
133 reinterpret_cast<void *>(reinterpret_cast<uptr>(Ptr) -
134 Chunk::getHeaderSize()));
135 }
136
137 private:
138 ThisT &Allocator;
139 SizeClassAllocatorT &SizeClassAllocator;
140 };
141
142 typedef GlobalQuarantine<QuarantineCallback, void> QuarantineT;
143 typedef typename QuarantineT::CacheT QuarantineCacheT;
144
145 void init() {
146 // Make sure that the page size is initialized if it's not a constant.
147 CHECK_NE(getPageSizeCached(), 0U);
148
149 performSanityChecks();
150
151 // Check if hardware CRC32 is supported in the binary and by the platform,
152 // if so, opt for the CRC32 hardware version of the checksum.
153 if (&computeHardwareCRC32 && hasHardwareCRC32())
154 HashAlgorithm = Checksum::HardwareCRC32;
155
156 if (UNLIKELY(!getRandom(&Cookie, sizeof(Cookie))))
157 Cookie = static_cast<u32>(getMonotonicTime() ^
158 (reinterpret_cast<uptr>(this) >> 4));
159
160 initFlags();
161 reportUnrecognizedFlags();
162
163 // Store some flags locally.
164 if (getFlags()->may_return_null)
165 Primary.Options.set(OptionBit::MayReturnNull);
166 if (getFlags()->zero_contents)
167 Primary.Options.setFillContentsMode(ZeroFill);
168 else if (getFlags()->pattern_fill_contents)
169 Primary.Options.setFillContentsMode(PatternOrZeroFill);
170 if (getFlags()->dealloc_type_mismatch)
171 Primary.Options.set(OptionBit::DeallocTypeMismatch);
172 if (getFlags()->dealloc_align_mismatch)
173 Primary.Options.set(OptionBit::DeallocAlignMismatch);
174 if (getFlags()->delete_size_mismatch)
175 Primary.Options.set(OptionBit::DeleteSizeMismatch);
176 if (systemSupportsMemoryTagging())
177 Primary.Options.set(OptionBit::UseMemoryTagging);
178
179 QuarantineMaxChunkSize =
180 static_cast<u32>(getFlags()->quarantine_max_chunk_size);
181#if SCUDO_FUCHSIA
182 ZeroOnDeallocMaxSize =
183 static_cast<u32>(getFlags()->zero_on_dealloc_max_size);
184#endif
185
186 Stats.init();
187 // TODO(chiahungduan): Given that we support setting the default value in
188 // the PrimaryConfig and CacheConfig, consider to deprecate the use of
189 // `release_to_os_interval_ms` flag.
190 const s32 ReleaseToOsIntervalMs = getFlags()->release_to_os_interval_ms;
191 Primary.init(ReleaseToOsIntervalMs);
192 Secondary.init(&Stats, ReleaseToOsIntervalMs);
193 if (!AllocatorConfig::getQuarantineDisabled()) {
194 Quarantine.init(
195 static_cast<uptr>(getFlags()->quarantine_size_kb << 10),
196 static_cast<uptr>(getFlags()->thread_local_quarantine_size_kb << 10));
197 }
198 }
199
200 void enableRingBuffer() NO_THREAD_SAFETY_ANALYSIS {
201 AllocationRingBuffer *RB = getRingBuffer();
202 if (RB)
203 RB->Depot->enable();
204 RingBufferInitLock.unlock();
205 }
206
207 void disableRingBuffer() NO_THREAD_SAFETY_ANALYSIS {
208 RingBufferInitLock.lock();
209 AllocationRingBuffer *RB = getRingBuffer();
210 if (RB)
211 RB->Depot->disable();
212 }
213
214 // Initialize the embedded GWP-ASan instance. Requires the main allocator to
215 // be functional, best called from PostInitCallback.
216 void initGwpAsan() {
217#ifdef GWP_ASAN_HOOKS
218 gwp_asan::options::Options Opt;
219 Opt.Enabled = getFlags()->GWP_ASAN_Enabled;
220 Opt.MaxSimultaneousAllocations =
221 getFlags()->GWP_ASAN_MaxSimultaneousAllocations;
222 Opt.SampleRate = getFlags()->GWP_ASAN_SampleRate;
223 Opt.InstallSignalHandlers = getFlags()->GWP_ASAN_InstallSignalHandlers;
224 Opt.Recoverable = getFlags()->GWP_ASAN_Recoverable;
225 // Embedded GWP-ASan is locked through the Scudo atfork handler (via
226 // Allocator::disable calling GWPASan.disable). Disable GWP-ASan's atfork
227 // handler.
228 Opt.InstallForkHandlers = false;
229 Opt.Backtrace = gwp_asan::backtrace::getBacktraceFunction();
230 GuardedAlloc.init(Opts: Opt);
231
232 if (Opt.InstallSignalHandlers)
233 gwp_asan::segv_handler::installSignalHandlers(
234 GPA: &GuardedAlloc, Printf,
235 PrintBacktrace: gwp_asan::backtrace::getPrintBacktraceFunction(),
236 SegvBacktrace: gwp_asan::backtrace::getSegvBacktraceFunction(),
237 Recoverable: Opt.Recoverable);
238
239 GuardedAllocSlotSize =
240 GuardedAlloc.getAllocatorState()->maximumAllocationSize();
241 Stats.add(I: StatFree, V: static_cast<uptr>(Opt.MaxSimultaneousAllocations) *
242 GuardedAllocSlotSize);
243#endif // GWP_ASAN_HOOKS
244 }
245
246#ifdef GWP_ASAN_HOOKS
247 const gwp_asan::AllocationMetadata *getGwpAsanAllocationMetadata() {
248 return GuardedAlloc.getMetadataRegion();
249 }
250
251 const gwp_asan::AllocatorState *getGwpAsanAllocatorState() {
252 return GuardedAlloc.getAllocatorState();
253 }
254#endif // GWP_ASAN_HOOKS
255
256 ALWAYS_INLINE void initThreadMaybe(bool MinimalInit = false) {
257 TSDRegistry.initThreadMaybe(this, MinimalInit);
258 }
259
260 void unmapTestOnly() {
261 unmapRingBuffer();
262 TSDRegistry.unmapTestOnly(this);
263 Primary.unmapTestOnly();
264 Secondary.unmapTestOnly();
265#ifdef GWP_ASAN_HOOKS
266 if (getFlags()->GWP_ASAN_InstallSignalHandlers)
267 gwp_asan::segv_handler::uninstallSignalHandlers();
268 GuardedAlloc.uninitTestOnly();
269#endif // GWP_ASAN_HOOKS
270 }
271
272 TSDRegistryT *getTSDRegistry() { return &TSDRegistry; }
273 QuarantineT *getQuarantine() { return &Quarantine; }
274
275 // The Cache must be provided zero-initialized.
276 void initAllocator(SizeClassAllocatorT *SizeClassAllocator) {
277 SizeClassAllocator->init(&Stats, &Primary);
278 }
279
280 // Release the resources used by a TSD, which involves:
281 // - draining the local quarantine cache to the global quarantine;
282 // - releasing the cached pointers back to the Primary;
283 // - unlinking the local stats from the global ones (destroying the cache does
284 // the last two items).
285 void commitBack(TSD<ThisT> *TSD) {
286 TSD->assertLocked(/*BypassCheck=*/true);
287 if (!AllocatorConfig::getQuarantineDisabled()) {
288 Quarantine.drain(&TSD->getQuarantineCache(),
289 QuarantineCallback(*this, TSD->getSizeClassAllocator()));
290 }
291 TSD->getSizeClassAllocator().destroy(&Stats);
292 }
293
294 void drainCache(TSD<ThisT> *TSD) {
295 TSD->assertLocked(/*BypassCheck=*/true);
296 if (!AllocatorConfig::getQuarantineDisabled()) {
297 Quarantine.drainAndRecycle(
298 &TSD->getQuarantineCache(),
299 QuarantineCallback(*this, TSD->getSizeClassAllocator()));
300 }
301 TSD->getSizeClassAllocator().drain();
302 }
303 void drainCaches() { TSDRegistry.drainCaches(this); }
304
305 ALWAYS_INLINE void *getHeaderTaggedPointer(void *Ptr) {
306 if (!allocatorSupportsMemoryTagging<AllocatorConfig>())
307 return Ptr;
308 auto UntaggedPtr = untagPointer(Ptr);
309 if (UntaggedPtr != Ptr)
310 return UntaggedPtr;
311 // Secondary, or pointer allocated while memory tagging is unsupported or
312 // disabled. The tag mismatch is okay in the latter case because tags will
313 // not be checked.
314 return addHeaderTag(Ptr);
315 }
316
317 ALWAYS_INLINE uptr addHeaderTag(uptr Ptr) {
318 if (!allocatorSupportsMemoryTagging<AllocatorConfig>())
319 return Ptr;
320 return addFixedTag(Ptr, Tag: 2);
321 }
322
323 ALWAYS_INLINE void *addHeaderTag(void *Ptr) {
324 return reinterpret_cast<void *>(addHeaderTag(reinterpret_cast<uptr>(Ptr)));
325 }
326
327 NOINLINE u32 collectStackTrace(UNUSED StackDepot *Depot) {
328#ifdef HAVE_ANDROID_UNSAFE_FRAME_POINTER_CHASE
329 // Discard collectStackTrace() frame and allocator function frame.
330 constexpr uptr DiscardFrames = 2;
331 uptr Stack[MaxTraceSize + DiscardFrames];
332 uptr Size =
333 android_unsafe_frame_pointer_chase(Stack, MaxTraceSize + DiscardFrames);
334 Size = Min<uptr>(Size, MaxTraceSize + DiscardFrames);
335 return Depot->insert(Stack + Min<uptr>(DiscardFrames, Size), Stack + Size);
336#else
337 return 0;
338#endif
339 }
340
341 uptr computeOddEvenMaskForPointerMaybe(const Options &Options, uptr Ptr,
342 uptr ClassId) {
343 if (!Options.get(Opt: OptionBit::UseOddEvenTags))
344 return 0;
345
346 // If a chunk's tag is odd, we want the tags of the surrounding blocks to be
347 // even, and vice versa. Blocks are laid out Size bytes apart, and adding
348 // Size to Ptr will flip the least significant set bit of Size in Ptr, so
349 // that bit will have the pattern 010101... for consecutive blocks, which we
350 // can use to determine which tag mask to use.
351 return 0x5555U << ((Ptr >> SizeClassMap::getSizeLSBByClassId(ClassId)) & 1);
352 }
353
354 NOINLINE void *allocate(uptr Size, Chunk::Origin Origin,
355 uptr Alignment = MinAlignment,
356 bool ZeroContents = false) NO_THREAD_SAFETY_ANALYSIS {
357 initThreadMaybe();
358
359 const Options Options = Primary.Options.load();
360 if (UNLIKELY(Alignment > MaxAlignment)) {
361 if (Options.get(Opt: OptionBit::MayReturnNull))
362 return nullptr;
363 reportAlignmentTooBig(Alignment, MaxAlignment);
364 }
365 if (Alignment < MinAlignment)
366 Alignment = MinAlignment;
367
368#ifdef GWP_ASAN_HOOKS
369 if (UNLIKELY(GuardedAlloc.shouldSample())) {
370 if (void *Ptr = GuardedAlloc.allocate(Size, Alignment)) {
371 Stats.lock();
372 Stats.add(I: StatAllocated, V: GuardedAllocSlotSize);
373 Stats.sub(I: StatFree, V: GuardedAllocSlotSize);
374 Stats.unlock();
375 return Ptr;
376 }
377 }
378#endif // GWP_ASAN_HOOKS
379
380 const FillContentsMode FillContents = ZeroContents ? ZeroFill
381 : TSDRegistry.getDisableMemInit()
382 ? NoFill
383 : Options.getFillContentsMode();
384
385 // If the requested size happens to be 0 (more common than you might think),
386 // allocate MinAlignment bytes on top of the header. Then add the extra
387 // bytes required to fulfill the alignment requirements: we allocate enough
388 // to be sure that there will be an address in the block that will satisfy
389 // the alignment.
390 const uptr NeededSize =
391 roundUp(X: Size, Boundary: MinAlignment) +
392 ((Alignment > MinAlignment) ? Alignment : Chunk::getHeaderSize());
393
394 // Takes care of extravagantly large sizes as well as integer overflows.
395 static_assert(MaxAllowedMallocSize < UINTPTR_MAX - MaxAlignment, "");
396 if (UNLIKELY(Size >= MaxAllowedMallocSize)) {
397 if (Options.get(Opt: OptionBit::MayReturnNull))
398 return nullptr;
399 reportAllocationSizeTooBig(UserSize: Size, TotalSize: NeededSize, MaxSize: MaxAllowedMallocSize);
400 }
401 DCHECK_LE(Size, NeededSize);
402
403 void *Block = nullptr;
404 uptr ClassId = 0;
405 uptr SecondaryBlockEnd = 0;
406 if (LIKELY(PrimaryT::canAllocate(NeededSize))) {
407 ClassId = SizeClassMap::getClassIdBySize(NeededSize);
408 DCHECK_NE(ClassId, 0U);
409 typename TSDRegistryT::ScopedTSD TSD(TSDRegistry);
410 Block = TSD->getSizeClassAllocator().allocate(ClassId);
411 // If the allocation failed, retry in each successively larger class until
412 // it fits. If it fails to fit in the largest class, fallback to the
413 // Secondary.
414 if (UNLIKELY(!Block)) {
415 while (ClassId < SizeClassMap::LargestClassId && !Block)
416 Block = TSD->getSizeClassAllocator().allocate(++ClassId);
417 if (!Block)
418 ClassId = 0;
419 }
420 }
421 if (UNLIKELY(ClassId == 0)) {
422 Block = Secondary.allocate(Options, Size, Alignment, &SecondaryBlockEnd,
423 FillContents);
424 }
425
426 if (UNLIKELY(!Block)) {
427 if (Options.get(Opt: OptionBit::MayReturnNull))
428 return nullptr;
429 printStats();
430 reportOutOfMemory(RequestedSize: NeededSize);
431 }
432
433 const uptr UserPtr = roundUp(
434 X: reinterpret_cast<uptr>(Block) + Chunk::getHeaderSize(), Boundary: Alignment);
435 const uptr SizeOrUnusedBytes =
436 ClassId ? Size : SecondaryBlockEnd - (UserPtr + Size);
437
438 if (LIKELY(!useMemoryTagging<AllocatorConfig>(Options))) {
439 return initChunk(ClassId, Origin, Block, UserPtr, SizeOrUnusedBytes,
440 FillContents);
441 }
442
443 return initChunkWithMemoryTagging(ClassId, Origin, Block, UserPtr, Size,
444 SizeOrUnusedBytes, FillContents);
445 }
446
447 ALWAYS_INLINE void deallocate(void *Ptr, Chunk::Origin Origin) {
448 deallocate(Ptr, Origin, /*DeleteSize=*/0, /*DeleteAlignment=*/0);
449 }
450
451 ALWAYS_INLINE void deallocateSized(void *Ptr, Chunk::Origin Origin,
452 uptr DeleteSize) {
453 deallocate(Ptr, Origin | Chunk::Origin::Size, DeleteSize,
454 /*DeleteAlignment=*/0);
455 }
456
457 ALWAYS_INLINE void deallocateSizedAligned(void *Ptr, Chunk::Origin Origin,
458 uptr DeleteSize,
459 uptr DeleteAlignment) {
460 deallocate(Ptr, Origin | Chunk::Origin::Size | Chunk::Origin::Align,
461 DeleteSize, DeleteAlignment);
462 }
463
464 ALWAYS_INLINE void deallocateAligned(void *Ptr, Chunk::Origin Origin,
465 uptr DeleteAlignment) {
466 deallocate(Ptr, Origin | Chunk::Origin::Align,
467 /*DeleteSize=*/0, /*DeleteAlignment=*/DeleteAlignment);
468 }
469
470 ALWAYS_INLINE void checkSizeMatch(const void *Ptr,
471 Chunk::UnpackedHeader *Header, uptr Size,
472 uptr DeallocSize) {
473 if (AllocatorConfig::getExactUsableSize()) {
474 if (DeallocSize != Size)
475 reportDeleteSizeMismatch(Ptr, Size: DeallocSize, ExpectedSize: Size);
476 } else if (DeallocSize != Size && DeallocSize != getUsableSize(Ptr, Header))
477 reportDeleteSizeMismatch(Ptr, DeallocSize, Size,
478 getUsableSize(Ptr, Header));
479 }
480
481 ALWAYS_INLINE void checkTypeMatch(AllocatorAction Action, const void *Ptr,
482 u8 AllocOrigin, u8 DeallocOrigin) {
483 if (UNLIKELY(Chunk::originBaseType(AllocOrigin) !=
484 Chunk::originBaseType(DeallocOrigin)))
485 reportDeallocTypeMismatch(Action, Ptr, AllocOrigin, DeallocOrigin);
486
487 // There is no way to store that a new/new [] did an aligned allocate,
488 // so skip that part of the verification.
489 if (UNLIKELY(AllocOrigin == Chunk::Origin::New ||
490 AllocOrigin == Chunk::Origin::NewArray))
491 return;
492
493 if (Chunk::originAligned(Origin: AllocOrigin)) {
494 // Only disallow an aligned allocation and a non-aligned deallocation
495 // if this is a realloc.
496 if (Action == AllocatorAction::Reallocating &&
497 !Chunk::originAligned(Origin: DeallocOrigin))
498 reportDeallocTypeMismatch(Action, Ptr, AllocOrigin, DeallocOrigin);
499 } else if (Chunk::originAligned(Origin: DeallocOrigin)) {
500 // Origin not aligned, dealloc aligned.
501 reportDeallocTypeMismatch(Action, Ptr, AllocOrigin, DeallocOrigin);
502 }
503 }
504
505 NOINLINE void deallocate(void *Ptr, u8 DeallocOrigin, uptr DeleteSize,
506 uptr DeleteAlignment) {
507 if (UNLIKELY(!Ptr))
508 return;
509
510 // For a deallocation, we only ensure minimal initialization, meaning thread
511 // local data will be left uninitialized for now (when using ELF TLS). The
512 // fallback cache will be used instead. This is a workaround for a situation
513 // where the only heap operation performed in a thread would be a free past
514 // the TLS destructors, ending up in initialized thread specific data never
515 // being destroyed properly. Any other heap operation will do a full init.
516 initThreadMaybe(/*MinimalInit=*/MinimalInit: true);
517
518#ifdef GWP_ASAN_HOOKS
519 if (UNLIKELY(GuardedAlloc.pointerIsMine(Ptr))) {
520 GuardedAlloc.deallocate(Ptr);
521 Stats.lock();
522 Stats.add(I: StatFree, V: GuardedAllocSlotSize);
523 Stats.sub(I: StatAllocated, V: GuardedAllocSlotSize);
524 Stats.unlock();
525 return;
526 }
527#endif // GWP_ASAN_HOOKS
528
529 if (UNLIKELY(!isAligned(reinterpret_cast<uptr>(Ptr), MinAlignment)))
530 reportMisalignedPointer(Action: AllocatorAction::Deallocating, Ptr);
531
532 if (UNLIKELY(Chunk::originAligned(DeallocOrigin) &&
533 !isPowerOfTwo(DeleteAlignment)))
534 reportAlignmentNotPowerOfTwo(Alignment: DeleteAlignment);
535
536 void *TaggedPtr = Ptr;
537 Ptr = getHeaderTaggedPointer(Ptr);
538
539 Chunk::UnpackedHeader Header;
540 Chunk::loadHeader(Cookie, Ptr, NewUnpackedHeader: &Header);
541
542 if (UNLIKELY(Header.State != Chunk::State::Allocated))
543 reportInvalidChunkState(Action: AllocatorAction::Deallocating, Ptr);
544
545 const Options Options = Primary.Options.load();
546 const uptr Size = getSize(Ptr, Header: &Header);
547 if (AllocatorConfig::getAbortOnDeallocSizeMismatch() &&
548 Chunk::originSized(Origin: DeallocOrigin) &&
549 Options.get(Opt: OptionBit::DeleteSizeMismatch))
550 checkSizeMatch(Ptr, Header: &Header, Size, DeallocSize: DeleteSize);
551
552 if (AllocatorConfig::getAbortOnDeallocTypeMismatch() &&
553 Options.get(Opt: OptionBit::DeallocTypeMismatch))
554 checkTypeMatch(Action: AllocatorAction::Deallocating, Ptr, AllocOrigin: Header.getOrigin(),
555 DeallocOrigin);
556
557 if (UNLIKELY(AllocatorConfig::getAbortOnDeallocAlignmentMismatch() &&
558 Chunk::originAligned(DeallocOrigin) &&
559 Options.get(OptionBit::DeallocAlignMismatch) &&
560 !isAligned(reinterpret_cast<uptr>(Ptr), DeleteAlignment)))
561 reportDeleteAlignmentMismatch(Ptr, Alignment: DeleteAlignment);
562
563 quarantineOrDeallocateChunk(Options, TaggedPtr, Header: &Header, Size);
564 }
565
566 void *reallocate(void *OldPtr, uptr NewSize, uptr Alignment = MinAlignment) {
567 initThreadMaybe();
568
569 const Options Options = Primary.Options.load();
570 if (UNLIKELY(NewSize >= MaxAllowedMallocSize)) {
571 if (Options.get(Opt: OptionBit::MayReturnNull))
572 return nullptr;
573 reportAllocationSizeTooBig(UserSize: NewSize, TotalSize: 0, MaxSize: MaxAllowedMallocSize);
574 }
575
576 // The following cases are handled by the C wrappers.
577 DCHECK_NE(OldPtr, nullptr);
578 DCHECK_NE(NewSize, 0);
579
580#ifdef GWP_ASAN_HOOKS
581 if (UNLIKELY(GuardedAlloc.pointerIsMine(OldPtr))) {
582 uptr OldSize = GuardedAlloc.getSize(Ptr: OldPtr);
583 void *NewPtr = allocate(Size: NewSize, Origin: Chunk::Origin::Malloc, Alignment);
584 if (NewPtr)
585 memcpy(dest: NewPtr, src: OldPtr, n: (NewSize < OldSize) ? NewSize : OldSize);
586 GuardedAlloc.deallocate(Ptr: OldPtr);
587 Stats.lock();
588 Stats.add(I: StatFree, V: GuardedAllocSlotSize);
589 Stats.sub(I: StatAllocated, V: GuardedAllocSlotSize);
590 Stats.unlock();
591 return NewPtr;
592 }
593#endif // GWP_ASAN_HOOKS
594
595 void *OldTaggedPtr = OldPtr;
596 OldPtr = getHeaderTaggedPointer(Ptr: OldPtr);
597
598 if (UNLIKELY(!isAligned(reinterpret_cast<uptr>(OldPtr), MinAlignment)))
599 reportMisalignedPointer(Action: AllocatorAction::Reallocating, Ptr: OldPtr);
600
601 Chunk::UnpackedHeader Header;
602 Chunk::loadHeader(Cookie, Ptr: OldPtr, NewUnpackedHeader: &Header);
603
604 if (UNLIKELY(Header.State != Chunk::State::Allocated))
605 reportInvalidChunkState(Action: AllocatorAction::Reallocating, Ptr: OldPtr);
606
607 // Pointer has to be allocated with a malloc-type function. Some
608 // applications think that it is OK to realloc a memalign'ed pointer, which
609 // will trigger this check. It really isn't.
610 if (AllocatorConfig::getAbortOnDeallocTypeMismatch() &&
611 Options.get(Opt: OptionBit::DeallocTypeMismatch))
612 checkTypeMatch(Action: AllocatorAction::Reallocating, Ptr: OldPtr, AllocOrigin: Header.getOrigin(),
613 DeallocOrigin: Chunk::Origin::Malloc);
614
615 void *BlockBegin = getBlockBegin(Ptr: OldTaggedPtr, Header: &Header);
616 uptr BlockEnd;
617 bool ExactSize = AllocatorConfig::getExactUsableSize() ||
618 useMemoryTagging<AllocatorConfig>(Options);
619 const uptr ClassId = Header.ClassId;
620 uptr OldSize;
621 if (LIKELY(ClassId)) {
622 BlockEnd = reinterpret_cast<uptr>(BlockBegin) +
623 SizeClassMap::getSizeByClassId(ClassId);
624 if (ExactSize)
625 OldSize = Header.SizeOrUnusedBytes;
626 else
627 OldSize = BlockEnd - reinterpret_cast<uptr>(OldTaggedPtr);
628 } else {
629 BlockEnd = SecondaryT::getBlockEnd(BlockBegin);
630 OldSize = BlockEnd - reinterpret_cast<uptr>(OldTaggedPtr);
631 if (ExactSize)
632 OldSize -= Header.SizeOrUnusedBytes;
633 }
634 // If the new chunk still fits in the previously allocated block (with a
635 // reasonable delta), we just keep the old block, and update the chunk
636 // header to reflect the size change.
637 if (reinterpret_cast<uptr>(OldTaggedPtr) + NewSize <= BlockEnd) {
638 if (NewSize > OldSize || (OldSize - NewSize) < getPageSizeCached()) {
639 // If we have reduced the size, set the extra bytes to the fill value
640 // so that we are ready to grow it again in the future.
641 if (NewSize < OldSize) {
642 const FillContentsMode FillContents =
643 TSDRegistry.getDisableMemInit() ? NoFill
644 : Options.getFillContentsMode();
645 if (FillContents != NoFill) {
646 memset(s: reinterpret_cast<char *>(OldTaggedPtr) + NewSize,
647 c: FillContents == ZeroFill ? 0 : PatternFillByte,
648 n: OldSize - NewSize);
649 }
650 }
651
652 Header.SizeOrUnusedBytes =
653 (ClassId ? NewSize
654 : BlockEnd -
655 (reinterpret_cast<uptr>(OldTaggedPtr) + NewSize)) &
656 Chunk::SizeOrUnusedBytesMask;
657 Chunk::storeHeader(Cookie, Ptr: OldPtr, NewUnpackedHeader: &Header);
658 if (UNLIKELY(useMemoryTagging<AllocatorConfig>(Options))) {
659 if (ClassId) {
660 resizeTaggedChunk(OldPtr: reinterpret_cast<uptr>(OldTaggedPtr) + OldSize,
661 NewPtr: reinterpret_cast<uptr>(OldTaggedPtr) + NewSize,
662 NewSize, BlockEnd: untagPointer(Ptr: BlockEnd));
663 storePrimaryAllocationStackMaybe(Options, Ptr: OldPtr);
664 } else {
665 storeSecondaryAllocationStackMaybe(Options, Ptr: OldPtr, Size: NewSize);
666 }
667 }
668 return OldTaggedPtr;
669 }
670 }
671
672 // Otherwise we allocate a new one, and deallocate the old one. Some
673 // allocators will allocate an even larger chunk (by a fixed factor) to
674 // allow for potential further in-place realloc. The gains of such a trick
675 // are currently unclear.
676 void *NewPtr = allocate(Size: NewSize, Origin: Chunk::Origin::Malloc, Alignment);
677 if (LIKELY(NewPtr)) {
678 memcpy(dest: NewPtr, src: OldTaggedPtr, n: Min(A: NewSize, B: OldSize));
679 quarantineOrDeallocateChunk(Options, TaggedPtr: OldTaggedPtr, Header: &Header, Size: OldSize);
680 }
681 return NewPtr;
682 }
683
684 // TODO(kostyak): disable() is currently best-effort. There are some small
685 // windows of time when an allocation could still succeed after
686 // this function finishes. We will revisit that later.
687 void disable() NO_THREAD_SAFETY_ANALYSIS {
688 initThreadMaybe();
689#ifdef GWP_ASAN_HOOKS
690 GuardedAlloc.disable();
691#endif
692 TSDRegistry.disable();
693 Stats.disable();
694 if (!AllocatorConfig::getQuarantineDisabled())
695 Quarantine.disable();
696 Primary.disable();
697 Secondary.disable();
698 disableRingBuffer();
699 }
700
701 void enable() NO_THREAD_SAFETY_ANALYSIS {
702 initThreadMaybe();
703 enableRingBuffer();
704 Secondary.enable();
705 Primary.enable();
706 if (!AllocatorConfig::getQuarantineDisabled())
707 Quarantine.enable();
708 Stats.enable();
709 TSDRegistry.enable();
710#ifdef GWP_ASAN_HOOKS
711 GuardedAlloc.enable();
712#endif
713 }
714
715 // The function returns the amount of bytes required to store the statistics,
716 // which might be larger than the amount of bytes provided. Note that the
717 // statistics buffer is not necessarily constant between calls to this
718 // function. This can be called with a null buffer or zero size for buffer
719 // sizing purposes.
720 uptr getStats(char *Buffer, uptr Size) {
721 ScopedString Str;
722 const uptr Length = getStats(&Str) + 1;
723 if (Length < Size)
724 Size = Length;
725 if (Buffer && Size) {
726 memcpy(dest: Buffer, src: Str.data(), n: Size);
727 Buffer[Size - 1] = '\0';
728 }
729 return Length;
730 }
731
732 void printStats() {
733 ScopedString Str;
734 getStats(&Str);
735 Str.output();
736 }
737
738 void printFragmentationInfo() {
739 ScopedString Str;
740 Primary.getFragmentationInfo(&Str);
741 // Secondary allocator dumps the fragmentation data in getStats().
742 Str.output();
743 }
744
745 void releaseToOS(ReleaseToOS ReleaseType) {
746 initThreadMaybe();
747 SCUDO_SCOPED_TRACE(GetReleaseToOSTraceName(ReleaseType));
748 if (ReleaseType == ReleaseToOS::ForceAll)
749 drainCaches();
750 Primary.releaseToOS(ReleaseType);
751 Secondary.releaseToOS(ReleaseType);
752 }
753
754 // Iterate over all chunks and call a callback for all busy chunks located
755 // within the provided memory range. Said callback must not use this allocator
756 // or a deadlock can ensue. This fits Android's malloc_iterate() needs.
757 void iterateOverChunks(uptr Base, uptr Size, iterate_callback Callback,
758 void *Arg) {
759 initThreadMaybe();
760 if (archSupportsMemoryTagging())
761 Base = untagPointer(Ptr: Base);
762 const uptr From = Base;
763 const uptr To = Base + Size;
764 const Options Options = Primary.Options.load();
765 bool MayHaveTaggedPrimary = useMemoryTagging<AllocatorConfig>(Options);
766 auto Lambda = [this, From, To, MayHaveTaggedPrimary, Callback,
767 Arg](uptr Block) {
768 if (Block < From || Block >= To)
769 return;
770 uptr Chunk;
771 Chunk::UnpackedHeader Header;
772 if (UNLIKELY(MayHaveTaggedPrimary)) {
773 // A chunk header can either have a zero tag (tagged primary) or the
774 // header tag (secondary, or untagged primary). We don't know which so
775 // try both.
776 ScopedDisableMemoryTagChecks x;
777 if (!getChunkFromBlock(Block, Chunk: &Chunk, Header: &Header) &&
778 !getChunkFromBlock(Block: addHeaderTag(Block), Chunk: &Chunk, Header: &Header))
779 return;
780 } else if (!getChunkFromBlock(Block: addHeaderTag(Block), Chunk: &Chunk, Header: &Header)) {
781 return;
782 }
783
784 if (Header.State != Chunk::State::Allocated)
785 return;
786
787 uptr TaggedChunk = Chunk;
788 if (allocatorSupportsMemoryTagging<AllocatorConfig>())
789 TaggedChunk = untagPointer(Ptr: TaggedChunk);
790 uptr Size;
791 if (UNLIKELY(useMemoryTagging<AllocatorConfig>(Primary.Options.load()))) {
792 TaggedChunk = loadTag(Ptr: Chunk);
793 Size = getSize(Ptr: reinterpret_cast<void *>(Chunk), Header: &Header);
794 } else if (AllocatorConfig::getExactUsableSize()) {
795 Size = getSize(Ptr: reinterpret_cast<void *>(Chunk), Header: &Header);
796 } else {
797 Size = getUsableSize(reinterpret_cast<void *>(Chunk), &Header);
798 }
799 Callback(TaggedChunk, Size, Arg);
800 };
801 Primary.iterateOverBlocks(Lambda);
802 Secondary.iterateOverBlocks(Lambda);
803#ifdef GWP_ASAN_HOOKS
804 GuardedAlloc.iterate(Base: reinterpret_cast<void *>(Base), Size, Cb: Callback, Arg);
805#endif
806 }
807
808 bool canReturnNull() {
809 initThreadMaybe();
810 return Primary.Options.load().get(OptionBit::MayReturnNull);
811 }
812
813 bool setOption(Option O, sptr Value) {
814 initThreadMaybe();
815 if (O == Option::MemtagTuning) {
816 // Enabling odd/even tags involves a tradeoff between use-after-free
817 // detection and buffer overflow detection. Odd/even tags make it more
818 // likely for buffer overflows to be detected by increasing the size of
819 // the guaranteed "red zone" around the allocation, but on the other hand
820 // use-after-free is less likely to be detected because the tag space for
821 // any particular chunk is cut in half. Therefore we use this tuning
822 // setting to control whether odd/even tags are enabled.
823 if (Value == M_MEMTAG_TUNING_BUFFER_OVERFLOW)
824 Primary.Options.set(OptionBit::UseOddEvenTags);
825 else if (Value == M_MEMTAG_TUNING_UAF)
826 Primary.Options.clear(OptionBit::UseOddEvenTags);
827 return true;
828 } else {
829 // We leave it to the various sub-components to decide whether or not they
830 // want to handle the option, but we do not want to short-circuit
831 // execution if one of the setOption was to return false.
832 const bool PrimaryResult = Primary.setOption(O, Value);
833 const bool SecondaryResult = Secondary.setOption(O, Value);
834 const bool RegistryResult = TSDRegistry.setOption(O, Value);
835 return PrimaryResult && SecondaryResult && RegistryResult;
836 }
837 return false;
838 }
839
840 ALWAYS_INLINE uptr getUsableSize(const void *Ptr,
841 Chunk::UnpackedHeader *Header) {
842 void *BlockBegin = getBlockBegin(Ptr, Header);
843 if (LIKELY(Header->ClassId)) {
844 return SizeClassMap::getSizeByClassId(Header->ClassId) -
845 (reinterpret_cast<uptr>(Ptr) - reinterpret_cast<uptr>(BlockBegin));
846 }
847
848 uptr UntaggedPtr = reinterpret_cast<uptr>(Ptr);
849 if (allocatorSupportsMemoryTagging<AllocatorConfig>()) {
850 UntaggedPtr = untagPointer(Ptr: UntaggedPtr);
851 BlockBegin = untagPointer(Ptr: BlockBegin);
852 }
853 return SecondaryT::getBlockEnd(BlockBegin) - UntaggedPtr;
854 }
855
856 // Return the usable size for a given chunk. If MTE is enabled or if the
857 // ExactUsableSize config parameter is true, we report the exact size of
858 // the original allocation size. Otherwise, we will return the total
859 // actual usable size.
860 uptr getUsableSize(const void *Ptr) {
861 if (UNLIKELY(!Ptr))
862 return 0;
863
864 if (AllocatorConfig::getExactUsableSize() ||
865 UNLIKELY(useMemoryTagging<AllocatorConfig>(Primary.Options.load())))
866 return getAllocSize(Ptr);
867
868 initThreadMaybe();
869
870#ifdef GWP_ASAN_HOOKS
871 if (UNLIKELY(GuardedAlloc.pointerIsMine(Ptr)))
872 return GuardedAlloc.getSize(Ptr);
873#endif // GWP_ASAN_HOOKS
874
875 Ptr = getHeaderTaggedPointer(Ptr: const_cast<void *>(Ptr));
876 Chunk::UnpackedHeader Header;
877 Chunk::loadHeader(Cookie, Ptr, NewUnpackedHeader: &Header);
878
879 // Getting the alloc size of a chunk only makes sense if it's allocated.
880 if (UNLIKELY(Header.State != Chunk::State::Allocated))
881 reportInvalidChunkState(Action: AllocatorAction::Sizing, Ptr);
882
883 return getUsableSize(Ptr, &Header);
884 }
885
886 uptr getAllocSize(const void *Ptr) {
887 initThreadMaybe();
888
889#ifdef GWP_ASAN_HOOKS
890 if (UNLIKELY(GuardedAlloc.pointerIsMine(Ptr)))
891 return GuardedAlloc.getSize(Ptr);
892#endif // GWP_ASAN_HOOKS
893
894 Ptr = getHeaderTaggedPointer(Ptr: const_cast<void *>(Ptr));
895 Chunk::UnpackedHeader Header;
896 Chunk::loadHeader(Cookie, Ptr, NewUnpackedHeader: &Header);
897
898 // Getting the alloc size of a chunk only makes sense if it's allocated.
899 if (UNLIKELY(Header.State != Chunk::State::Allocated))
900 reportInvalidChunkState(Action: AllocatorAction::Sizing, Ptr);
901
902 return getSize(Ptr, Header: &Header);
903 }
904
905 void getStats(StatCounters S) {
906 initThreadMaybe();
907 Stats.get(S);
908 }
909
910 // Returns true if the pointer provided was allocated by the current
911 // allocator instance, which is compliant with tcmalloc's ownership concept.
912 // A corrupted chunk will not be reported as owned, which is WAI.
913 bool isOwned(const void *Ptr) {
914 initThreadMaybe();
915 // If the allocation is not owned, the tags could be wrong.
916 ScopedDisableMemoryTagChecks x(
917 useMemoryTagging<AllocatorConfig>(Primary.Options.load()));
918#ifdef GWP_ASAN_HOOKS
919 if (GuardedAlloc.pointerIsMine(Ptr))
920 return true;
921#endif // GWP_ASAN_HOOKS
922 if (!Ptr || !isAligned(X: reinterpret_cast<uptr>(Ptr), Alignment: MinAlignment))
923 return false;
924 Ptr = getHeaderTaggedPointer(Ptr: const_cast<void *>(Ptr));
925 Chunk::UnpackedHeader Header;
926 return Chunk::isValid(Cookie, Ptr, NewUnpackedHeader: &Header) &&
927 Header.State == Chunk::State::Allocated;
928 }
929
930 bool useMemoryTaggingTestOnly() const {
931 return useMemoryTagging<AllocatorConfig>(Primary.Options.load());
932 }
933 void disableMemoryTagging() {
934 // If we haven't been initialized yet, we need to initialize now in order to
935 // prevent a future call to initThreadMaybe() from enabling memory tagging
936 // based on feature detection. But don't call initThreadMaybe() because it
937 // may end up calling the allocator (via pthread_atfork, via the post-init
938 // callback), which may cause mappings to be created with memory tagging
939 // enabled.
940 TSDRegistry.initOnceMaybe(this);
941 if (allocatorSupportsMemoryTagging<AllocatorConfig>()) {
942 Secondary.disableMemoryTagging();
943 Primary.Options.clear(OptionBit::UseMemoryTagging);
944 }
945 }
946
947 void setTrackAllocationStacks(bool Track) {
948 initThreadMaybe();
949 if (getFlags()->allocation_ring_buffer_size <= 0) {
950 DCHECK(!Primary.Options.load().get(OptionBit::TrackAllocationStacks));
951 return;
952 }
953
954 if (Track) {
955 initRingBufferMaybe();
956 Primary.Options.set(OptionBit::TrackAllocationStacks);
957 } else
958 Primary.Options.clear(OptionBit::TrackAllocationStacks);
959 }
960
961 void setFillContents(FillContentsMode FillContents) {
962 initThreadMaybe();
963 Primary.Options.setFillContentsMode(FillContents);
964 }
965
966 void setAddLargeAllocationSlack(bool AddSlack) {
967 initThreadMaybe();
968 if (AddSlack)
969 Primary.Options.set(OptionBit::AddLargeAllocationSlack);
970 else
971 Primary.Options.clear(OptionBit::AddLargeAllocationSlack);
972 }
973
974 const char *getStackDepotAddress() {
975 initThreadMaybe();
976 AllocationRingBuffer *RB = getRingBuffer();
977 return RB ? reinterpret_cast<char *>(RB->Depot) : nullptr;
978 }
979
980 uptr getStackDepotSize() {
981 initThreadMaybe();
982 AllocationRingBuffer *RB = getRingBuffer();
983 return RB ? RB->StackDepotSize : 0;
984 }
985
986 const char *getRegionInfoArrayAddress() const {
987 return Primary.getRegionInfoArrayAddress();
988 }
989
990 static uptr getRegionInfoArraySize() {
991 return PrimaryT::getRegionInfoArraySize();
992 }
993
994 const char *getRingBufferAddress() {
995 initThreadMaybe();
996 return reinterpret_cast<char *>(getRingBuffer());
997 }
998
999 uptr getRingBufferSize() {
1000 initThreadMaybe();
1001 AllocationRingBuffer *RB = getRingBuffer();
1002 return RB && RB->RingBufferElements
1003 ? ringBufferSizeInBytes(RingBufferElements: RB->RingBufferElements)
1004 : 0;
1005 }
1006
1007 static const uptr MaxTraceSize = 64;
1008
1009 static void collectTraceMaybe(const StackDepot *Depot,
1010 uintptr_t (&Trace)[MaxTraceSize], u32 Hash) {
1011 uptr RingPos, Size;
1012 if (!Depot->find(Hash, RingPosPtr: &RingPos, SizePtr: &Size))
1013 return;
1014 for (unsigned I = 0; I != Size && I != MaxTraceSize; ++I)
1015 Trace[I] = static_cast<uintptr_t>(Depot->at(RingPos: RingPos + I));
1016 }
1017
1018 static void getErrorInfo(struct scudo_error_info *ErrorInfo,
1019 uintptr_t FaultAddr, const char *DepotPtr,
1020 size_t DepotSize, const char *RegionInfoPtr,
1021 const char *RingBufferPtr, size_t RingBufferSize,
1022 const char *Memory, const char *MemoryTags,
1023 uintptr_t MemoryAddr, size_t MemorySize) {
1024 // N.B. we need to support corrupted data in any of the buffers here. We get
1025 // this information from an external process (the crashing process) that
1026 // should not be able to crash the crash dumper (crash_dump on Android).
1027 // See also the get_error_info_fuzzer.
1028 *ErrorInfo = {};
1029 if (!allocatorSupportsMemoryTagging<AllocatorConfig>() ||
1030 MemoryAddr + MemorySize < MemoryAddr)
1031 return;
1032
1033 const StackDepot *Depot = nullptr;
1034 if (DepotPtr) {
1035 // check for corrupted StackDepot. First we need to check whether we can
1036 // read the metadata, then whether the metadata matches the size.
1037 if (DepotSize < sizeof(*Depot))
1038 return;
1039 Depot = reinterpret_cast<const StackDepot *>(DepotPtr);
1040 if (!Depot->isValid(BufSize: DepotSize))
1041 return;
1042 }
1043
1044 size_t NextErrorReport = 0;
1045
1046 // Check for OOB in the current block and the two surrounding blocks. Beyond
1047 // that, UAF is more likely.
1048 if (extractTag(Ptr: FaultAddr) != 0)
1049 getInlineErrorInfo(ErrorInfo, NextErrorReport, FaultAddr, Depot,
1050 RegionInfoPtr, Memory, MemoryTags, MemoryAddr,
1051 MemorySize, MinDistance: 0, MaxDistance: 2);
1052
1053 // Check the ring buffer. For primary allocations this will only find UAF;
1054 // for secondary allocations we can find either UAF or OOB.
1055 getRingBufferErrorInfo(ErrorInfo, NextErrorReport, FaultAddr, Depot,
1056 RingBufferPtr, RingBufferSize);
1057
1058 // Check for OOB in the 28 blocks surrounding the 3 we checked earlier.
1059 // Beyond that we are likely to hit false positives.
1060 if (extractTag(Ptr: FaultAddr) != 0)
1061 getInlineErrorInfo(ErrorInfo, NextErrorReport, FaultAddr, Depot,
1062 RegionInfoPtr, Memory, MemoryTags, MemoryAddr,
1063 MemorySize, MinDistance: 2, MaxDistance: 16);
1064 }
1065
1066 uptr getBlockBeginTestOnly(const void *Ptr) {
1067 Chunk::UnpackedHeader Header;
1068 Chunk::loadHeader(Cookie, Ptr, NewUnpackedHeader: &Header);
1069 DCHECK(Header.State == Chunk::State::Allocated);
1070
1071 if (allocatorSupportsMemoryTagging<AllocatorConfig>())
1072 Ptr = untagPointer(Ptr: const_cast<void *>(Ptr));
1073 void *Begin = getBlockBegin(Ptr, Header: &Header);
1074 if (allocatorSupportsMemoryTagging<AllocatorConfig>())
1075 Begin = untagPointer(Ptr: Begin);
1076 return reinterpret_cast<uptr>(Begin);
1077 }
1078
1079private:
1080 typedef typename PrimaryT::SizeClassMap SizeClassMap;
1081
1082 static const uptr MinAlignmentLog = SCUDO_MIN_ALIGNMENT_LOG;
1083 static const uptr MaxAlignmentLog = 24U; // 16 MB seems reasonable.
1084 static const uptr MinAlignment = 1UL << MinAlignmentLog;
1085 static const uptr MaxAlignment = 1UL << MaxAlignmentLog;
1086 static const uptr MaxAllowedMallocSize =
1087 FIRST_32_SECOND_64(1UL << 31, 1ULL << 40);
1088
1089 static_assert(MinAlignment >= sizeof(Chunk::PackedHeader),
1090 "Minimal alignment must at least cover a chunk header.");
1091 static_assert(!allocatorSupportsMemoryTagging<AllocatorConfig>() ||
1092 MinAlignment >= archMemoryTagGranuleSize(),
1093 "");
1094
1095 static const u32 BlockMarker = 0x44554353U;
1096
1097 // These are indexes into an "array" of 32-bit values that store information
1098 // inline with a chunk that is relevant to diagnosing memory tag faults, where
1099 // 0 corresponds to the address of the user memory. This means that only
1100 // negative indexes may be used. The smallest index that may be used is -2,
1101 // which corresponds to 8 bytes before the user memory, because the chunk
1102 // header size is 8 bytes and in allocators that support memory tagging the
1103 // minimum alignment is at least the tag granule size (16 on aarch64).
1104 static const sptr MemTagAllocationTraceIndex = -2;
1105 static const sptr MemTagAllocationTidIndex = -1;
1106
1107 u32 Cookie = 0;
1108 u32 QuarantineMaxChunkSize = 0;
1109#if SCUDO_FUCHSIA
1110 u32 ZeroOnDeallocMaxSize = 0;
1111#endif
1112
1113 GlobalStats Stats;
1114 PrimaryT Primary;
1115 SecondaryT Secondary;
1116 QuarantineT Quarantine;
1117 TSDRegistryT TSDRegistry;
1118 pthread_once_t PostInitNonce = PTHREAD_ONCE_INIT;
1119
1120#ifdef GWP_ASAN_HOOKS
1121 gwp_asan::GuardedPoolAllocator GuardedAlloc;
1122 uptr GuardedAllocSlotSize = 0;
1123#endif // GWP_ASAN_HOOKS
1124
1125 struct AllocationRingBuffer {
1126 struct Entry {
1127 atomic_uptr Ptr;
1128 atomic_uptr AllocationSize;
1129 atomic_u32 AllocationTrace;
1130 atomic_u32 AllocationTid;
1131 atomic_u32 DeallocationTrace;
1132 atomic_u32 DeallocationTid;
1133 };
1134 StackDepot *Depot = nullptr;
1135 uptr StackDepotSize = 0;
1136 MemMapT RawRingBufferMap;
1137 MemMapT RawStackDepotMap;
1138 u32 RingBufferElements = 0;
1139 atomic_uptr Pos;
1140 // An array of Size (at least one) elements of type Entry is immediately
1141 // following to this struct.
1142 };
1143 static_assert(sizeof(AllocationRingBuffer) %
1144 alignof(typename AllocationRingBuffer::Entry) ==
1145 0,
1146 "invalid alignment");
1147
1148 // Lock to initialize the RingBuffer
1149 HybridMutex RingBufferInitLock;
1150
1151 // Pointer to memory mapped area starting with AllocationRingBuffer struct,
1152 // and immediately followed by Size elements of type Entry.
1153 atomic_uptr RingBufferAddress = {};
1154
1155 AllocationRingBuffer *getRingBuffer() {
1156 return reinterpret_cast<AllocationRingBuffer *>(
1157 atomic_load(A: &RingBufferAddress, MO: memory_order_acquire));
1158 }
1159
1160 // The following might get optimized out by the compiler.
1161 NOINLINE void performSanityChecks() {
1162 // Verify that the header offset field can hold the maximum offset. In the
1163 // case of the Secondary allocator, it takes care of alignment and the
1164 // offset will always be small. In the case of the Primary, the worst case
1165 // scenario happens in the last size class, when the backend allocation
1166 // would already be aligned on the requested alignment, which would happen
1167 // to be the maximum alignment that would fit in that size class. As a
1168 // result, the maximum offset will be at most the maximum alignment for the
1169 // last size class minus the header size, in multiples of MinAlignment.
1170 Chunk::UnpackedHeader Header = {};
1171 const uptr MaxPrimaryAlignment = 1UL << getMostSignificantSetBitIndex(
1172 SizeClassMap::MaxSize - MinAlignment);
1173 const uptr MaxOffset =
1174 (MaxPrimaryAlignment - Chunk::getHeaderSize()) >> MinAlignmentLog;
1175 Header.Offset = MaxOffset & Chunk::OffsetMask;
1176 if (UNLIKELY(Header.Offset != MaxOffset))
1177 reportSanityCheckError(Field: "offset");
1178
1179 // Verify that we can fit the maximum size or amount of unused bytes in the
1180 // header. Given that the Secondary fits the allocation to a page, the worst
1181 // case scenario happens in the Primary. It will depend on the second to
1182 // last and last class sizes, as well as the dynamic base for the Primary.
1183 // The following is an over-approximation that works for our needs.
1184 const uptr MaxSizeOrUnusedBytes = SizeClassMap::MaxSize - 1;
1185 Header.SizeOrUnusedBytes = MaxSizeOrUnusedBytes;
1186 if (UNLIKELY(Header.SizeOrUnusedBytes != MaxSizeOrUnusedBytes))
1187 reportSanityCheckError(Field: "size (or unused bytes)");
1188
1189 const uptr LargestClassId = SizeClassMap::LargestClassId;
1190 Header.ClassId = LargestClassId;
1191 if (UNLIKELY(Header.ClassId != LargestClassId))
1192 reportSanityCheckError(Field: "class ID");
1193 }
1194
1195 static inline void *getBlockBegin(const void *Ptr,
1196 Chunk::UnpackedHeader *Header) {
1197 return reinterpret_cast<void *>(
1198 reinterpret_cast<uptr>(Ptr) - Chunk::getHeaderSize() -
1199 (static_cast<uptr>(Header->Offset) << MinAlignmentLog));
1200 }
1201
1202 // Return the size of a chunk as requested during its allocation.
1203 inline uptr getSize(const void *Ptr, Chunk::UnpackedHeader *Header) {
1204 const uptr SizeOrUnusedBytes = Header->SizeOrUnusedBytes;
1205 if (LIKELY(Header->ClassId))
1206 return SizeOrUnusedBytes;
1207 if (allocatorSupportsMemoryTagging<AllocatorConfig>())
1208 Ptr = untagPointer(Ptr: const_cast<void *>(Ptr));
1209 return SecondaryT::getBlockEnd(getBlockBegin(Ptr, Header)) -
1210 reinterpret_cast<uptr>(Ptr) - SizeOrUnusedBytes;
1211 }
1212
1213 ALWAYS_INLINE void *initChunk(const uptr ClassId, const Chunk::Origin Origin,
1214 void *Block, const uptr UserPtr,
1215 const uptr SizeOrUnusedBytes,
1216 const FillContentsMode FillContents) {
1217 // Compute the default pointer before adding the header tag
1218 const uptr DefaultAlignedPtr =
1219 reinterpret_cast<uptr>(Block) + Chunk::getHeaderSize();
1220
1221 Block = addHeaderTag(Block);
1222 // Only do content fill when it's from primary allocator because secondary
1223 // allocator has filled the content.
1224 if (ClassId != 0 && UNLIKELY(FillContents != NoFill)) {
1225 // This condition is not necessarily unlikely, but since memset is
1226 // costly, we might as well mark it as such.
1227 memset(Block, FillContents == ZeroFill ? 0 : PatternFillByte,
1228 PrimaryT::getSizeByClassId(ClassId));
1229 }
1230
1231 Chunk::UnpackedHeader Header = {};
1232
1233 if (UNLIKELY(DefaultAlignedPtr != UserPtr)) {
1234 const uptr Offset = UserPtr - DefaultAlignedPtr;
1235 DCHECK_GE(Offset, 2 * sizeof(u32));
1236 // The BlockMarker has no security purpose, but is specifically meant for
1237 // the chunk iteration function that can be used in debugging situations.
1238 // It is the only situation where we have to locate the start of a chunk
1239 // based on its block address.
1240 reinterpret_cast<u32 *>(Block)[0] = BlockMarker;
1241 reinterpret_cast<u32 *>(Block)[1] = static_cast<u32>(Offset);
1242 Header.Offset = (Offset >> MinAlignmentLog) & Chunk::OffsetMask;
1243 }
1244
1245 Header.ClassId = ClassId & Chunk::ClassIdMask;
1246 Header.State = Chunk::State::Allocated;
1247 Header.setOrigin(Origin);
1248 Header.SizeOrUnusedBytes = SizeOrUnusedBytes & Chunk::SizeOrUnusedBytesMask;
1249 Chunk::storeHeader(Cookie, Ptr: reinterpret_cast<void *>(addHeaderTag(UserPtr)),
1250 NewUnpackedHeader: &Header);
1251
1252 return reinterpret_cast<void *>(UserPtr);
1253 }
1254
1255 NOINLINE void *
1256 initChunkWithMemoryTagging(const uptr ClassId, const Chunk::Origin Origin,
1257 void *Block, const uptr UserPtr, const uptr Size,
1258 const uptr SizeOrUnusedBytes,
1259 const FillContentsMode FillContents) {
1260 const Options Options = Primary.Options.load();
1261 DCHECK(useMemoryTagging<AllocatorConfig>(Options));
1262
1263 // Compute the default pointer before adding the header tag
1264 const uptr DefaultAlignedPtr =
1265 reinterpret_cast<uptr>(Block) + Chunk::getHeaderSize();
1266
1267 void *Ptr = reinterpret_cast<void *>(UserPtr);
1268 void *TaggedPtr = Ptr;
1269
1270 if (LIKELY(ClassId)) {
1271 // Init the primary chunk.
1272 //
1273 // We only need to zero or tag the contents for Primary backed
1274 // allocations. We only set tags for primary allocations in order to avoid
1275 // faulting potentially large numbers of pages for large secondary
1276 // allocations. We assume that guard pages are enough to protect these
1277 // allocations.
1278 //
1279 // FIXME: When the kernel provides a way to set the background tag of a
1280 // mapping, we should be able to tag secondary allocations as well.
1281 //
1282 // When memory tagging is enabled, zeroing the contents is done as part of
1283 // setting the tag.
1284
1285 Chunk::UnpackedHeader Header;
1286 const uptr BlockSize = PrimaryT::getSizeByClassId(ClassId);
1287 const uptr BlockUptr = reinterpret_cast<uptr>(Block);
1288 const uptr BlockEnd = BlockUptr + BlockSize;
1289 // If possible, try to reuse the UAF tag that was set by deallocate().
1290 // For simplicity, only reuse tags if we have the same start address as
1291 // the previous allocation. This handles the majority of cases since
1292 // most allocations will not be more aligned than the minimum alignment.
1293 //
1294 // We need to handle situations involving reclaimed chunks, and retag
1295 // the reclaimed portions if necessary. In the case where the chunk is
1296 // fully reclaimed, the chunk's header will be zero, which will trigger
1297 // the code path for new mappings and invalid chunks that prepares the
1298 // chunk from scratch. There are three possibilities for partial
1299 // reclaiming:
1300 //
1301 // (1) Header was reclaimed, data was partially reclaimed.
1302 // (2) Header was not reclaimed, all data was reclaimed (e.g. because
1303 // data started on a page boundary).
1304 // (3) Header was not reclaimed, data was partially reclaimed.
1305 //
1306 // Case (1) will be handled in the same way as for full reclaiming,
1307 // since the header will be zero.
1308 //
1309 // We can detect case (2) by loading the tag from the start
1310 // of the chunk. If it is zero, it means that either all data was
1311 // reclaimed (since we never use zero as the chunk tag), or that the
1312 // previous allocation was of size zero. Either way, we need to prepare
1313 // a new chunk from scratch.
1314 //
1315 // We can detect case (3) by moving to the next page (if covered by the
1316 // chunk) and loading the tag of its first granule. If it is zero, it
1317 // means that all following pages may need to be retagged. On the other
1318 // hand, if it is nonzero, we can assume that all following pages are
1319 // still tagged, according to the logic that if any of the pages
1320 // following the next page were reclaimed, the next page would have been
1321 // reclaimed as well.
1322 uptr TaggedUserPtr;
1323 uptr PrevUserPtr;
1324 if (getChunkFromBlock(Block: BlockUptr, Chunk: &PrevUserPtr, Header: &Header) &&
1325 PrevUserPtr == UserPtr &&
1326 (TaggedUserPtr = loadTag(Ptr: UserPtr)) != UserPtr) {
1327 uptr PrevEnd = TaggedUserPtr + Header.SizeOrUnusedBytes;
1328 const uptr NextPage = roundUp(X: TaggedUserPtr, Boundary: getPageSizeCached());
1329 if (NextPage < PrevEnd && loadTag(Ptr: NextPage) != NextPage)
1330 PrevEnd = NextPage;
1331 TaggedPtr = reinterpret_cast<void *>(TaggedUserPtr);
1332 resizeTaggedChunk(OldPtr: PrevEnd, NewPtr: TaggedUserPtr + Size, NewSize: Size, BlockEnd);
1333 if (UNLIKELY(FillContents != NoFill && !Header.OriginOrWasZeroed)) {
1334 // If an allocation needs to be zeroed (i.e. calloc) we can normally
1335 // avoid zeroing the memory now since we can rely on memory having
1336 // been zeroed on free, as this is normally done while setting the
1337 // UAF tag. But if tagging was disabled per-thread when the memory
1338 // was freed, it would not have been retagged and thus zeroed, and
1339 // therefore it needs to be zeroed now.
1340 memset(s: TaggedPtr, c: 0,
1341 n: Min(A: Size, B: roundUp(X: PrevEnd - TaggedUserPtr,
1342 Boundary: archMemoryTagGranuleSize())));
1343 } else if (Size) {
1344 // Clear any stack metadata that may have previously been stored in
1345 // the chunk data.
1346 memset(s: TaggedPtr, c: 0, n: archMemoryTagGranuleSize());
1347 }
1348 } else {
1349 const uptr OddEvenMask =
1350 computeOddEvenMaskForPointerMaybe(Options, Ptr: BlockUptr, ClassId);
1351 TaggedPtr = prepareTaggedChunk(Ptr, Size, ExcludeMask: OddEvenMask, BlockEnd);
1352 }
1353 storePrimaryAllocationStackMaybe(Options, Ptr);
1354 } else {
1355 // Init the secondary chunk.
1356
1357 Block = addHeaderTag(Block);
1358 Ptr = addHeaderTag(Ptr);
1359 storeTags(Begin: reinterpret_cast<uptr>(Block), End: reinterpret_cast<uptr>(Ptr));
1360 storeSecondaryAllocationStackMaybe(Options, Ptr, Size);
1361 }
1362
1363 Chunk::UnpackedHeader Header = {};
1364
1365 if (UNLIKELY(DefaultAlignedPtr != UserPtr)) {
1366 const uptr Offset = UserPtr - DefaultAlignedPtr;
1367 DCHECK_GE(Offset, 2 * sizeof(u32));
1368 // The BlockMarker has no security purpose, but is specifically meant for
1369 // the chunk iteration function that can be used in debugging situations.
1370 // It is the only situation where we have to locate the start of a chunk
1371 // based on its block address.
1372 reinterpret_cast<u32 *>(Block)[0] = BlockMarker;
1373 reinterpret_cast<u32 *>(Block)[1] = static_cast<u32>(Offset);
1374 Header.Offset = (Offset >> MinAlignmentLog) & Chunk::OffsetMask;
1375 }
1376
1377 Header.ClassId = ClassId & Chunk::ClassIdMask;
1378 Header.State = Chunk::State::Allocated;
1379 Header.setOrigin(Origin);
1380 Header.SizeOrUnusedBytes = SizeOrUnusedBytes & Chunk::SizeOrUnusedBytesMask;
1381 Chunk::storeHeader(Cookie, Ptr, NewUnpackedHeader: &Header);
1382
1383 return TaggedPtr;
1384 }
1385
1386 void quarantineOrDeallocateChunk(const Options &Options, void *TaggedPtr,
1387 Chunk::UnpackedHeader *Header,
1388 uptr Size) NO_THREAD_SAFETY_ANALYSIS {
1389 void *Ptr = getHeaderTaggedPointer(Ptr: TaggedPtr);
1390 // If the quarantine is disabled, the actual size of a chunk is 0 or larger
1391 // than the maximum allowed, we return a chunk directly to the backend.
1392 // This purposefully underflows for Size == 0.
1393 const bool BypassQuarantine = AllocatorConfig::getQuarantineDisabled() ||
1394 !Quarantine.getCacheSize() ||
1395 ((Size - 1) >= QuarantineMaxChunkSize) ||
1396 !Header->ClassId;
1397 if (BypassQuarantine)
1398 Header->State = Chunk::State::Available;
1399 else
1400 Header->State = Chunk::State::Quarantined;
1401
1402 if (LIKELY(!useMemoryTagging<AllocatorConfig>(Options)))
1403 Header->OriginOrWasZeroed = 0U;
1404 else {
1405 Header->OriginOrWasZeroed =
1406 Header->ClassId && !TSDRegistry.getDisableMemInit();
1407 }
1408
1409 Chunk::storeHeader(Cookie, Ptr, NewUnpackedHeader: Header);
1410
1411 if (BypassQuarantine) {
1412 void *BlockBegin;
1413 if (LIKELY(!useMemoryTagging<AllocatorConfig>(Options))) {
1414 // Must do this after storeHeader because loadHeader uses a tagged ptr.
1415 if (allocatorSupportsMemoryTagging<AllocatorConfig>())
1416 Ptr = untagPointer(Ptr);
1417 BlockBegin = getBlockBegin(Ptr, Header);
1418 } else {
1419 BlockBegin = retagBlock(Options, TaggedPtr, Ptr, Header, Size, BypassQuarantine: true);
1420 }
1421
1422#if SCUDO_FUCHSIA
1423 if (AllocatorConfig::getEnableZeroOnDealloc()) {
1424 // Clearing the header is incompatible with quarantine and tagging.
1425 // Hence, it is fine to implement it only when quarantine is bypassed.
1426 DCHECK(!useMemoryTagging<AllocatorConfig>(Options));
1427 uptr length = reinterpret_cast<uptr>(Ptr) + Size -
1428 reinterpret_cast<uptr>(BlockBegin);
1429 if (length <= ZeroOnDeallocMaxSize)
1430 memset(BlockBegin, 0, length);
1431 }
1432#endif // SCUDO_FUCHSIA
1433
1434 const uptr ClassId = Header->ClassId;
1435 if (LIKELY(ClassId)) {
1436 bool CacheDrained;
1437 {
1438 typename TSDRegistryT::ScopedTSD TSD(TSDRegistry);
1439 CacheDrained =
1440 TSD->getSizeClassAllocator().deallocate(ClassId, BlockBegin);
1441 }
1442 // When we have drained some blocks back to the Primary from TSD, that
1443 // implies that we may have the chance to release some pages as well.
1444 // Note that in order not to block other thread's accessing the TSD,
1445 // release the TSD first then try the page release.
1446 if (CacheDrained)
1447 Primary.tryReleaseToOS(ClassId, ReleaseToOS::Normal);
1448 } else {
1449 Secondary.deallocate(Options, BlockBegin);
1450 }
1451 } else {
1452 if (UNLIKELY(useMemoryTagging<AllocatorConfig>(Options)))
1453 retagBlock(Options, TaggedPtr, Ptr, Header, Size, BypassQuarantine: false);
1454 typename TSDRegistryT::ScopedTSD TSD(TSDRegistry);
1455 Quarantine.put(&TSD->getQuarantineCache(),
1456 QuarantineCallback(*this, TSD->getSizeClassAllocator()),
1457 Ptr, Size);
1458 }
1459 }
1460
1461 NOINLINE void *retagBlock(const Options &Options, void *TaggedPtr, void *&Ptr,
1462 Chunk::UnpackedHeader *Header, const uptr Size,
1463 bool BypassQuarantine) {
1464 DCHECK(useMemoryTagging<AllocatorConfig>(Options));
1465
1466 const u8 PrevTag = extractTag(Ptr: reinterpret_cast<uptr>(TaggedPtr));
1467 storeDeallocationStackMaybe(Options, Ptr, PrevTag, Size);
1468 if (Header->ClassId && !TSDRegistry.getDisableMemInit()) {
1469 uptr TaggedBegin, TaggedEnd;
1470 const uptr OddEvenMask = computeOddEvenMaskForPointerMaybe(
1471 Options, Ptr: reinterpret_cast<uptr>(getBlockBegin(Ptr, Header)),
1472 ClassId: Header->ClassId);
1473 // Exclude the previous tag so that immediate use after free is
1474 // detected 100% of the time.
1475 setRandomTag(Ptr, Size, ExcludeMask: OddEvenMask | (1UL << PrevTag), TaggedBegin: &TaggedBegin,
1476 TaggedEnd: &TaggedEnd);
1477 }
1478
1479 Ptr = untagPointer(Ptr);
1480 void *BlockBegin = getBlockBegin(Ptr, Header);
1481 if (BypassQuarantine && !Header->ClassId) {
1482 storeTags(Begin: reinterpret_cast<uptr>(BlockBegin),
1483 End: reinterpret_cast<uptr>(Ptr));
1484 }
1485
1486 return BlockBegin;
1487 }
1488
1489 bool getChunkFromBlock(uptr Block, uptr *Chunk,
1490 Chunk::UnpackedHeader *Header) {
1491 *Chunk =
1492 Block + getChunkOffsetFromBlock(Block: reinterpret_cast<const char *>(Block));
1493 return Chunk::isValid(Cookie, Ptr: reinterpret_cast<void *>(*Chunk), NewUnpackedHeader: Header);
1494 }
1495
1496 static uptr getChunkOffsetFromBlock(const char *Block) {
1497 u32 Offset = 0;
1498 if (reinterpret_cast<const u32 *>(Block)[0] == BlockMarker)
1499 Offset = reinterpret_cast<const u32 *>(Block)[1];
1500 return Offset + Chunk::getHeaderSize();
1501 }
1502
1503 // Set the tag of the granule past the end of the allocation to 0, to catch
1504 // linear overflows even if a previous larger allocation used the same block
1505 // and tag. Only do this if the granule past the end is in our block, because
1506 // this would otherwise lead to a SEGV if the allocation covers the entire
1507 // block and our block is at the end of a mapping. The tag of the next block's
1508 // header granule will be set to 0, so it will serve the purpose of catching
1509 // linear overflows in this case.
1510 //
1511 // For allocations of size 0 we do not end up storing the address tag to the
1512 // memory tag space, which getInlineErrorInfo() normally relies on to match
1513 // address tags against chunks. To allow matching in this case we store the
1514 // address tag in the first byte of the chunk.
1515 void storeEndMarker(uptr End, uptr Size, uptr BlockEnd) {
1516 DCHECK_EQ(BlockEnd, untagPointer(BlockEnd));
1517 uptr UntaggedEnd = untagPointer(Ptr: End);
1518 if (UntaggedEnd != BlockEnd) {
1519 storeTag(Ptr: UntaggedEnd);
1520 if (Size == 0)
1521 *reinterpret_cast<u8 *>(UntaggedEnd) = extractTag(Ptr: End);
1522 }
1523 }
1524
1525 void *prepareTaggedChunk(void *Ptr, uptr Size, uptr ExcludeMask,
1526 uptr BlockEnd) {
1527 // Prepare the granule before the chunk to store the chunk header by setting
1528 // its tag to 0. Normally its tag will already be 0, but in the case where a
1529 // chunk holding a low alignment allocation is reused for a higher alignment
1530 // allocation, the chunk may already have a non-zero tag from the previous
1531 // allocation.
1532 storeTag(Ptr: reinterpret_cast<uptr>(Ptr) - archMemoryTagGranuleSize());
1533
1534 uptr TaggedBegin, TaggedEnd;
1535 setRandomTag(Ptr, Size, ExcludeMask, TaggedBegin: &TaggedBegin, TaggedEnd: &TaggedEnd);
1536
1537 storeEndMarker(End: TaggedEnd, Size, BlockEnd);
1538 return reinterpret_cast<void *>(TaggedBegin);
1539 }
1540
1541 void resizeTaggedChunk(uptr OldPtr, uptr NewPtr, uptr NewSize,
1542 uptr BlockEnd) {
1543 uptr RoundOldPtr = roundUp(X: OldPtr, Boundary: archMemoryTagGranuleSize());
1544 uptr RoundNewPtr;
1545 if (RoundOldPtr >= NewPtr) {
1546 // If the allocation is shrinking we just need to set the tag past the end
1547 // of the allocation to 0. See explanation in storeEndMarker() above.
1548 RoundNewPtr = roundUp(X: NewPtr, Boundary: archMemoryTagGranuleSize());
1549 } else {
1550 // Set the memory tag of the region
1551 // [RoundOldPtr, roundUp(NewPtr, archMemoryTagGranuleSize()))
1552 // to the pointer tag stored in OldPtr.
1553 RoundNewPtr = storeTags(Begin: RoundOldPtr, End: NewPtr);
1554 }
1555 storeEndMarker(End: RoundNewPtr, Size: NewSize, BlockEnd);
1556 }
1557
1558 void storePrimaryAllocationStackMaybe(const Options &Options, void *Ptr) {
1559 if (!UNLIKELY(Options.get(OptionBit::TrackAllocationStacks)))
1560 return;
1561 AllocationRingBuffer *RB = getRingBuffer();
1562 if (!RB)
1563 return;
1564 auto *Ptr32 = reinterpret_cast<u32 *>(Ptr);
1565 Ptr32[MemTagAllocationTraceIndex] = collectStackTrace(Depot: RB->Depot);
1566 Ptr32[MemTagAllocationTidIndex] = getThreadID();
1567 }
1568
1569 void storeRingBufferEntry(AllocationRingBuffer *RB, void *Ptr,
1570 u32 AllocationTrace, u32 AllocationTid,
1571 uptr AllocationSize, u32 DeallocationTrace,
1572 u32 DeallocationTid) {
1573 uptr Pos = atomic_fetch_add(&RB->Pos, 1, memory_order_relaxed);
1574 typename AllocationRingBuffer::Entry *Entry =
1575 getRingBufferEntry(RB, Pos % RB->RingBufferElements);
1576
1577 // First invalidate our entry so that we don't attempt to interpret a
1578 // partially written state in getSecondaryErrorInfo(). The fences below
1579 // ensure that the compiler does not move the stores to Ptr in between the
1580 // stores to the other fields.
1581 atomic_store_relaxed(&Entry->Ptr, 0);
1582
1583 __atomic_signal_fence(__ATOMIC_SEQ_CST);
1584 atomic_store_relaxed(&Entry->AllocationTrace, AllocationTrace);
1585 atomic_store_relaxed(&Entry->AllocationTid, AllocationTid);
1586 atomic_store_relaxed(&Entry->AllocationSize, AllocationSize);
1587 atomic_store_relaxed(&Entry->DeallocationTrace, DeallocationTrace);
1588 atomic_store_relaxed(&Entry->DeallocationTid, DeallocationTid);
1589 __atomic_signal_fence(__ATOMIC_SEQ_CST);
1590
1591 atomic_store_relaxed(&Entry->Ptr, reinterpret_cast<uptr>(Ptr));
1592 }
1593
1594 void storeSecondaryAllocationStackMaybe(const Options &Options, void *Ptr,
1595 uptr Size) {
1596 if (!UNLIKELY(Options.get(OptionBit::TrackAllocationStacks)))
1597 return;
1598 AllocationRingBuffer *RB = getRingBuffer();
1599 if (!RB)
1600 return;
1601 u32 Trace = collectStackTrace(Depot: RB->Depot);
1602 u32 Tid = getThreadID();
1603
1604 auto *Ptr32 = reinterpret_cast<u32 *>(Ptr);
1605 Ptr32[MemTagAllocationTraceIndex] = Trace;
1606 Ptr32[MemTagAllocationTidIndex] = Tid;
1607
1608 storeRingBufferEntry(RB, Ptr: untagPointer(Ptr), AllocationTrace: Trace, AllocationTid: Tid, AllocationSize: Size, DeallocationTrace: 0, DeallocationTid: 0);
1609 }
1610
1611 void storeDeallocationStackMaybe(const Options &Options, void *Ptr,
1612 u8 PrevTag, uptr Size) {
1613 if (!UNLIKELY(Options.get(OptionBit::TrackAllocationStacks)))
1614 return;
1615 AllocationRingBuffer *RB = getRingBuffer();
1616 if (!RB)
1617 return;
1618 auto *Ptr32 = reinterpret_cast<u32 *>(Ptr);
1619 u32 AllocationTrace = Ptr32[MemTagAllocationTraceIndex];
1620 u32 AllocationTid = Ptr32[MemTagAllocationTidIndex];
1621
1622 u32 DeallocationTrace = collectStackTrace(Depot: RB->Depot);
1623 u32 DeallocationTid = getThreadID();
1624
1625 storeRingBufferEntry(RB, Ptr: addFixedTag(Ptr: untagPointer(Ptr), Tag: PrevTag),
1626 AllocationTrace, AllocationTid, AllocationSize: Size,
1627 DeallocationTrace, DeallocationTid);
1628 }
1629
1630 static const size_t NumErrorReports =
1631 sizeof(((scudo_error_info *)nullptr)->reports) /
1632 sizeof(((scudo_error_info *)nullptr)->reports[0]);
1633
1634 static void getInlineErrorInfo(struct scudo_error_info *ErrorInfo,
1635 size_t &NextErrorReport, uintptr_t FaultAddr,
1636 const StackDepot *Depot,
1637 const char *RegionInfoPtr, const char *Memory,
1638 const char *MemoryTags, uintptr_t MemoryAddr,
1639 size_t MemorySize, size_t MinDistance,
1640 size_t MaxDistance) {
1641 uptr UntaggedFaultAddr = untagPointer(Ptr: FaultAddr);
1642 u8 FaultAddrTag = extractTag(Ptr: FaultAddr);
1643 BlockInfo Info =
1644 PrimaryT::findNearestBlock(RegionInfoPtr, UntaggedFaultAddr);
1645
1646 auto GetGranule = [&](uptr Addr, const char **Data, uint8_t *Tag) -> bool {
1647 if (Addr < MemoryAddr || Addr + archMemoryTagGranuleSize() < Addr ||
1648 Addr + archMemoryTagGranuleSize() > MemoryAddr + MemorySize)
1649 return false;
1650 *Data = &Memory[Addr - MemoryAddr];
1651 *Tag = static_cast<u8>(
1652 MemoryTags[(Addr - MemoryAddr) / archMemoryTagGranuleSize()]);
1653 return true;
1654 };
1655
1656 auto ReadBlock = [&](uptr Addr, uptr *ChunkAddr,
1657 Chunk::UnpackedHeader *Header, const u32 **Data,
1658 u8 *Tag) {
1659 const char *BlockBegin;
1660 u8 BlockBeginTag;
1661 if (!GetGranule(Addr, &BlockBegin, &BlockBeginTag))
1662 return false;
1663 uptr ChunkOffset = getChunkOffsetFromBlock(Block: BlockBegin);
1664 *ChunkAddr = Addr + ChunkOffset;
1665
1666 const char *ChunkBegin;
1667 if (!GetGranule(*ChunkAddr, &ChunkBegin, Tag))
1668 return false;
1669 *Header = *reinterpret_cast<const Chunk::UnpackedHeader *>(
1670 ChunkBegin - Chunk::getHeaderSize());
1671 *Data = reinterpret_cast<const u32 *>(ChunkBegin);
1672
1673 // Allocations of size 0 will have stashed the tag in the first byte of
1674 // the chunk, see storeEndMarker().
1675 if (Header->SizeOrUnusedBytes == 0)
1676 *Tag = static_cast<u8>(*ChunkBegin);
1677
1678 return true;
1679 };
1680
1681 if (NextErrorReport == NumErrorReports)
1682 return;
1683
1684 auto CheckOOB = [&](uptr BlockAddr) {
1685 if (BlockAddr < Info.RegionBegin || BlockAddr >= Info.RegionEnd)
1686 return false;
1687
1688 uptr ChunkAddr;
1689 Chunk::UnpackedHeader Header;
1690 const u32 *Data;
1691 uint8_t Tag;
1692 if (!ReadBlock(BlockAddr, &ChunkAddr, &Header, &Data, &Tag) ||
1693 Header.State != Chunk::State::Allocated || Tag != FaultAddrTag)
1694 return false;
1695
1696 auto *R = &ErrorInfo->reports[NextErrorReport++];
1697 R->error_type =
1698 UntaggedFaultAddr < ChunkAddr ? BUFFER_UNDERFLOW : BUFFER_OVERFLOW;
1699 R->allocation_address = ChunkAddr;
1700 R->allocation_size = Header.SizeOrUnusedBytes;
1701 if (Depot) {
1702 collectTraceMaybe(Depot, Trace&: R->allocation_trace,
1703 Hash: Data[MemTagAllocationTraceIndex]);
1704 }
1705 R->allocation_tid = Data[MemTagAllocationTidIndex];
1706 return NextErrorReport == NumErrorReports;
1707 };
1708
1709 if (MinDistance == 0 && CheckOOB(Info.BlockBegin))
1710 return;
1711
1712 for (size_t I = Max<size_t>(A: MinDistance, B: 1); I != MaxDistance; ++I)
1713 if (CheckOOB(Info.BlockBegin + I * Info.BlockSize) ||
1714 CheckOOB(Info.BlockBegin - I * Info.BlockSize))
1715 return;
1716 }
1717
1718 static void getRingBufferErrorInfo(struct scudo_error_info *ErrorInfo,
1719 size_t &NextErrorReport,
1720 uintptr_t FaultAddr,
1721 const StackDepot *Depot,
1722 const char *RingBufferPtr,
1723 size_t RingBufferSize) {
1724 auto *RingBuffer =
1725 reinterpret_cast<const AllocationRingBuffer *>(RingBufferPtr);
1726 size_t RingBufferElements = ringBufferElementsFromBytes(Bytes: RingBufferSize);
1727 if (!RingBuffer || RingBufferElements == 0 || !Depot)
1728 return;
1729 uptr Pos = atomic_load_relaxed(&RingBuffer->Pos);
1730
1731 for (uptr I = Pos - 1; I != Pos - 1 - RingBufferElements &&
1732 NextErrorReport != NumErrorReports;
1733 --I) {
1734 auto *Entry = getRingBufferEntry(RingBuffer, I % RingBufferElements);
1735 uptr EntryPtr = atomic_load_relaxed(&Entry->Ptr);
1736 if (!EntryPtr)
1737 continue;
1738
1739 uptr UntaggedEntryPtr = untagPointer(Ptr: EntryPtr);
1740 uptr EntrySize = atomic_load_relaxed(&Entry->AllocationSize);
1741 u32 AllocationTrace = atomic_load_relaxed(&Entry->AllocationTrace);
1742 u32 AllocationTid = atomic_load_relaxed(&Entry->AllocationTid);
1743 u32 DeallocationTrace = atomic_load_relaxed(&Entry->DeallocationTrace);
1744 u32 DeallocationTid = atomic_load_relaxed(&Entry->DeallocationTid);
1745
1746 if (DeallocationTid) {
1747 // For UAF we only consider in-bounds fault addresses because
1748 // out-of-bounds UAF is rare and attempting to detect it is very likely
1749 // to result in false positives.
1750 if (FaultAddr < EntryPtr || FaultAddr >= EntryPtr + EntrySize)
1751 continue;
1752 } else {
1753 // Ring buffer OOB is only possible with secondary allocations. In this
1754 // case we are guaranteed a guard region of at least a page on either
1755 // side of the allocation (guard page on the right, guard page + tagged
1756 // region on the left), so ignore any faults outside of that range.
1757 if (FaultAddr < EntryPtr - getPageSizeCached() ||
1758 FaultAddr >= EntryPtr + EntrySize + getPageSizeCached())
1759 continue;
1760
1761 // For UAF the ring buffer will contain two entries, one for the
1762 // allocation and another for the deallocation. Don't report buffer
1763 // overflow/underflow using the allocation entry if we have already
1764 // collected a report from the deallocation entry.
1765 bool Found = false;
1766 for (uptr J = 0; J != NextErrorReport; ++J) {
1767 if (ErrorInfo->reports[J].allocation_address == UntaggedEntryPtr) {
1768 Found = true;
1769 break;
1770 }
1771 }
1772 if (Found)
1773 continue;
1774 }
1775
1776 auto *R = &ErrorInfo->reports[NextErrorReport++];
1777 if (DeallocationTid)
1778 R->error_type = USE_AFTER_FREE;
1779 else if (FaultAddr < EntryPtr)
1780 R->error_type = BUFFER_UNDERFLOW;
1781 else
1782 R->error_type = BUFFER_OVERFLOW;
1783
1784 R->allocation_address = UntaggedEntryPtr;
1785 R->allocation_size = EntrySize;
1786 collectTraceMaybe(Depot, Trace&: R->allocation_trace, Hash: AllocationTrace);
1787 R->allocation_tid = AllocationTid;
1788 collectTraceMaybe(Depot, Trace&: R->deallocation_trace, Hash: DeallocationTrace);
1789 R->deallocation_tid = DeallocationTid;
1790 }
1791 }
1792
1793 uptr getStats(ScopedString *Str) {
1794 Primary.getStats(Str);
1795 Secondary.getStats(Str);
1796 if (!AllocatorConfig::getQuarantineDisabled())
1797 Quarantine.getStats(Str);
1798 TSDRegistry.getStats(Str);
1799 return Str->length();
1800 }
1801
1802 static typename AllocationRingBuffer::Entry *
1803 getRingBufferEntry(AllocationRingBuffer *RB, uptr N) {
1804 char *RBEntryStart =
1805 &reinterpret_cast<char *>(RB)[sizeof(AllocationRingBuffer)];
1806 return &reinterpret_cast<typename AllocationRingBuffer::Entry *>(
1807 RBEntryStart)[N];
1808 }
1809 static const typename AllocationRingBuffer::Entry *
1810 getRingBufferEntry(const AllocationRingBuffer *RB, uptr N) {
1811 const char *RBEntryStart =
1812 &reinterpret_cast<const char *>(RB)[sizeof(AllocationRingBuffer)];
1813 return &reinterpret_cast<const typename AllocationRingBuffer::Entry *>(
1814 RBEntryStart)[N];
1815 }
1816
1817 void initRingBufferMaybe() {
1818 ScopedLock L(RingBufferInitLock);
1819 if (getRingBuffer() != nullptr)
1820 return;
1821
1822 int ring_buffer_size = getFlags()->allocation_ring_buffer_size;
1823 if (ring_buffer_size <= 0)
1824 return;
1825
1826 u32 AllocationRingBufferSize = static_cast<u32>(ring_buffer_size);
1827
1828 // We store alloc and free stacks for each entry.
1829 constexpr u32 kStacksPerRingBufferEntry = 2;
1830 constexpr u32 kMaxU32Pow2 = ~(UINT32_MAX >> 1);
1831 static_assert(isPowerOfTwo(X: kMaxU32Pow2));
1832 // On Android we always have 3 frames at the bottom: __start_main,
1833 // __libc_init, main, and 3 at the top: malloc, scudo_malloc and
1834 // Allocator::allocate. This leaves 10 frames for the user app. The next
1835 // smallest power of two (8) would only leave 2, which is clearly too
1836 // little.
1837 constexpr u32 kFramesPerStack = 16;
1838 static_assert(isPowerOfTwo(X: kFramesPerStack));
1839
1840 if (AllocationRingBufferSize > kMaxU32Pow2 / kStacksPerRingBufferEntry)
1841 return;
1842 u32 TabSize = static_cast<u32>(roundUpPowerOfTwo(Size: kStacksPerRingBufferEntry *
1843 AllocationRingBufferSize));
1844 if (TabSize > UINT32_MAX / kFramesPerStack)
1845 return;
1846 u32 RingSize = static_cast<u32>(TabSize * kFramesPerStack);
1847
1848 uptr StackDepotSize = sizeof(StackDepot) + sizeof(atomic_u64) * RingSize +
1849 sizeof(atomic_u32) * TabSize;
1850 MemMapT DepotMap;
1851 DepotMap.map(
1852 /*Addr=*/Addr: 0U, Size: roundUp(X: StackDepotSize, Boundary: getPageSizeCached()),
1853 Name: "scudo:stack_depot");
1854 auto *Depot = reinterpret_cast<StackDepot *>(DepotMap.getBase());
1855 Depot->init(RingSz: RingSize, TabSz: TabSize);
1856
1857 MemMapT MemMap;
1858 MemMap.map(
1859 /*Addr=*/Addr: 0U,
1860 Size: roundUp(X: ringBufferSizeInBytes(RingBufferElements: AllocationRingBufferSize),
1861 Boundary: getPageSizeCached()),
1862 Name: "scudo:ring_buffer");
1863 auto *RB = reinterpret_cast<AllocationRingBuffer *>(MemMap.getBase());
1864 RB->RawRingBufferMap = MemMap;
1865 RB->RingBufferElements = AllocationRingBufferSize;
1866 RB->Depot = Depot;
1867 RB->StackDepotSize = StackDepotSize;
1868 RB->RawStackDepotMap = DepotMap;
1869
1870 atomic_store(A: &RingBufferAddress, V: reinterpret_cast<uptr>(RB),
1871 MO: memory_order_release);
1872 }
1873
1874 void unmapRingBuffer() {
1875 AllocationRingBuffer *RB = getRingBuffer();
1876 if (RB == nullptr)
1877 return;
1878 // N.B. because RawStackDepotMap is part of RawRingBufferMap, the order
1879 // is very important.
1880 RB->RawStackDepotMap.unmap();
1881 // Note that the `RB->RawRingBufferMap` is stored on the pages managed by
1882 // itself. Take over the ownership before calling unmap() so that any
1883 // operation along with unmap() won't touch inaccessible pages.
1884 MemMapT RawRingBufferMap = RB->RawRingBufferMap;
1885 RawRingBufferMap.unmap();
1886 atomic_store(A: &RingBufferAddress, V: 0, MO: memory_order_release);
1887 }
1888
1889 static constexpr size_t ringBufferSizeInBytes(u32 RingBufferElements) {
1890 return sizeof(AllocationRingBuffer) +
1891 RingBufferElements * sizeof(typename AllocationRingBuffer::Entry);
1892 }
1893
1894 static constexpr size_t ringBufferElementsFromBytes(size_t Bytes) {
1895 if (Bytes < sizeof(AllocationRingBuffer)) {
1896 return 0;
1897 }
1898 return (Bytes - sizeof(AllocationRingBuffer)) /
1899 sizeof(typename AllocationRingBuffer::Entry);
1900 }
1901};
1902
1903} // namespace scudo
1904
1905#endif // SCUDO_COMBINED_H_
1906