1//===-- combined.h ----------------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#ifndef SCUDO_COMBINED_H_
10#define SCUDO_COMBINED_H_
11
12#include "allocator_config_wrapper.h"
13#include "atomic_helpers.h"
14#include "chunk.h"
15#include "common.h"
16#include "flags.h"
17#include "flags_parser.h"
18#include "mem_map.h"
19#include "memtag.h"
20#include "mutex.h"
21#include "options.h"
22#include "quarantine.h"
23#include "report.h"
24#include "secondary.h"
25#include "size_class_allocator.h"
26#include "stack_depot.h"
27#include "string_utils.h"
28#include "tsd.h"
29
30#include "scudo/interface.h"
31
32#ifdef GWP_ASAN_HOOKS
33#include "gwp_asan/guarded_pool_allocator.h"
34#include "gwp_asan/optional/backtrace.h"
35#include "gwp_asan/optional/segv_handler.h"
36#endif // GWP_ASAN_HOOKS
37
38extern "C" inline void EmptyCallback() {}
39
40#ifdef HAVE_ANDROID_UNSAFE_FRAME_POINTER_CHASE
41// This function is not part of the NDK so it does not appear in any public
42// header files. We only declare/use it when targeting the platform.
43extern "C" size_t android_unsafe_frame_pointer_chase(scudo::uptr *buf,
44 size_t num_entries);
45#endif
46
47namespace scudo {
48
49template <class Config, void (*PostInitCallback)(void) = EmptyCallback>
50class Allocator {
51public:
52 using AllocatorConfig = BaseConfig<Config>;
53 using PrimaryT =
54 typename AllocatorConfig::template PrimaryT<PrimaryConfig<Config>>;
55 using SecondaryT =
56 typename AllocatorConfig::template SecondaryT<SecondaryConfig<Config>>;
57 using SizeClassAllocatorT = typename PrimaryT::SizeClassAllocatorT;
58 typedef Allocator<Config, PostInitCallback> ThisT;
59 typedef typename AllocatorConfig::template TSDRegistryT<ThisT> TSDRegistryT;
60
61 void callPostInitCallback() {
62 pthread_once(once_control: &PostInitNonce, init_routine: PostInitCallback);
63 }
64
65 struct QuarantineCallback {
66 explicit QuarantineCallback(ThisT &Instance,
67 SizeClassAllocatorT &SizeClassAllocator)
68 : Allocator(Instance), SizeClassAllocator(SizeClassAllocator) {}
69
70 // Chunk recycling function, returns a quarantined chunk to the backend,
71 // first making sure it hasn't been tampered with.
72 void recycle(void *Ptr) {
73 Chunk::UnpackedHeader Header;
74 Chunk::loadHeader(Cookie: Allocator.Cookie, Ptr, NewUnpackedHeader: &Header);
75 if (UNLIKELY(Header.State != Chunk::State::Quarantined))
76 reportInvalidChunkState(Action: AllocatorAction::Recycling, Ptr);
77
78 Header.State = Chunk::State::Available;
79 Chunk::storeHeader(Cookie: Allocator.Cookie, Ptr, NewUnpackedHeader: &Header);
80
81 if (allocatorSupportsMemoryTagging<AllocatorConfig>())
82 Ptr = untagPointer(Ptr);
83 void *BlockBegin = Allocator::getBlockBegin(Ptr, Header: &Header);
84 SizeClassAllocator.deallocate(Header.ClassId, BlockBegin);
85 }
86
87 // We take a shortcut when allocating a quarantine batch by working with the
88 // appropriate class ID instead of using Size. The compiler should optimize
89 // the class ID computation and work with the associated cache directly.
90 void *allocate(UNUSED uptr Size) {
91 const uptr QuarantineClassId = SizeClassMap::getClassIdBySize(
92 sizeof(QuarantineBatch) + Chunk::getHeaderSize());
93 void *Ptr = SizeClassAllocator.allocate(QuarantineClassId);
94 // Quarantine batch allocation failure is fatal.
95 if (UNLIKELY(!Ptr))
96 reportOutOfMemory(SizeClassMap::getSizeByClassId(QuarantineClassId));
97
98 Ptr = reinterpret_cast<void *>(reinterpret_cast<uptr>(Ptr) +
99 Chunk::getHeaderSize());
100 Chunk::UnpackedHeader Header = {};
101 Header.ClassId = QuarantineClassId & Chunk::ClassIdMask;
102 Header.SizeOrUnusedBytes = sizeof(QuarantineBatch);
103 Header.State = Chunk::State::Allocated;
104 Chunk::storeHeader(Cookie: Allocator.Cookie, Ptr, NewUnpackedHeader: &Header);
105
106 // Reset tag to 0 as this chunk may have been previously used for a tagged
107 // user allocation.
108 if (UNLIKELY(useMemoryTagging<AllocatorConfig>(
109 Allocator.Primary.Options.load())))
110 storeTags(Begin: reinterpret_cast<uptr>(Ptr),
111 End: reinterpret_cast<uptr>(Ptr) + sizeof(QuarantineBatch));
112
113 return Ptr;
114 }
115
116 void deallocate(void *Ptr) {
117 const uptr QuarantineClassId = SizeClassMap::getClassIdBySize(
118 sizeof(QuarantineBatch) + Chunk::getHeaderSize());
119 Chunk::UnpackedHeader Header;
120 Chunk::loadHeader(Cookie: Allocator.Cookie, Ptr, NewUnpackedHeader: &Header);
121
122 if (UNLIKELY(Header.State != Chunk::State::Allocated))
123 reportInvalidChunkState(Action: AllocatorAction::Deallocating, Ptr);
124 DCHECK_EQ(Header.ClassId, QuarantineClassId);
125 DCHECK_EQ(Header.Offset, 0);
126 DCHECK_EQ(Header.SizeOrUnusedBytes, sizeof(QuarantineBatch));
127
128 Header.State = Chunk::State::Available;
129 Chunk::storeHeader(Cookie: Allocator.Cookie, Ptr, NewUnpackedHeader: &Header);
130 SizeClassAllocator.deallocate(
131 QuarantineClassId,
132 reinterpret_cast<void *>(reinterpret_cast<uptr>(Ptr) -
133 Chunk::getHeaderSize()));
134 }
135
136 private:
137 ThisT &Allocator;
138 SizeClassAllocatorT &SizeClassAllocator;
139 };
140
141 typedef GlobalQuarantine<QuarantineCallback, void> QuarantineT;
142 typedef typename QuarantineT::CacheT QuarantineCacheT;
143
144 void init() {
145 // Make sure that the page size is initialized if it's not a constant.
146 CHECK_NE(getPageSizeCached(), 0U);
147
148 performSanityChecks();
149
150 // Check if hardware CRC32 is supported in the binary and by the platform,
151 // if so, opt for the CRC32 hardware version of the checksum.
152 if (&computeHardwareCRC32 && hasHardwareCRC32())
153 HashAlgorithm = Checksum::HardwareCRC32;
154
155 if (UNLIKELY(!getRandom(&Cookie, sizeof(Cookie))))
156 Cookie = static_cast<u32>(getMonotonicTime() ^
157 (reinterpret_cast<uptr>(this) >> 4));
158
159 initFlags();
160 reportUnrecognizedFlags();
161
162 // Store some flags locally.
163 if (getFlags()->may_return_null)
164 Primary.Options.set(OptionBit::MayReturnNull);
165 if (getFlags()->zero_contents)
166 Primary.Options.setFillContentsMode(ZeroFill);
167 else if (getFlags()->pattern_fill_contents)
168 Primary.Options.setFillContentsMode(PatternOrZeroFill);
169 if (getFlags()->dealloc_type_mismatch)
170 Primary.Options.set(OptionBit::DeallocTypeMismatch);
171 if (getFlags()->delete_size_mismatch)
172 Primary.Options.set(OptionBit::DeleteSizeMismatch);
173 if (allocatorSupportsMemoryTagging<AllocatorConfig>() &&
174 systemSupportsMemoryTagging())
175 Primary.Options.set(OptionBit::UseMemoryTagging);
176
177 QuarantineMaxChunkSize =
178 static_cast<u32>(getFlags()->quarantine_max_chunk_size);
179
180 Stats.init();
181 // TODO(chiahungduan): Given that we support setting the default value in
182 // the PrimaryConfig and CacheConfig, consider to deprecate the use of
183 // `release_to_os_interval_ms` flag.
184 const s32 ReleaseToOsIntervalMs = getFlags()->release_to_os_interval_ms;
185 Primary.init(ReleaseToOsIntervalMs);
186 Secondary.init(&Stats, ReleaseToOsIntervalMs);
187 Quarantine.init(
188 static_cast<uptr>(getFlags()->quarantine_size_kb << 10),
189 static_cast<uptr>(getFlags()->thread_local_quarantine_size_kb << 10));
190 }
191
192 void enableRingBuffer() NO_THREAD_SAFETY_ANALYSIS {
193 AllocationRingBuffer *RB = getRingBuffer();
194 if (RB)
195 RB->Depot->enable();
196 RingBufferInitLock.unlock();
197 }
198
199 void disableRingBuffer() NO_THREAD_SAFETY_ANALYSIS {
200 RingBufferInitLock.lock();
201 AllocationRingBuffer *RB = getRingBuffer();
202 if (RB)
203 RB->Depot->disable();
204 }
205
206 // Initialize the embedded GWP-ASan instance. Requires the main allocator to
207 // be functional, best called from PostInitCallback.
208 void initGwpAsan() {
209#ifdef GWP_ASAN_HOOKS
210 gwp_asan::options::Options Opt;
211 Opt.Enabled = getFlags()->GWP_ASAN_Enabled;
212 Opt.MaxSimultaneousAllocations =
213 getFlags()->GWP_ASAN_MaxSimultaneousAllocations;
214 Opt.SampleRate = getFlags()->GWP_ASAN_SampleRate;
215 Opt.InstallSignalHandlers = getFlags()->GWP_ASAN_InstallSignalHandlers;
216 Opt.Recoverable = getFlags()->GWP_ASAN_Recoverable;
217 // Embedded GWP-ASan is locked through the Scudo atfork handler (via
218 // Allocator::disable calling GWPASan.disable). Disable GWP-ASan's atfork
219 // handler.
220 Opt.InstallForkHandlers = false;
221 Opt.Backtrace = gwp_asan::backtrace::getBacktraceFunction();
222 GuardedAlloc.init(Opts: Opt);
223
224 if (Opt.InstallSignalHandlers)
225 gwp_asan::segv_handler::installSignalHandlers(
226 GPA: &GuardedAlloc, Printf,
227 PrintBacktrace: gwp_asan::backtrace::getPrintBacktraceFunction(),
228 SegvBacktrace: gwp_asan::backtrace::getSegvBacktraceFunction(),
229 Recoverable: Opt.Recoverable);
230
231 GuardedAllocSlotSize =
232 GuardedAlloc.getAllocatorState()->maximumAllocationSize();
233 Stats.add(I: StatFree, V: static_cast<uptr>(Opt.MaxSimultaneousAllocations) *
234 GuardedAllocSlotSize);
235#endif // GWP_ASAN_HOOKS
236 }
237
238#ifdef GWP_ASAN_HOOKS
239 const gwp_asan::AllocationMetadata *getGwpAsanAllocationMetadata() {
240 return GuardedAlloc.getMetadataRegion();
241 }
242
243 const gwp_asan::AllocatorState *getGwpAsanAllocatorState() {
244 return GuardedAlloc.getAllocatorState();
245 }
246#endif // GWP_ASAN_HOOKS
247
248 ALWAYS_INLINE void initThreadMaybe(bool MinimalInit = false) {
249 TSDRegistry.initThreadMaybe(this, MinimalInit);
250 }
251
252 void unmapTestOnly() {
253 unmapRingBuffer();
254 TSDRegistry.unmapTestOnly(this);
255 Primary.unmapTestOnly();
256 Secondary.unmapTestOnly();
257#ifdef GWP_ASAN_HOOKS
258 if (getFlags()->GWP_ASAN_InstallSignalHandlers)
259 gwp_asan::segv_handler::uninstallSignalHandlers();
260 GuardedAlloc.uninitTestOnly();
261#endif // GWP_ASAN_HOOKS
262 }
263
264 TSDRegistryT *getTSDRegistry() { return &TSDRegistry; }
265 QuarantineT *getQuarantine() { return &Quarantine; }
266
267 // The Cache must be provided zero-initialized.
268 void initAllocator(SizeClassAllocatorT *SizeClassAllocator) {
269 SizeClassAllocator->init(&Stats, &Primary);
270 }
271
272 // Release the resources used by a TSD, which involves:
273 // - draining the local quarantine cache to the global quarantine;
274 // - releasing the cached pointers back to the Primary;
275 // - unlinking the local stats from the global ones (destroying the cache does
276 // the last two items).
277 void commitBack(TSD<ThisT> *TSD) {
278 TSD->assertLocked(/*BypassCheck=*/true);
279 Quarantine.drain(&TSD->getQuarantineCache(),
280 QuarantineCallback(*this, TSD->getSizeClassAllocator()));
281 TSD->getSizeClassAllocator().destroy(&Stats);
282 }
283
284 void drainCache(TSD<ThisT> *TSD) {
285 TSD->assertLocked(/*BypassCheck=*/true);
286 Quarantine.drainAndRecycle(
287 &TSD->getQuarantineCache(),
288 QuarantineCallback(*this, TSD->getSizeClassAllocator()));
289 TSD->getSizeClassAllocator().drain();
290 }
291 void drainCaches() { TSDRegistry.drainCaches(this); }
292
293 ALWAYS_INLINE void *getHeaderTaggedPointer(void *Ptr) {
294 if (!allocatorSupportsMemoryTagging<AllocatorConfig>())
295 return Ptr;
296 auto UntaggedPtr = untagPointer(Ptr);
297 if (UntaggedPtr != Ptr)
298 return UntaggedPtr;
299 // Secondary, or pointer allocated while memory tagging is unsupported or
300 // disabled. The tag mismatch is okay in the latter case because tags will
301 // not be checked.
302 return addHeaderTag(Ptr);
303 }
304
305 ALWAYS_INLINE uptr addHeaderTag(uptr Ptr) {
306 if (!allocatorSupportsMemoryTagging<AllocatorConfig>())
307 return Ptr;
308 return addFixedTag(Ptr, Tag: 2);
309 }
310
311 ALWAYS_INLINE void *addHeaderTag(void *Ptr) {
312 return reinterpret_cast<void *>(addHeaderTag(reinterpret_cast<uptr>(Ptr)));
313 }
314
315 NOINLINE u32 collectStackTrace(UNUSED StackDepot *Depot) {
316#ifdef HAVE_ANDROID_UNSAFE_FRAME_POINTER_CHASE
317 // Discard collectStackTrace() frame and allocator function frame.
318 constexpr uptr DiscardFrames = 2;
319 uptr Stack[MaxTraceSize + DiscardFrames];
320 uptr Size =
321 android_unsafe_frame_pointer_chase(Stack, MaxTraceSize + DiscardFrames);
322 Size = Min<uptr>(Size, MaxTraceSize + DiscardFrames);
323 return Depot->insert(Stack + Min<uptr>(DiscardFrames, Size), Stack + Size);
324#else
325 return 0;
326#endif
327 }
328
329 uptr computeOddEvenMaskForPointerMaybe(const Options &Options, uptr Ptr,
330 uptr ClassId) {
331 if (!Options.get(Opt: OptionBit::UseOddEvenTags))
332 return 0;
333
334 // If a chunk's tag is odd, we want the tags of the surrounding blocks to be
335 // even, and vice versa. Blocks are laid out Size bytes apart, and adding
336 // Size to Ptr will flip the least significant set bit of Size in Ptr, so
337 // that bit will have the pattern 010101... for consecutive blocks, which we
338 // can use to determine which tag mask to use.
339 return 0x5555U << ((Ptr >> SizeClassMap::getSizeLSBByClassId(ClassId)) & 1);
340 }
341
342 NOINLINE void *allocate(uptr Size, Chunk::Origin Origin,
343 uptr Alignment = MinAlignment,
344 bool ZeroContents = false) NO_THREAD_SAFETY_ANALYSIS {
345 initThreadMaybe();
346
347 const Options Options = Primary.Options.load();
348 if (UNLIKELY(Alignment > MaxAlignment)) {
349 if (Options.get(Opt: OptionBit::MayReturnNull))
350 return nullptr;
351 reportAlignmentTooBig(Alignment, MaxAlignment);
352 }
353 if (Alignment < MinAlignment)
354 Alignment = MinAlignment;
355
356#ifdef GWP_ASAN_HOOKS
357 if (UNLIKELY(GuardedAlloc.shouldSample())) {
358 if (void *Ptr = GuardedAlloc.allocate(Size, Alignment)) {
359 Stats.lock();
360 Stats.add(I: StatAllocated, V: GuardedAllocSlotSize);
361 Stats.sub(I: StatFree, V: GuardedAllocSlotSize);
362 Stats.unlock();
363 return Ptr;
364 }
365 }
366#endif // GWP_ASAN_HOOKS
367
368 const FillContentsMode FillContents = ZeroContents ? ZeroFill
369 : TSDRegistry.getDisableMemInit()
370 ? NoFill
371 : Options.getFillContentsMode();
372
373 // If the requested size happens to be 0 (more common than you might think),
374 // allocate MinAlignment bytes on top of the header. Then add the extra
375 // bytes required to fulfill the alignment requirements: we allocate enough
376 // to be sure that there will be an address in the block that will satisfy
377 // the alignment.
378 const uptr NeededSize =
379 roundUp(X: Size, Boundary: MinAlignment) +
380 ((Alignment > MinAlignment) ? Alignment : Chunk::getHeaderSize());
381
382 // Takes care of extravagantly large sizes as well as integer overflows.
383 static_assert(MaxAllowedMallocSize < UINTPTR_MAX - MaxAlignment, "");
384 if (UNLIKELY(Size >= MaxAllowedMallocSize)) {
385 if (Options.get(Opt: OptionBit::MayReturnNull))
386 return nullptr;
387 reportAllocationSizeTooBig(UserSize: Size, TotalSize: NeededSize, MaxSize: MaxAllowedMallocSize);
388 }
389 DCHECK_LE(Size, NeededSize);
390
391 void *Block = nullptr;
392 uptr ClassId = 0;
393 uptr SecondaryBlockEnd = 0;
394 if (LIKELY(PrimaryT::canAllocate(NeededSize))) {
395 ClassId = SizeClassMap::getClassIdBySize(NeededSize);
396 DCHECK_NE(ClassId, 0U);
397 typename TSDRegistryT::ScopedTSD TSD(TSDRegistry);
398 Block = TSD->getSizeClassAllocator().allocate(ClassId);
399 // If the allocation failed, retry in each successively larger class until
400 // it fits. If it fails to fit in the largest class, fallback to the
401 // Secondary.
402 if (UNLIKELY(!Block)) {
403 while (ClassId < SizeClassMap::LargestClassId && !Block)
404 Block = TSD->getSizeClassAllocator().allocate(++ClassId);
405 if (!Block)
406 ClassId = 0;
407 }
408 }
409 if (UNLIKELY(ClassId == 0)) {
410 Block = Secondary.allocate(Options, Size, Alignment, &SecondaryBlockEnd,
411 FillContents);
412 }
413
414 if (UNLIKELY(!Block)) {
415 if (Options.get(Opt: OptionBit::MayReturnNull))
416 return nullptr;
417 printStats();
418 reportOutOfMemory(RequestedSize: NeededSize);
419 }
420
421 const uptr UserPtr = roundUp(
422 X: reinterpret_cast<uptr>(Block) + Chunk::getHeaderSize(), Boundary: Alignment);
423 const uptr SizeOrUnusedBytes =
424 ClassId ? Size : SecondaryBlockEnd - (UserPtr + Size);
425
426 if (LIKELY(!useMemoryTagging<AllocatorConfig>(Options))) {
427 return initChunk(ClassId, Origin, Block, UserPtr, SizeOrUnusedBytes,
428 FillContents);
429 }
430
431 return initChunkWithMemoryTagging(ClassId, Origin, Block, UserPtr, Size,
432 SizeOrUnusedBytes, FillContents);
433 }
434
435 NOINLINE void deallocate(void *Ptr, Chunk::Origin Origin, uptr DeleteSize = 0,
436 UNUSED uptr Alignment = MinAlignment) {
437 if (UNLIKELY(!Ptr))
438 return;
439
440 // For a deallocation, we only ensure minimal initialization, meaning thread
441 // local data will be left uninitialized for now (when using ELF TLS). The
442 // fallback cache will be used instead. This is a workaround for a situation
443 // where the only heap operation performed in a thread would be a free past
444 // the TLS destructors, ending up in initialized thread specific data never
445 // being destroyed properly. Any other heap operation will do a full init.
446 initThreadMaybe(/*MinimalInit=*/MinimalInit: true);
447
448#ifdef GWP_ASAN_HOOKS
449 if (UNLIKELY(GuardedAlloc.pointerIsMine(Ptr))) {
450 GuardedAlloc.deallocate(Ptr);
451 Stats.lock();
452 Stats.add(I: StatFree, V: GuardedAllocSlotSize);
453 Stats.sub(I: StatAllocated, V: GuardedAllocSlotSize);
454 Stats.unlock();
455 return;
456 }
457#endif // GWP_ASAN_HOOKS
458
459 if (UNLIKELY(!isAligned(reinterpret_cast<uptr>(Ptr), MinAlignment)))
460 reportMisalignedPointer(Action: AllocatorAction::Deallocating, Ptr);
461
462 void *TaggedPtr = Ptr;
463 Ptr = getHeaderTaggedPointer(Ptr);
464
465 Chunk::UnpackedHeader Header;
466 Chunk::loadHeader(Cookie, Ptr, NewUnpackedHeader: &Header);
467
468 if (UNLIKELY(Header.State != Chunk::State::Allocated))
469 reportInvalidChunkState(Action: AllocatorAction::Deallocating, Ptr);
470
471 const Options Options = Primary.Options.load();
472 if (Options.get(Opt: OptionBit::DeallocTypeMismatch)) {
473 if (UNLIKELY(Header.OriginOrWasZeroed != Origin)) {
474 // With the exception of memalign'd chunks, that can be still be free'd.
475 if (Header.OriginOrWasZeroed != Chunk::Origin::Memalign ||
476 Origin != Chunk::Origin::Malloc)
477 reportDeallocTypeMismatch(Action: AllocatorAction::Deallocating, Ptr,
478 TypeA: Header.OriginOrWasZeroed, TypeB: Origin);
479 }
480 }
481
482 const uptr Size = getSize(Ptr, Header: &Header);
483 if (DeleteSize && Options.get(Opt: OptionBit::DeleteSizeMismatch)) {
484 if (UNLIKELY(DeleteSize != Size))
485 reportDeleteSizeMismatch(Ptr, Size: DeleteSize, ExpectedSize: Size);
486 }
487
488 quarantineOrDeallocateChunk(Options, TaggedPtr, Header: &Header, Size);
489 }
490
491 void *reallocate(void *OldPtr, uptr NewSize, uptr Alignment = MinAlignment) {
492 initThreadMaybe();
493
494 const Options Options = Primary.Options.load();
495 if (UNLIKELY(NewSize >= MaxAllowedMallocSize)) {
496 if (Options.get(Opt: OptionBit::MayReturnNull))
497 return nullptr;
498 reportAllocationSizeTooBig(UserSize: NewSize, TotalSize: 0, MaxSize: MaxAllowedMallocSize);
499 }
500
501 // The following cases are handled by the C wrappers.
502 DCHECK_NE(OldPtr, nullptr);
503 DCHECK_NE(NewSize, 0);
504
505#ifdef GWP_ASAN_HOOKS
506 if (UNLIKELY(GuardedAlloc.pointerIsMine(OldPtr))) {
507 uptr OldSize = GuardedAlloc.getSize(Ptr: OldPtr);
508 void *NewPtr = allocate(Size: NewSize, Origin: Chunk::Origin::Malloc, Alignment);
509 if (NewPtr)
510 memcpy(dest: NewPtr, src: OldPtr, n: (NewSize < OldSize) ? NewSize : OldSize);
511 GuardedAlloc.deallocate(Ptr: OldPtr);
512 Stats.lock();
513 Stats.add(I: StatFree, V: GuardedAllocSlotSize);
514 Stats.sub(I: StatAllocated, V: GuardedAllocSlotSize);
515 Stats.unlock();
516 return NewPtr;
517 }
518#endif // GWP_ASAN_HOOKS
519
520 void *OldTaggedPtr = OldPtr;
521 OldPtr = getHeaderTaggedPointer(Ptr: OldPtr);
522
523 if (UNLIKELY(!isAligned(reinterpret_cast<uptr>(OldPtr), MinAlignment)))
524 reportMisalignedPointer(Action: AllocatorAction::Reallocating, Ptr: OldPtr);
525
526 Chunk::UnpackedHeader Header;
527 Chunk::loadHeader(Cookie, Ptr: OldPtr, NewUnpackedHeader: &Header);
528
529 if (UNLIKELY(Header.State != Chunk::State::Allocated))
530 reportInvalidChunkState(Action: AllocatorAction::Reallocating, Ptr: OldPtr);
531
532 // Pointer has to be allocated with a malloc-type function. Some
533 // applications think that it is OK to realloc a memalign'ed pointer, which
534 // will trigger this check. It really isn't.
535 if (Options.get(Opt: OptionBit::DeallocTypeMismatch)) {
536 if (UNLIKELY(Header.OriginOrWasZeroed != Chunk::Origin::Malloc))
537 reportDeallocTypeMismatch(Action: AllocatorAction::Reallocating, Ptr: OldPtr,
538 TypeA: Header.OriginOrWasZeroed,
539 TypeB: Chunk::Origin::Malloc);
540 }
541
542 void *BlockBegin = getBlockBegin(Ptr: OldTaggedPtr, Header: &Header);
543 uptr BlockEnd;
544 uptr OldSize;
545 const uptr ClassId = Header.ClassId;
546 if (LIKELY(ClassId)) {
547 BlockEnd = reinterpret_cast<uptr>(BlockBegin) +
548 SizeClassMap::getSizeByClassId(ClassId);
549 OldSize = Header.SizeOrUnusedBytes;
550 } else {
551 BlockEnd = SecondaryT::getBlockEnd(BlockBegin);
552 OldSize = BlockEnd - (reinterpret_cast<uptr>(OldTaggedPtr) +
553 Header.SizeOrUnusedBytes);
554 }
555 // If the new chunk still fits in the previously allocated block (with a
556 // reasonable delta), we just keep the old block, and update the chunk
557 // header to reflect the size change.
558 if (reinterpret_cast<uptr>(OldTaggedPtr) + NewSize <= BlockEnd) {
559 if (NewSize > OldSize || (OldSize - NewSize) < getPageSizeCached()) {
560 // If we have reduced the size, set the extra bytes to the fill value
561 // so that we are ready to grow it again in the future.
562 if (NewSize < OldSize) {
563 const FillContentsMode FillContents =
564 TSDRegistry.getDisableMemInit() ? NoFill
565 : Options.getFillContentsMode();
566 if (FillContents != NoFill) {
567 memset(s: reinterpret_cast<char *>(OldTaggedPtr) + NewSize,
568 c: FillContents == ZeroFill ? 0 : PatternFillByte,
569 n: OldSize - NewSize);
570 }
571 }
572
573 Header.SizeOrUnusedBytes =
574 (ClassId ? NewSize
575 : BlockEnd -
576 (reinterpret_cast<uptr>(OldTaggedPtr) + NewSize)) &
577 Chunk::SizeOrUnusedBytesMask;
578 Chunk::storeHeader(Cookie, Ptr: OldPtr, NewUnpackedHeader: &Header);
579 if (UNLIKELY(useMemoryTagging<AllocatorConfig>(Options))) {
580 if (ClassId) {
581 resizeTaggedChunk(OldPtr: reinterpret_cast<uptr>(OldTaggedPtr) + OldSize,
582 NewPtr: reinterpret_cast<uptr>(OldTaggedPtr) + NewSize,
583 NewSize, BlockEnd: untagPointer(Ptr: BlockEnd));
584 storePrimaryAllocationStackMaybe(Options, Ptr: OldPtr);
585 } else {
586 storeSecondaryAllocationStackMaybe(Options, Ptr: OldPtr, Size: NewSize);
587 }
588 }
589 return OldTaggedPtr;
590 }
591 }
592
593 // Otherwise we allocate a new one, and deallocate the old one. Some
594 // allocators will allocate an even larger chunk (by a fixed factor) to
595 // allow for potential further in-place realloc. The gains of such a trick
596 // are currently unclear.
597 void *NewPtr = allocate(Size: NewSize, Origin: Chunk::Origin::Malloc, Alignment);
598 if (LIKELY(NewPtr)) {
599 memcpy(dest: NewPtr, src: OldTaggedPtr, n: Min(A: NewSize, B: OldSize));
600 quarantineOrDeallocateChunk(Options, TaggedPtr: OldTaggedPtr, Header: &Header, Size: OldSize);
601 }
602 return NewPtr;
603 }
604
605 // TODO(kostyak): disable() is currently best-effort. There are some small
606 // windows of time when an allocation could still succeed after
607 // this function finishes. We will revisit that later.
608 void disable() NO_THREAD_SAFETY_ANALYSIS {
609 initThreadMaybe();
610#ifdef GWP_ASAN_HOOKS
611 GuardedAlloc.disable();
612#endif
613 TSDRegistry.disable();
614 Stats.disable();
615 Quarantine.disable();
616 Primary.disable();
617 Secondary.disable();
618 disableRingBuffer();
619 }
620
621 void enable() NO_THREAD_SAFETY_ANALYSIS {
622 initThreadMaybe();
623 enableRingBuffer();
624 Secondary.enable();
625 Primary.enable();
626 Quarantine.enable();
627 Stats.enable();
628 TSDRegistry.enable();
629#ifdef GWP_ASAN_HOOKS
630 GuardedAlloc.enable();
631#endif
632 }
633
634 // The function returns the amount of bytes required to store the statistics,
635 // which might be larger than the amount of bytes provided. Note that the
636 // statistics buffer is not necessarily constant between calls to this
637 // function. This can be called with a null buffer or zero size for buffer
638 // sizing purposes.
639 uptr getStats(char *Buffer, uptr Size) {
640 ScopedString Str;
641 const uptr Length = getStats(&Str) + 1;
642 if (Length < Size)
643 Size = Length;
644 if (Buffer && Size) {
645 memcpy(dest: Buffer, src: Str.data(), n: Size);
646 Buffer[Size - 1] = '\0';
647 }
648 return Length;
649 }
650
651 void printStats() {
652 ScopedString Str;
653 getStats(&Str);
654 Str.output();
655 }
656
657 void printFragmentationInfo() {
658 ScopedString Str;
659 Primary.getFragmentationInfo(&Str);
660 // Secondary allocator dumps the fragmentation data in getStats().
661 Str.output();
662 }
663
664 void releaseToOS(ReleaseToOS ReleaseType) {
665 initThreadMaybe();
666 if (ReleaseType == ReleaseToOS::ForceAll)
667 drainCaches();
668 Primary.releaseToOS(ReleaseType);
669 Secondary.releaseToOS();
670 }
671
672 // Iterate over all chunks and call a callback for all busy chunks located
673 // within the provided memory range. Said callback must not use this allocator
674 // or a deadlock can ensue. This fits Android's malloc_iterate() needs.
675 void iterateOverChunks(uptr Base, uptr Size, iterate_callback Callback,
676 void *Arg) {
677 initThreadMaybe();
678 if (archSupportsMemoryTagging())
679 Base = untagPointer(Ptr: Base);
680 const uptr From = Base;
681 const uptr To = Base + Size;
682 bool MayHaveTaggedPrimary =
683 allocatorSupportsMemoryTagging<AllocatorConfig>() &&
684 systemSupportsMemoryTagging();
685 auto Lambda = [this, From, To, MayHaveTaggedPrimary, Callback,
686 Arg](uptr Block) {
687 if (Block < From || Block >= To)
688 return;
689 uptr Chunk;
690 Chunk::UnpackedHeader Header;
691 if (MayHaveTaggedPrimary) {
692 // A chunk header can either have a zero tag (tagged primary) or the
693 // header tag (secondary, or untagged primary). We don't know which so
694 // try both.
695 ScopedDisableMemoryTagChecks x;
696 if (!getChunkFromBlock(Block, Chunk: &Chunk, Header: &Header) &&
697 !getChunkFromBlock(Block: addHeaderTag(Block), Chunk: &Chunk, Header: &Header))
698 return;
699 } else {
700 if (!getChunkFromBlock(Block: addHeaderTag(Block), Chunk: &Chunk, Header: &Header))
701 return;
702 }
703 if (Header.State == Chunk::State::Allocated) {
704 uptr TaggedChunk = Chunk;
705 if (allocatorSupportsMemoryTagging<AllocatorConfig>())
706 TaggedChunk = untagPointer(Ptr: TaggedChunk);
707 if (useMemoryTagging<AllocatorConfig>(Primary.Options.load()))
708 TaggedChunk = loadTag(Ptr: Chunk);
709 Callback(TaggedChunk, getSize(Ptr: reinterpret_cast<void *>(Chunk), Header: &Header),
710 Arg);
711 }
712 };
713 Primary.iterateOverBlocks(Lambda);
714 Secondary.iterateOverBlocks(Lambda);
715#ifdef GWP_ASAN_HOOKS
716 GuardedAlloc.iterate(Base: reinterpret_cast<void *>(Base), Size, Cb: Callback, Arg);
717#endif
718 }
719
720 bool canReturnNull() {
721 initThreadMaybe();
722 return Primary.Options.load().get(OptionBit::MayReturnNull);
723 }
724
725 bool setOption(Option O, sptr Value) {
726 initThreadMaybe();
727 if (O == Option::MemtagTuning) {
728 // Enabling odd/even tags involves a tradeoff between use-after-free
729 // detection and buffer overflow detection. Odd/even tags make it more
730 // likely for buffer overflows to be detected by increasing the size of
731 // the guaranteed "red zone" around the allocation, but on the other hand
732 // use-after-free is less likely to be detected because the tag space for
733 // any particular chunk is cut in half. Therefore we use this tuning
734 // setting to control whether odd/even tags are enabled.
735 if (Value == M_MEMTAG_TUNING_BUFFER_OVERFLOW)
736 Primary.Options.set(OptionBit::UseOddEvenTags);
737 else if (Value == M_MEMTAG_TUNING_UAF)
738 Primary.Options.clear(OptionBit::UseOddEvenTags);
739 return true;
740 } else {
741 // We leave it to the various sub-components to decide whether or not they
742 // want to handle the option, but we do not want to short-circuit
743 // execution if one of the setOption was to return false.
744 const bool PrimaryResult = Primary.setOption(O, Value);
745 const bool SecondaryResult = Secondary.setOption(O, Value);
746 const bool RegistryResult = TSDRegistry.setOption(O, Value);
747 return PrimaryResult && SecondaryResult && RegistryResult;
748 }
749 return false;
750 }
751
752 // Return the usable size for a given chunk. Technically we lie, as we just
753 // report the actual size of a chunk. This is done to counteract code actively
754 // writing past the end of a chunk (like sqlite3) when the usable size allows
755 // for it, which then forces realloc to copy the usable size of a chunk as
756 // opposed to its actual size.
757 uptr getUsableSize(const void *Ptr) {
758 if (UNLIKELY(!Ptr))
759 return 0;
760
761 return getAllocSize(Ptr);
762 }
763
764 uptr getAllocSize(const void *Ptr) {
765 initThreadMaybe();
766
767#ifdef GWP_ASAN_HOOKS
768 if (UNLIKELY(GuardedAlloc.pointerIsMine(Ptr)))
769 return GuardedAlloc.getSize(Ptr);
770#endif // GWP_ASAN_HOOKS
771
772 Ptr = getHeaderTaggedPointer(Ptr: const_cast<void *>(Ptr));
773 Chunk::UnpackedHeader Header;
774 Chunk::loadHeader(Cookie, Ptr, NewUnpackedHeader: &Header);
775
776 // Getting the alloc size of a chunk only makes sense if it's allocated.
777 if (UNLIKELY(Header.State != Chunk::State::Allocated))
778 reportInvalidChunkState(Action: AllocatorAction::Sizing, Ptr);
779
780 return getSize(Ptr, Header: &Header);
781 }
782
783 void getStats(StatCounters S) {
784 initThreadMaybe();
785 Stats.get(S);
786 }
787
788 // Returns true if the pointer provided was allocated by the current
789 // allocator instance, which is compliant with tcmalloc's ownership concept.
790 // A corrupted chunk will not be reported as owned, which is WAI.
791 bool isOwned(const void *Ptr) {
792 initThreadMaybe();
793 // If the allocation is not owned, the tags could be wrong.
794 ScopedDisableMemoryTagChecks x(
795 useMemoryTagging<AllocatorConfig>(Primary.Options.load()));
796#ifdef GWP_ASAN_HOOKS
797 if (GuardedAlloc.pointerIsMine(Ptr))
798 return true;
799#endif // GWP_ASAN_HOOKS
800 if (!Ptr || !isAligned(X: reinterpret_cast<uptr>(Ptr), Alignment: MinAlignment))
801 return false;
802 Ptr = getHeaderTaggedPointer(Ptr: const_cast<void *>(Ptr));
803 Chunk::UnpackedHeader Header;
804 return Chunk::isValid(Cookie, Ptr, NewUnpackedHeader: &Header) &&
805 Header.State == Chunk::State::Allocated;
806 }
807
808 bool useMemoryTaggingTestOnly() const {
809 return useMemoryTagging<AllocatorConfig>(Primary.Options.load());
810 }
811 void disableMemoryTagging() {
812 // If we haven't been initialized yet, we need to initialize now in order to
813 // prevent a future call to initThreadMaybe() from enabling memory tagging
814 // based on feature detection. But don't call initThreadMaybe() because it
815 // may end up calling the allocator (via pthread_atfork, via the post-init
816 // callback), which may cause mappings to be created with memory tagging
817 // enabled.
818 TSDRegistry.initOnceMaybe(this);
819 if (allocatorSupportsMemoryTagging<AllocatorConfig>()) {
820 Secondary.disableMemoryTagging();
821 Primary.Options.clear(OptionBit::UseMemoryTagging);
822 }
823 }
824
825 void setTrackAllocationStacks(bool Track) {
826 initThreadMaybe();
827 if (getFlags()->allocation_ring_buffer_size <= 0) {
828 DCHECK(!Primary.Options.load().get(OptionBit::TrackAllocationStacks));
829 return;
830 }
831
832 if (Track) {
833 initRingBufferMaybe();
834 Primary.Options.set(OptionBit::TrackAllocationStacks);
835 } else
836 Primary.Options.clear(OptionBit::TrackAllocationStacks);
837 }
838
839 void setFillContents(FillContentsMode FillContents) {
840 initThreadMaybe();
841 Primary.Options.setFillContentsMode(FillContents);
842 }
843
844 void setAddLargeAllocationSlack(bool AddSlack) {
845 initThreadMaybe();
846 if (AddSlack)
847 Primary.Options.set(OptionBit::AddLargeAllocationSlack);
848 else
849 Primary.Options.clear(OptionBit::AddLargeAllocationSlack);
850 }
851
852 const char *getStackDepotAddress() {
853 initThreadMaybe();
854 AllocationRingBuffer *RB = getRingBuffer();
855 return RB ? reinterpret_cast<char *>(RB->Depot) : nullptr;
856 }
857
858 uptr getStackDepotSize() {
859 initThreadMaybe();
860 AllocationRingBuffer *RB = getRingBuffer();
861 return RB ? RB->StackDepotSize : 0;
862 }
863
864 const char *getRegionInfoArrayAddress() const {
865 return Primary.getRegionInfoArrayAddress();
866 }
867
868 static uptr getRegionInfoArraySize() {
869 return PrimaryT::getRegionInfoArraySize();
870 }
871
872 const char *getRingBufferAddress() {
873 initThreadMaybe();
874 return reinterpret_cast<char *>(getRingBuffer());
875 }
876
877 uptr getRingBufferSize() {
878 initThreadMaybe();
879 AllocationRingBuffer *RB = getRingBuffer();
880 return RB && RB->RingBufferElements
881 ? ringBufferSizeInBytes(RingBufferElements: RB->RingBufferElements)
882 : 0;
883 }
884
885 static const uptr MaxTraceSize = 64;
886
887 static void collectTraceMaybe(const StackDepot *Depot,
888 uintptr_t (&Trace)[MaxTraceSize], u32 Hash) {
889 uptr RingPos, Size;
890 if (!Depot->find(Hash, RingPosPtr: &RingPos, SizePtr: &Size))
891 return;
892 for (unsigned I = 0; I != Size && I != MaxTraceSize; ++I)
893 Trace[I] = static_cast<uintptr_t>(Depot->at(RingPos: RingPos + I));
894 }
895
896 static void getErrorInfo(struct scudo_error_info *ErrorInfo,
897 uintptr_t FaultAddr, const char *DepotPtr,
898 size_t DepotSize, const char *RegionInfoPtr,
899 const char *RingBufferPtr, size_t RingBufferSize,
900 const char *Memory, const char *MemoryTags,
901 uintptr_t MemoryAddr, size_t MemorySize) {
902 // N.B. we need to support corrupted data in any of the buffers here. We get
903 // this information from an external process (the crashing process) that
904 // should not be able to crash the crash dumper (crash_dump on Android).
905 // See also the get_error_info_fuzzer.
906 *ErrorInfo = {};
907 if (!allocatorSupportsMemoryTagging<AllocatorConfig>() ||
908 MemoryAddr + MemorySize < MemoryAddr)
909 return;
910
911 const StackDepot *Depot = nullptr;
912 if (DepotPtr) {
913 // check for corrupted StackDepot. First we need to check whether we can
914 // read the metadata, then whether the metadata matches the size.
915 if (DepotSize < sizeof(*Depot))
916 return;
917 Depot = reinterpret_cast<const StackDepot *>(DepotPtr);
918 if (!Depot->isValid(BufSize: DepotSize))
919 return;
920 }
921
922 size_t NextErrorReport = 0;
923
924 // Check for OOB in the current block and the two surrounding blocks. Beyond
925 // that, UAF is more likely.
926 if (extractTag(Ptr: FaultAddr) != 0)
927 getInlineErrorInfo(ErrorInfo, NextErrorReport, FaultAddr, Depot,
928 RegionInfoPtr, Memory, MemoryTags, MemoryAddr,
929 MemorySize, MinDistance: 0, MaxDistance: 2);
930
931 // Check the ring buffer. For primary allocations this will only find UAF;
932 // for secondary allocations we can find either UAF or OOB.
933 getRingBufferErrorInfo(ErrorInfo, NextErrorReport, FaultAddr, Depot,
934 RingBufferPtr, RingBufferSize);
935
936 // Check for OOB in the 28 blocks surrounding the 3 we checked earlier.
937 // Beyond that we are likely to hit false positives.
938 if (extractTag(Ptr: FaultAddr) != 0)
939 getInlineErrorInfo(ErrorInfo, NextErrorReport, FaultAddr, Depot,
940 RegionInfoPtr, Memory, MemoryTags, MemoryAddr,
941 MemorySize, MinDistance: 2, MaxDistance: 16);
942 }
943
944private:
945 typedef typename PrimaryT::SizeClassMap SizeClassMap;
946
947 static const uptr MinAlignmentLog = SCUDO_MIN_ALIGNMENT_LOG;
948 static const uptr MaxAlignmentLog = 24U; // 16 MB seems reasonable.
949 static const uptr MinAlignment = 1UL << MinAlignmentLog;
950 static const uptr MaxAlignment = 1UL << MaxAlignmentLog;
951 static const uptr MaxAllowedMallocSize =
952 FIRST_32_SECOND_64(1UL << 31, 1ULL << 40);
953
954 static_assert(MinAlignment >= sizeof(Chunk::PackedHeader),
955 "Minimal alignment must at least cover a chunk header.");
956 static_assert(!allocatorSupportsMemoryTagging<AllocatorConfig>() ||
957 MinAlignment >= archMemoryTagGranuleSize(),
958 "");
959
960 static const u32 BlockMarker = 0x44554353U;
961
962 // These are indexes into an "array" of 32-bit values that store information
963 // inline with a chunk that is relevant to diagnosing memory tag faults, where
964 // 0 corresponds to the address of the user memory. This means that only
965 // negative indexes may be used. The smallest index that may be used is -2,
966 // which corresponds to 8 bytes before the user memory, because the chunk
967 // header size is 8 bytes and in allocators that support memory tagging the
968 // minimum alignment is at least the tag granule size (16 on aarch64).
969 static const sptr MemTagAllocationTraceIndex = -2;
970 static const sptr MemTagAllocationTidIndex = -1;
971
972 u32 Cookie = 0;
973 u32 QuarantineMaxChunkSize = 0;
974
975 GlobalStats Stats;
976 PrimaryT Primary;
977 SecondaryT Secondary;
978 QuarantineT Quarantine;
979 TSDRegistryT TSDRegistry;
980 pthread_once_t PostInitNonce = PTHREAD_ONCE_INIT;
981
982#ifdef GWP_ASAN_HOOKS
983 gwp_asan::GuardedPoolAllocator GuardedAlloc;
984 uptr GuardedAllocSlotSize = 0;
985#endif // GWP_ASAN_HOOKS
986
987 struct AllocationRingBuffer {
988 struct Entry {
989 atomic_uptr Ptr;
990 atomic_uptr AllocationSize;
991 atomic_u32 AllocationTrace;
992 atomic_u32 AllocationTid;
993 atomic_u32 DeallocationTrace;
994 atomic_u32 DeallocationTid;
995 };
996 StackDepot *Depot = nullptr;
997 uptr StackDepotSize = 0;
998 MemMapT RawRingBufferMap;
999 MemMapT RawStackDepotMap;
1000 u32 RingBufferElements = 0;
1001 atomic_uptr Pos;
1002 // An array of Size (at least one) elements of type Entry is immediately
1003 // following to this struct.
1004 };
1005 static_assert(sizeof(AllocationRingBuffer) %
1006 alignof(typename AllocationRingBuffer::Entry) ==
1007 0,
1008 "invalid alignment");
1009
1010 // Lock to initialize the RingBuffer
1011 HybridMutex RingBufferInitLock;
1012
1013 // Pointer to memory mapped area starting with AllocationRingBuffer struct,
1014 // and immediately followed by Size elements of type Entry.
1015 atomic_uptr RingBufferAddress = {};
1016
1017 AllocationRingBuffer *getRingBuffer() {
1018 return reinterpret_cast<AllocationRingBuffer *>(
1019 atomic_load(A: &RingBufferAddress, MO: memory_order_acquire));
1020 }
1021
1022 // The following might get optimized out by the compiler.
1023 NOINLINE void performSanityChecks() {
1024 // Verify that the header offset field can hold the maximum offset. In the
1025 // case of the Secondary allocator, it takes care of alignment and the
1026 // offset will always be small. In the case of the Primary, the worst case
1027 // scenario happens in the last size class, when the backend allocation
1028 // would already be aligned on the requested alignment, which would happen
1029 // to be the maximum alignment that would fit in that size class. As a
1030 // result, the maximum offset will be at most the maximum alignment for the
1031 // last size class minus the header size, in multiples of MinAlignment.
1032 Chunk::UnpackedHeader Header = {};
1033 const uptr MaxPrimaryAlignment = 1UL << getMostSignificantSetBitIndex(
1034 SizeClassMap::MaxSize - MinAlignment);
1035 const uptr MaxOffset =
1036 (MaxPrimaryAlignment - Chunk::getHeaderSize()) >> MinAlignmentLog;
1037 Header.Offset = MaxOffset & Chunk::OffsetMask;
1038 if (UNLIKELY(Header.Offset != MaxOffset))
1039 reportSanityCheckError(Field: "offset");
1040
1041 // Verify that we can fit the maximum size or amount of unused bytes in the
1042 // header. Given that the Secondary fits the allocation to a page, the worst
1043 // case scenario happens in the Primary. It will depend on the second to
1044 // last and last class sizes, as well as the dynamic base for the Primary.
1045 // The following is an over-approximation that works for our needs.
1046 const uptr MaxSizeOrUnusedBytes = SizeClassMap::MaxSize - 1;
1047 Header.SizeOrUnusedBytes = MaxSizeOrUnusedBytes;
1048 if (UNLIKELY(Header.SizeOrUnusedBytes != MaxSizeOrUnusedBytes))
1049 reportSanityCheckError(Field: "size (or unused bytes)");
1050
1051 const uptr LargestClassId = SizeClassMap::LargestClassId;
1052 Header.ClassId = LargestClassId;
1053 if (UNLIKELY(Header.ClassId != LargestClassId))
1054 reportSanityCheckError(Field: "class ID");
1055 }
1056
1057 static inline void *getBlockBegin(const void *Ptr,
1058 Chunk::UnpackedHeader *Header) {
1059 return reinterpret_cast<void *>(
1060 reinterpret_cast<uptr>(Ptr) - Chunk::getHeaderSize() -
1061 (static_cast<uptr>(Header->Offset) << MinAlignmentLog));
1062 }
1063
1064 // Return the size of a chunk as requested during its allocation.
1065 inline uptr getSize(const void *Ptr, Chunk::UnpackedHeader *Header) {
1066 const uptr SizeOrUnusedBytes = Header->SizeOrUnusedBytes;
1067 if (LIKELY(Header->ClassId))
1068 return SizeOrUnusedBytes;
1069 if (allocatorSupportsMemoryTagging<AllocatorConfig>())
1070 Ptr = untagPointer(Ptr: const_cast<void *>(Ptr));
1071 return SecondaryT::getBlockEnd(getBlockBegin(Ptr, Header)) -
1072 reinterpret_cast<uptr>(Ptr) - SizeOrUnusedBytes;
1073 }
1074
1075 ALWAYS_INLINE void *initChunk(const uptr ClassId, const Chunk::Origin Origin,
1076 void *Block, const uptr UserPtr,
1077 const uptr SizeOrUnusedBytes,
1078 const FillContentsMode FillContents) {
1079 // Compute the default pointer before adding the header tag
1080 const uptr DefaultAlignedPtr =
1081 reinterpret_cast<uptr>(Block) + Chunk::getHeaderSize();
1082
1083 Block = addHeaderTag(Block);
1084 // Only do content fill when it's from primary allocator because secondary
1085 // allocator has filled the content.
1086 if (ClassId != 0 && UNLIKELY(FillContents != NoFill)) {
1087 // This condition is not necessarily unlikely, but since memset is
1088 // costly, we might as well mark it as such.
1089 memset(Block, FillContents == ZeroFill ? 0 : PatternFillByte,
1090 PrimaryT::getSizeByClassId(ClassId));
1091 }
1092
1093 Chunk::UnpackedHeader Header = {};
1094
1095 if (UNLIKELY(DefaultAlignedPtr != UserPtr)) {
1096 const uptr Offset = UserPtr - DefaultAlignedPtr;
1097 DCHECK_GE(Offset, 2 * sizeof(u32));
1098 // The BlockMarker has no security purpose, but is specifically meant for
1099 // the chunk iteration function that can be used in debugging situations.
1100 // It is the only situation where we have to locate the start of a chunk
1101 // based on its block address.
1102 reinterpret_cast<u32 *>(Block)[0] = BlockMarker;
1103 reinterpret_cast<u32 *>(Block)[1] = static_cast<u32>(Offset);
1104 Header.Offset = (Offset >> MinAlignmentLog) & Chunk::OffsetMask;
1105 }
1106
1107 Header.ClassId = ClassId & Chunk::ClassIdMask;
1108 Header.State = Chunk::State::Allocated;
1109 Header.OriginOrWasZeroed = Origin & Chunk::OriginMask;
1110 Header.SizeOrUnusedBytes = SizeOrUnusedBytes & Chunk::SizeOrUnusedBytesMask;
1111 Chunk::storeHeader(Cookie, Ptr: reinterpret_cast<void *>(addHeaderTag(UserPtr)),
1112 NewUnpackedHeader: &Header);
1113
1114 return reinterpret_cast<void *>(UserPtr);
1115 }
1116
1117 NOINLINE void *
1118 initChunkWithMemoryTagging(const uptr ClassId, const Chunk::Origin Origin,
1119 void *Block, const uptr UserPtr, const uptr Size,
1120 const uptr SizeOrUnusedBytes,
1121 const FillContentsMode FillContents) {
1122 const Options Options = Primary.Options.load();
1123 DCHECK(useMemoryTagging<AllocatorConfig>(Options));
1124
1125 // Compute the default pointer before adding the header tag
1126 const uptr DefaultAlignedPtr =
1127 reinterpret_cast<uptr>(Block) + Chunk::getHeaderSize();
1128
1129 void *Ptr = reinterpret_cast<void *>(UserPtr);
1130 void *TaggedPtr = Ptr;
1131
1132 if (LIKELY(ClassId)) {
1133 // Init the primary chunk.
1134 //
1135 // We only need to zero or tag the contents for Primary backed
1136 // allocations. We only set tags for primary allocations in order to avoid
1137 // faulting potentially large numbers of pages for large secondary
1138 // allocations. We assume that guard pages are enough to protect these
1139 // allocations.
1140 //
1141 // FIXME: When the kernel provides a way to set the background tag of a
1142 // mapping, we should be able to tag secondary allocations as well.
1143 //
1144 // When memory tagging is enabled, zeroing the contents is done as part of
1145 // setting the tag.
1146
1147 Chunk::UnpackedHeader Header;
1148 const uptr BlockSize = PrimaryT::getSizeByClassId(ClassId);
1149 const uptr BlockUptr = reinterpret_cast<uptr>(Block);
1150 const uptr BlockEnd = BlockUptr + BlockSize;
1151 // If possible, try to reuse the UAF tag that was set by deallocate().
1152 // For simplicity, only reuse tags if we have the same start address as
1153 // the previous allocation. This handles the majority of cases since
1154 // most allocations will not be more aligned than the minimum alignment.
1155 //
1156 // We need to handle situations involving reclaimed chunks, and retag
1157 // the reclaimed portions if necessary. In the case where the chunk is
1158 // fully reclaimed, the chunk's header will be zero, which will trigger
1159 // the code path for new mappings and invalid chunks that prepares the
1160 // chunk from scratch. There are three possibilities for partial
1161 // reclaiming:
1162 //
1163 // (1) Header was reclaimed, data was partially reclaimed.
1164 // (2) Header was not reclaimed, all data was reclaimed (e.g. because
1165 // data started on a page boundary).
1166 // (3) Header was not reclaimed, data was partially reclaimed.
1167 //
1168 // Case (1) will be handled in the same way as for full reclaiming,
1169 // since the header will be zero.
1170 //
1171 // We can detect case (2) by loading the tag from the start
1172 // of the chunk. If it is zero, it means that either all data was
1173 // reclaimed (since we never use zero as the chunk tag), or that the
1174 // previous allocation was of size zero. Either way, we need to prepare
1175 // a new chunk from scratch.
1176 //
1177 // We can detect case (3) by moving to the next page (if covered by the
1178 // chunk) and loading the tag of its first granule. If it is zero, it
1179 // means that all following pages may need to be retagged. On the other
1180 // hand, if it is nonzero, we can assume that all following pages are
1181 // still tagged, according to the logic that if any of the pages
1182 // following the next page were reclaimed, the next page would have been
1183 // reclaimed as well.
1184 uptr TaggedUserPtr;
1185 uptr PrevUserPtr;
1186 if (getChunkFromBlock(Block: BlockUptr, Chunk: &PrevUserPtr, Header: &Header) &&
1187 PrevUserPtr == UserPtr &&
1188 (TaggedUserPtr = loadTag(Ptr: UserPtr)) != UserPtr) {
1189 uptr PrevEnd = TaggedUserPtr + Header.SizeOrUnusedBytes;
1190 const uptr NextPage = roundUp(X: TaggedUserPtr, Boundary: getPageSizeCached());
1191 if (NextPage < PrevEnd && loadTag(Ptr: NextPage) != NextPage)
1192 PrevEnd = NextPage;
1193 TaggedPtr = reinterpret_cast<void *>(TaggedUserPtr);
1194 resizeTaggedChunk(OldPtr: PrevEnd, NewPtr: TaggedUserPtr + Size, NewSize: Size, BlockEnd);
1195 if (UNLIKELY(FillContents != NoFill && !Header.OriginOrWasZeroed)) {
1196 // If an allocation needs to be zeroed (i.e. calloc) we can normally
1197 // avoid zeroing the memory now since we can rely on memory having
1198 // been zeroed on free, as this is normally done while setting the
1199 // UAF tag. But if tagging was disabled per-thread when the memory
1200 // was freed, it would not have been retagged and thus zeroed, and
1201 // therefore it needs to be zeroed now.
1202 memset(s: TaggedPtr, c: 0,
1203 n: Min(A: Size, B: roundUp(X: PrevEnd - TaggedUserPtr,
1204 Boundary: archMemoryTagGranuleSize())));
1205 } else if (Size) {
1206 // Clear any stack metadata that may have previously been stored in
1207 // the chunk data.
1208 memset(s: TaggedPtr, c: 0, n: archMemoryTagGranuleSize());
1209 }
1210 } else {
1211 const uptr OddEvenMask =
1212 computeOddEvenMaskForPointerMaybe(Options, Ptr: BlockUptr, ClassId);
1213 TaggedPtr = prepareTaggedChunk(Ptr, Size, ExcludeMask: OddEvenMask, BlockEnd);
1214 }
1215 storePrimaryAllocationStackMaybe(Options, Ptr);
1216 } else {
1217 // Init the secondary chunk.
1218
1219 Block = addHeaderTag(Block);
1220 Ptr = addHeaderTag(Ptr);
1221 storeTags(Begin: reinterpret_cast<uptr>(Block), End: reinterpret_cast<uptr>(Ptr));
1222 storeSecondaryAllocationStackMaybe(Options, Ptr, Size);
1223 }
1224
1225 Chunk::UnpackedHeader Header = {};
1226
1227 if (UNLIKELY(DefaultAlignedPtr != UserPtr)) {
1228 const uptr Offset = UserPtr - DefaultAlignedPtr;
1229 DCHECK_GE(Offset, 2 * sizeof(u32));
1230 // The BlockMarker has no security purpose, but is specifically meant for
1231 // the chunk iteration function that can be used in debugging situations.
1232 // It is the only situation where we have to locate the start of a chunk
1233 // based on its block address.
1234 reinterpret_cast<u32 *>(Block)[0] = BlockMarker;
1235 reinterpret_cast<u32 *>(Block)[1] = static_cast<u32>(Offset);
1236 Header.Offset = (Offset >> MinAlignmentLog) & Chunk::OffsetMask;
1237 }
1238
1239 Header.ClassId = ClassId & Chunk::ClassIdMask;
1240 Header.State = Chunk::State::Allocated;
1241 Header.OriginOrWasZeroed = Origin & Chunk::OriginMask;
1242 Header.SizeOrUnusedBytes = SizeOrUnusedBytes & Chunk::SizeOrUnusedBytesMask;
1243 Chunk::storeHeader(Cookie, Ptr, NewUnpackedHeader: &Header);
1244
1245 return TaggedPtr;
1246 }
1247
1248 void quarantineOrDeallocateChunk(const Options &Options, void *TaggedPtr,
1249 Chunk::UnpackedHeader *Header,
1250 uptr Size) NO_THREAD_SAFETY_ANALYSIS {
1251 void *Ptr = getHeaderTaggedPointer(Ptr: TaggedPtr);
1252 // If the quarantine is disabled, the actual size of a chunk is 0 or larger
1253 // than the maximum allowed, we return a chunk directly to the backend.
1254 // This purposefully underflows for Size == 0.
1255 const bool BypassQuarantine = !Quarantine.getCacheSize() ||
1256 ((Size - 1) >= QuarantineMaxChunkSize) ||
1257 !Header->ClassId;
1258 if (BypassQuarantine)
1259 Header->State = Chunk::State::Available;
1260 else
1261 Header->State = Chunk::State::Quarantined;
1262
1263 if (LIKELY(!useMemoryTagging<AllocatorConfig>(Options)))
1264 Header->OriginOrWasZeroed = 0U;
1265 else {
1266 Header->OriginOrWasZeroed =
1267 Header->ClassId && !TSDRegistry.getDisableMemInit();
1268 }
1269
1270 Chunk::storeHeader(Cookie, Ptr, NewUnpackedHeader: Header);
1271
1272 if (BypassQuarantine) {
1273 void *BlockBegin;
1274 if (LIKELY(!useMemoryTagging<AllocatorConfig>(Options))) {
1275 // Must do this after storeHeader because loadHeader uses a tagged ptr.
1276 if (allocatorSupportsMemoryTagging<AllocatorConfig>())
1277 Ptr = untagPointer(Ptr);
1278 BlockBegin = getBlockBegin(Ptr, Header);
1279 } else {
1280 BlockBegin = retagBlock(Options, TaggedPtr, Ptr, Header, Size, BypassQuarantine: true);
1281 }
1282
1283 const uptr ClassId = Header->ClassId;
1284 if (LIKELY(ClassId)) {
1285 bool CacheDrained;
1286 {
1287 typename TSDRegistryT::ScopedTSD TSD(TSDRegistry);
1288 CacheDrained =
1289 TSD->getSizeClassAllocator().deallocate(ClassId, BlockBegin);
1290 }
1291 // When we have drained some blocks back to the Primary from TSD, that
1292 // implies that we may have the chance to release some pages as well.
1293 // Note that in order not to block other thread's accessing the TSD,
1294 // release the TSD first then try the page release.
1295 if (CacheDrained)
1296 Primary.tryReleaseToOS(ClassId, ReleaseToOS::Normal);
1297 } else {
1298 Secondary.deallocate(Options, BlockBegin);
1299 }
1300 } else {
1301 if (UNLIKELY(useMemoryTagging<AllocatorConfig>(Options)))
1302 retagBlock(Options, TaggedPtr, Ptr, Header, Size, BypassQuarantine: false);
1303 typename TSDRegistryT::ScopedTSD TSD(TSDRegistry);
1304 Quarantine.put(&TSD->getQuarantineCache(),
1305 QuarantineCallback(*this, TSD->getSizeClassAllocator()),
1306 Ptr, Size);
1307 }
1308 }
1309
1310 NOINLINE void *retagBlock(const Options &Options, void *TaggedPtr, void *&Ptr,
1311 Chunk::UnpackedHeader *Header, const uptr Size,
1312 bool BypassQuarantine) {
1313 DCHECK(useMemoryTagging<AllocatorConfig>(Options));
1314
1315 const u8 PrevTag = extractTag(Ptr: reinterpret_cast<uptr>(TaggedPtr));
1316 storeDeallocationStackMaybe(Options, Ptr, PrevTag, Size);
1317 if (Header->ClassId && !TSDRegistry.getDisableMemInit()) {
1318 uptr TaggedBegin, TaggedEnd;
1319 const uptr OddEvenMask = computeOddEvenMaskForPointerMaybe(
1320 Options, Ptr: reinterpret_cast<uptr>(getBlockBegin(Ptr, Header)),
1321 ClassId: Header->ClassId);
1322 // Exclude the previous tag so that immediate use after free is
1323 // detected 100% of the time.
1324 setRandomTag(Ptr, Size, ExcludeMask: OddEvenMask | (1UL << PrevTag), TaggedBegin: &TaggedBegin,
1325 TaggedEnd: &TaggedEnd);
1326 }
1327
1328 Ptr = untagPointer(Ptr);
1329 void *BlockBegin = getBlockBegin(Ptr, Header);
1330 if (BypassQuarantine && !Header->ClassId) {
1331 storeTags(Begin: reinterpret_cast<uptr>(BlockBegin),
1332 End: reinterpret_cast<uptr>(Ptr));
1333 }
1334
1335 return BlockBegin;
1336 }
1337
1338 bool getChunkFromBlock(uptr Block, uptr *Chunk,
1339 Chunk::UnpackedHeader *Header) {
1340 *Chunk =
1341 Block + getChunkOffsetFromBlock(Block: reinterpret_cast<const char *>(Block));
1342 return Chunk::isValid(Cookie, Ptr: reinterpret_cast<void *>(*Chunk), NewUnpackedHeader: Header);
1343 }
1344
1345 static uptr getChunkOffsetFromBlock(const char *Block) {
1346 u32 Offset = 0;
1347 if (reinterpret_cast<const u32 *>(Block)[0] == BlockMarker)
1348 Offset = reinterpret_cast<const u32 *>(Block)[1];
1349 return Offset + Chunk::getHeaderSize();
1350 }
1351
1352 // Set the tag of the granule past the end of the allocation to 0, to catch
1353 // linear overflows even if a previous larger allocation used the same block
1354 // and tag. Only do this if the granule past the end is in our block, because
1355 // this would otherwise lead to a SEGV if the allocation covers the entire
1356 // block and our block is at the end of a mapping. The tag of the next block's
1357 // header granule will be set to 0, so it will serve the purpose of catching
1358 // linear overflows in this case.
1359 //
1360 // For allocations of size 0 we do not end up storing the address tag to the
1361 // memory tag space, which getInlineErrorInfo() normally relies on to match
1362 // address tags against chunks. To allow matching in this case we store the
1363 // address tag in the first byte of the chunk.
1364 void storeEndMarker(uptr End, uptr Size, uptr BlockEnd) {
1365 DCHECK_EQ(BlockEnd, untagPointer(BlockEnd));
1366 uptr UntaggedEnd = untagPointer(Ptr: End);
1367 if (UntaggedEnd != BlockEnd) {
1368 storeTag(Ptr: UntaggedEnd);
1369 if (Size == 0)
1370 *reinterpret_cast<u8 *>(UntaggedEnd) = extractTag(Ptr: End);
1371 }
1372 }
1373
1374 void *prepareTaggedChunk(void *Ptr, uptr Size, uptr ExcludeMask,
1375 uptr BlockEnd) {
1376 // Prepare the granule before the chunk to store the chunk header by setting
1377 // its tag to 0. Normally its tag will already be 0, but in the case where a
1378 // chunk holding a low alignment allocation is reused for a higher alignment
1379 // allocation, the chunk may already have a non-zero tag from the previous
1380 // allocation.
1381 storeTag(Ptr: reinterpret_cast<uptr>(Ptr) - archMemoryTagGranuleSize());
1382
1383 uptr TaggedBegin, TaggedEnd;
1384 setRandomTag(Ptr, Size, ExcludeMask, TaggedBegin: &TaggedBegin, TaggedEnd: &TaggedEnd);
1385
1386 storeEndMarker(End: TaggedEnd, Size, BlockEnd);
1387 return reinterpret_cast<void *>(TaggedBegin);
1388 }
1389
1390 void resizeTaggedChunk(uptr OldPtr, uptr NewPtr, uptr NewSize,
1391 uptr BlockEnd) {
1392 uptr RoundOldPtr = roundUp(X: OldPtr, Boundary: archMemoryTagGranuleSize());
1393 uptr RoundNewPtr;
1394 if (RoundOldPtr >= NewPtr) {
1395 // If the allocation is shrinking we just need to set the tag past the end
1396 // of the allocation to 0. See explanation in storeEndMarker() above.
1397 RoundNewPtr = roundUp(X: NewPtr, Boundary: archMemoryTagGranuleSize());
1398 } else {
1399 // Set the memory tag of the region
1400 // [RoundOldPtr, roundUp(NewPtr, archMemoryTagGranuleSize()))
1401 // to the pointer tag stored in OldPtr.
1402 RoundNewPtr = storeTags(Begin: RoundOldPtr, End: NewPtr);
1403 }
1404 storeEndMarker(End: RoundNewPtr, Size: NewSize, BlockEnd);
1405 }
1406
1407 void storePrimaryAllocationStackMaybe(const Options &Options, void *Ptr) {
1408 if (!UNLIKELY(Options.get(OptionBit::TrackAllocationStacks)))
1409 return;
1410 AllocationRingBuffer *RB = getRingBuffer();
1411 if (!RB)
1412 return;
1413 auto *Ptr32 = reinterpret_cast<u32 *>(Ptr);
1414 Ptr32[MemTagAllocationTraceIndex] = collectStackTrace(Depot: RB->Depot);
1415 Ptr32[MemTagAllocationTidIndex] = getThreadID();
1416 }
1417
1418 void storeRingBufferEntry(AllocationRingBuffer *RB, void *Ptr,
1419 u32 AllocationTrace, u32 AllocationTid,
1420 uptr AllocationSize, u32 DeallocationTrace,
1421 u32 DeallocationTid) {
1422 uptr Pos = atomic_fetch_add(&RB->Pos, 1, memory_order_relaxed);
1423 typename AllocationRingBuffer::Entry *Entry =
1424 getRingBufferEntry(RB, Pos % RB->RingBufferElements);
1425
1426 // First invalidate our entry so that we don't attempt to interpret a
1427 // partially written state in getSecondaryErrorInfo(). The fences below
1428 // ensure that the compiler does not move the stores to Ptr in between the
1429 // stores to the other fields.
1430 atomic_store_relaxed(&Entry->Ptr, 0);
1431
1432 __atomic_signal_fence(__ATOMIC_SEQ_CST);
1433 atomic_store_relaxed(&Entry->AllocationTrace, AllocationTrace);
1434 atomic_store_relaxed(&Entry->AllocationTid, AllocationTid);
1435 atomic_store_relaxed(&Entry->AllocationSize, AllocationSize);
1436 atomic_store_relaxed(&Entry->DeallocationTrace, DeallocationTrace);
1437 atomic_store_relaxed(&Entry->DeallocationTid, DeallocationTid);
1438 __atomic_signal_fence(__ATOMIC_SEQ_CST);
1439
1440 atomic_store_relaxed(&Entry->Ptr, reinterpret_cast<uptr>(Ptr));
1441 }
1442
1443 void storeSecondaryAllocationStackMaybe(const Options &Options, void *Ptr,
1444 uptr Size) {
1445 if (!UNLIKELY(Options.get(OptionBit::TrackAllocationStacks)))
1446 return;
1447 AllocationRingBuffer *RB = getRingBuffer();
1448 if (!RB)
1449 return;
1450 u32 Trace = collectStackTrace(Depot: RB->Depot);
1451 u32 Tid = getThreadID();
1452
1453 auto *Ptr32 = reinterpret_cast<u32 *>(Ptr);
1454 Ptr32[MemTagAllocationTraceIndex] = Trace;
1455 Ptr32[MemTagAllocationTidIndex] = Tid;
1456
1457 storeRingBufferEntry(RB, Ptr: untagPointer(Ptr), AllocationTrace: Trace, AllocationTid: Tid, AllocationSize: Size, DeallocationTrace: 0, DeallocationTid: 0);
1458 }
1459
1460 void storeDeallocationStackMaybe(const Options &Options, void *Ptr,
1461 u8 PrevTag, uptr Size) {
1462 if (!UNLIKELY(Options.get(OptionBit::TrackAllocationStacks)))
1463 return;
1464 AllocationRingBuffer *RB = getRingBuffer();
1465 if (!RB)
1466 return;
1467 auto *Ptr32 = reinterpret_cast<u32 *>(Ptr);
1468 u32 AllocationTrace = Ptr32[MemTagAllocationTraceIndex];
1469 u32 AllocationTid = Ptr32[MemTagAllocationTidIndex];
1470
1471 u32 DeallocationTrace = collectStackTrace(Depot: RB->Depot);
1472 u32 DeallocationTid = getThreadID();
1473
1474 storeRingBufferEntry(RB, Ptr: addFixedTag(Ptr: untagPointer(Ptr), Tag: PrevTag),
1475 AllocationTrace, AllocationTid, AllocationSize: Size,
1476 DeallocationTrace, DeallocationTid);
1477 }
1478
1479 static const size_t NumErrorReports =
1480 sizeof(((scudo_error_info *)nullptr)->reports) /
1481 sizeof(((scudo_error_info *)nullptr)->reports[0]);
1482
1483 static void getInlineErrorInfo(struct scudo_error_info *ErrorInfo,
1484 size_t &NextErrorReport, uintptr_t FaultAddr,
1485 const StackDepot *Depot,
1486 const char *RegionInfoPtr, const char *Memory,
1487 const char *MemoryTags, uintptr_t MemoryAddr,
1488 size_t MemorySize, size_t MinDistance,
1489 size_t MaxDistance) {
1490 uptr UntaggedFaultAddr = untagPointer(Ptr: FaultAddr);
1491 u8 FaultAddrTag = extractTag(Ptr: FaultAddr);
1492 BlockInfo Info =
1493 PrimaryT::findNearestBlock(RegionInfoPtr, UntaggedFaultAddr);
1494
1495 auto GetGranule = [&](uptr Addr, const char **Data, uint8_t *Tag) -> bool {
1496 if (Addr < MemoryAddr || Addr + archMemoryTagGranuleSize() < Addr ||
1497 Addr + archMemoryTagGranuleSize() > MemoryAddr + MemorySize)
1498 return false;
1499 *Data = &Memory[Addr - MemoryAddr];
1500 *Tag = static_cast<u8>(
1501 MemoryTags[(Addr - MemoryAddr) / archMemoryTagGranuleSize()]);
1502 return true;
1503 };
1504
1505 auto ReadBlock = [&](uptr Addr, uptr *ChunkAddr,
1506 Chunk::UnpackedHeader *Header, const u32 **Data,
1507 u8 *Tag) {
1508 const char *BlockBegin;
1509 u8 BlockBeginTag;
1510 if (!GetGranule(Addr, &BlockBegin, &BlockBeginTag))
1511 return false;
1512 uptr ChunkOffset = getChunkOffsetFromBlock(Block: BlockBegin);
1513 *ChunkAddr = Addr + ChunkOffset;
1514
1515 const char *ChunkBegin;
1516 if (!GetGranule(*ChunkAddr, &ChunkBegin, Tag))
1517 return false;
1518 *Header = *reinterpret_cast<const Chunk::UnpackedHeader *>(
1519 ChunkBegin - Chunk::getHeaderSize());
1520 *Data = reinterpret_cast<const u32 *>(ChunkBegin);
1521
1522 // Allocations of size 0 will have stashed the tag in the first byte of
1523 // the chunk, see storeEndMarker().
1524 if (Header->SizeOrUnusedBytes == 0)
1525 *Tag = static_cast<u8>(*ChunkBegin);
1526
1527 return true;
1528 };
1529
1530 if (NextErrorReport == NumErrorReports)
1531 return;
1532
1533 auto CheckOOB = [&](uptr BlockAddr) {
1534 if (BlockAddr < Info.RegionBegin || BlockAddr >= Info.RegionEnd)
1535 return false;
1536
1537 uptr ChunkAddr;
1538 Chunk::UnpackedHeader Header;
1539 const u32 *Data;
1540 uint8_t Tag;
1541 if (!ReadBlock(BlockAddr, &ChunkAddr, &Header, &Data, &Tag) ||
1542 Header.State != Chunk::State::Allocated || Tag != FaultAddrTag)
1543 return false;
1544
1545 auto *R = &ErrorInfo->reports[NextErrorReport++];
1546 R->error_type =
1547 UntaggedFaultAddr < ChunkAddr ? BUFFER_UNDERFLOW : BUFFER_OVERFLOW;
1548 R->allocation_address = ChunkAddr;
1549 R->allocation_size = Header.SizeOrUnusedBytes;
1550 if (Depot) {
1551 collectTraceMaybe(Depot, Trace&: R->allocation_trace,
1552 Hash: Data[MemTagAllocationTraceIndex]);
1553 }
1554 R->allocation_tid = Data[MemTagAllocationTidIndex];
1555 return NextErrorReport == NumErrorReports;
1556 };
1557
1558 if (MinDistance == 0 && CheckOOB(Info.BlockBegin))
1559 return;
1560
1561 for (size_t I = Max<size_t>(A: MinDistance, B: 1); I != MaxDistance; ++I)
1562 if (CheckOOB(Info.BlockBegin + I * Info.BlockSize) ||
1563 CheckOOB(Info.BlockBegin - I * Info.BlockSize))
1564 return;
1565 }
1566
1567 static void getRingBufferErrorInfo(struct scudo_error_info *ErrorInfo,
1568 size_t &NextErrorReport,
1569 uintptr_t FaultAddr,
1570 const StackDepot *Depot,
1571 const char *RingBufferPtr,
1572 size_t RingBufferSize) {
1573 auto *RingBuffer =
1574 reinterpret_cast<const AllocationRingBuffer *>(RingBufferPtr);
1575 size_t RingBufferElements = ringBufferElementsFromBytes(Bytes: RingBufferSize);
1576 if (!RingBuffer || RingBufferElements == 0 || !Depot)
1577 return;
1578 uptr Pos = atomic_load_relaxed(&RingBuffer->Pos);
1579
1580 for (uptr I = Pos - 1; I != Pos - 1 - RingBufferElements &&
1581 NextErrorReport != NumErrorReports;
1582 --I) {
1583 auto *Entry = getRingBufferEntry(RingBuffer, I % RingBufferElements);
1584 uptr EntryPtr = atomic_load_relaxed(&Entry->Ptr);
1585 if (!EntryPtr)
1586 continue;
1587
1588 uptr UntaggedEntryPtr = untagPointer(Ptr: EntryPtr);
1589 uptr EntrySize = atomic_load_relaxed(&Entry->AllocationSize);
1590 u32 AllocationTrace = atomic_load_relaxed(&Entry->AllocationTrace);
1591 u32 AllocationTid = atomic_load_relaxed(&Entry->AllocationTid);
1592 u32 DeallocationTrace = atomic_load_relaxed(&Entry->DeallocationTrace);
1593 u32 DeallocationTid = atomic_load_relaxed(&Entry->DeallocationTid);
1594
1595 if (DeallocationTid) {
1596 // For UAF we only consider in-bounds fault addresses because
1597 // out-of-bounds UAF is rare and attempting to detect it is very likely
1598 // to result in false positives.
1599 if (FaultAddr < EntryPtr || FaultAddr >= EntryPtr + EntrySize)
1600 continue;
1601 } else {
1602 // Ring buffer OOB is only possible with secondary allocations. In this
1603 // case we are guaranteed a guard region of at least a page on either
1604 // side of the allocation (guard page on the right, guard page + tagged
1605 // region on the left), so ignore any faults outside of that range.
1606 if (FaultAddr < EntryPtr - getPageSizeCached() ||
1607 FaultAddr >= EntryPtr + EntrySize + getPageSizeCached())
1608 continue;
1609
1610 // For UAF the ring buffer will contain two entries, one for the
1611 // allocation and another for the deallocation. Don't report buffer
1612 // overflow/underflow using the allocation entry if we have already
1613 // collected a report from the deallocation entry.
1614 bool Found = false;
1615 for (uptr J = 0; J != NextErrorReport; ++J) {
1616 if (ErrorInfo->reports[J].allocation_address == UntaggedEntryPtr) {
1617 Found = true;
1618 break;
1619 }
1620 }
1621 if (Found)
1622 continue;
1623 }
1624
1625 auto *R = &ErrorInfo->reports[NextErrorReport++];
1626 if (DeallocationTid)
1627 R->error_type = USE_AFTER_FREE;
1628 else if (FaultAddr < EntryPtr)
1629 R->error_type = BUFFER_UNDERFLOW;
1630 else
1631 R->error_type = BUFFER_OVERFLOW;
1632
1633 R->allocation_address = UntaggedEntryPtr;
1634 R->allocation_size = EntrySize;
1635 collectTraceMaybe(Depot, Trace&: R->allocation_trace, Hash: AllocationTrace);
1636 R->allocation_tid = AllocationTid;
1637 collectTraceMaybe(Depot, Trace&: R->deallocation_trace, Hash: DeallocationTrace);
1638 R->deallocation_tid = DeallocationTid;
1639 }
1640 }
1641
1642 uptr getStats(ScopedString *Str) {
1643 Primary.getStats(Str);
1644 Secondary.getStats(Str);
1645 Quarantine.getStats(Str);
1646 TSDRegistry.getStats(Str);
1647 return Str->length();
1648 }
1649
1650 static typename AllocationRingBuffer::Entry *
1651 getRingBufferEntry(AllocationRingBuffer *RB, uptr N) {
1652 char *RBEntryStart =
1653 &reinterpret_cast<char *>(RB)[sizeof(AllocationRingBuffer)];
1654 return &reinterpret_cast<typename AllocationRingBuffer::Entry *>(
1655 RBEntryStart)[N];
1656 }
1657 static const typename AllocationRingBuffer::Entry *
1658 getRingBufferEntry(const AllocationRingBuffer *RB, uptr N) {
1659 const char *RBEntryStart =
1660 &reinterpret_cast<const char *>(RB)[sizeof(AllocationRingBuffer)];
1661 return &reinterpret_cast<const typename AllocationRingBuffer::Entry *>(
1662 RBEntryStart)[N];
1663 }
1664
1665 void initRingBufferMaybe() {
1666 ScopedLock L(RingBufferInitLock);
1667 if (getRingBuffer() != nullptr)
1668 return;
1669
1670 int ring_buffer_size = getFlags()->allocation_ring_buffer_size;
1671 if (ring_buffer_size <= 0)
1672 return;
1673
1674 u32 AllocationRingBufferSize = static_cast<u32>(ring_buffer_size);
1675
1676 // We store alloc and free stacks for each entry.
1677 constexpr u32 kStacksPerRingBufferEntry = 2;
1678 constexpr u32 kMaxU32Pow2 = ~(UINT32_MAX >> 1);
1679 static_assert(isPowerOfTwo(X: kMaxU32Pow2));
1680 // On Android we always have 3 frames at the bottom: __start_main,
1681 // __libc_init, main, and 3 at the top: malloc, scudo_malloc and
1682 // Allocator::allocate. This leaves 10 frames for the user app. The next
1683 // smallest power of two (8) would only leave 2, which is clearly too
1684 // little.
1685 constexpr u32 kFramesPerStack = 16;
1686 static_assert(isPowerOfTwo(X: kFramesPerStack));
1687
1688 if (AllocationRingBufferSize > kMaxU32Pow2 / kStacksPerRingBufferEntry)
1689 return;
1690 u32 TabSize = static_cast<u32>(roundUpPowerOfTwo(Size: kStacksPerRingBufferEntry *
1691 AllocationRingBufferSize));
1692 if (TabSize > UINT32_MAX / kFramesPerStack)
1693 return;
1694 u32 RingSize = static_cast<u32>(TabSize * kFramesPerStack);
1695
1696 uptr StackDepotSize = sizeof(StackDepot) + sizeof(atomic_u64) * RingSize +
1697 sizeof(atomic_u32) * TabSize;
1698 MemMapT DepotMap;
1699 DepotMap.map(
1700 /*Addr=*/Addr: 0U, Size: roundUp(X: StackDepotSize, Boundary: getPageSizeCached()),
1701 Name: "scudo:stack_depot");
1702 auto *Depot = reinterpret_cast<StackDepot *>(DepotMap.getBase());
1703 Depot->init(RingSz: RingSize, TabSz: TabSize);
1704
1705 MemMapT MemMap;
1706 MemMap.map(
1707 /*Addr=*/Addr: 0U,
1708 Size: roundUp(X: ringBufferSizeInBytes(RingBufferElements: AllocationRingBufferSize),
1709 Boundary: getPageSizeCached()),
1710 Name: "scudo:ring_buffer");
1711 auto *RB = reinterpret_cast<AllocationRingBuffer *>(MemMap.getBase());
1712 RB->RawRingBufferMap = MemMap;
1713 RB->RingBufferElements = AllocationRingBufferSize;
1714 RB->Depot = Depot;
1715 RB->StackDepotSize = StackDepotSize;
1716 RB->RawStackDepotMap = DepotMap;
1717
1718 atomic_store(A: &RingBufferAddress, V: reinterpret_cast<uptr>(RB),
1719 MO: memory_order_release);
1720 }
1721
1722 void unmapRingBuffer() {
1723 AllocationRingBuffer *RB = getRingBuffer();
1724 if (RB == nullptr)
1725 return;
1726 // N.B. because RawStackDepotMap is part of RawRingBufferMap, the order
1727 // is very important.
1728 RB->RawStackDepotMap.unmap();
1729 // Note that the `RB->RawRingBufferMap` is stored on the pages managed by
1730 // itself. Take over the ownership before calling unmap() so that any
1731 // operation along with unmap() won't touch inaccessible pages.
1732 MemMapT RawRingBufferMap = RB->RawRingBufferMap;
1733 RawRingBufferMap.unmap();
1734 atomic_store(A: &RingBufferAddress, V: 0, MO: memory_order_release);
1735 }
1736
1737 static constexpr size_t ringBufferSizeInBytes(u32 RingBufferElements) {
1738 return sizeof(AllocationRingBuffer) +
1739 RingBufferElements * sizeof(typename AllocationRingBuffer::Entry);
1740 }
1741
1742 static constexpr size_t ringBufferElementsFromBytes(size_t Bytes) {
1743 if (Bytes < sizeof(AllocationRingBuffer)) {
1744 return 0;
1745 }
1746 return (Bytes - sizeof(AllocationRingBuffer)) /
1747 sizeof(typename AllocationRingBuffer::Entry);
1748 }
1749};
1750
1751} // namespace scudo
1752
1753#endif // SCUDO_COMBINED_H_
1754