1//===-- combined.h ----------------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#ifndef SCUDO_COMBINED_H_
10#define SCUDO_COMBINED_H_
11
12#include "allocator_config_wrapper.h"
13#include "atomic_helpers.h"
14#include "chunk.h"
15#include "common.h"
16#include "flags.h"
17#include "flags_parser.h"
18#include "local_cache.h"
19#include "mem_map.h"
20#include "memtag.h"
21#include "mutex.h"
22#include "options.h"
23#include "quarantine.h"
24#include "report.h"
25#include "secondary.h"
26#include "stack_depot.h"
27#include "string_utils.h"
28#include "tsd.h"
29
30#include "scudo/interface.h"
31
32#ifdef GWP_ASAN_HOOKS
33#include "gwp_asan/guarded_pool_allocator.h"
34#include "gwp_asan/optional/backtrace.h"
35#include "gwp_asan/optional/segv_handler.h"
36#endif // GWP_ASAN_HOOKS
37
38extern "C" inline void EmptyCallback() {}
39
40#ifdef HAVE_ANDROID_UNSAFE_FRAME_POINTER_CHASE
41// This function is not part of the NDK so it does not appear in any public
42// header files. We only declare/use it when targeting the platform.
43extern "C" size_t android_unsafe_frame_pointer_chase(scudo::uptr *buf,
44 size_t num_entries);
45#endif
46
47namespace scudo {
48
49template <class Config, void (*PostInitCallback)(void) = EmptyCallback>
50class Allocator {
51public:
52 using AllocatorConfig = BaseConfig<Config>;
53 using PrimaryT =
54 typename AllocatorConfig::template PrimaryT<PrimaryConfig<Config>>;
55 using SecondaryT =
56 typename AllocatorConfig::template SecondaryT<SecondaryConfig<Config>>;
57 using CacheT = typename PrimaryT::CacheT;
58 typedef Allocator<Config, PostInitCallback> ThisT;
59 typedef typename AllocatorConfig::template TSDRegistryT<ThisT> TSDRegistryT;
60
61 void callPostInitCallback() {
62 pthread_once(once_control: &PostInitNonce, init_routine: PostInitCallback);
63 }
64
65 struct QuarantineCallback {
66 explicit QuarantineCallback(ThisT &Instance, CacheT &LocalCache)
67 : Allocator(Instance), Cache(LocalCache) {}
68
69 // Chunk recycling function, returns a quarantined chunk to the backend,
70 // first making sure it hasn't been tampered with.
71 void recycle(void *Ptr) {
72 Chunk::UnpackedHeader Header;
73 Chunk::loadHeader(Cookie: Allocator.Cookie, Ptr, NewUnpackedHeader: &Header);
74 if (UNLIKELY(Header.State != Chunk::State::Quarantined))
75 reportInvalidChunkState(Action: AllocatorAction::Recycling, Ptr);
76
77 Header.State = Chunk::State::Available;
78 Chunk::storeHeader(Cookie: Allocator.Cookie, Ptr, NewUnpackedHeader: &Header);
79
80 if (allocatorSupportsMemoryTagging<AllocatorConfig>())
81 Ptr = untagPointer(Ptr);
82 void *BlockBegin = Allocator::getBlockBegin(Ptr, Header: &Header);
83 Cache.deallocate(Header.ClassId, BlockBegin);
84 }
85
86 // We take a shortcut when allocating a quarantine batch by working with the
87 // appropriate class ID instead of using Size. The compiler should optimize
88 // the class ID computation and work with the associated cache directly.
89 void *allocate(UNUSED uptr Size) {
90 const uptr QuarantineClassId = SizeClassMap::getClassIdBySize(
91 sizeof(QuarantineBatch) + Chunk::getHeaderSize());
92 void *Ptr = Cache.allocate(QuarantineClassId);
93 // Quarantine batch allocation failure is fatal.
94 if (UNLIKELY(!Ptr))
95 reportOutOfMemory(SizeClassMap::getSizeByClassId(QuarantineClassId));
96
97 Ptr = reinterpret_cast<void *>(reinterpret_cast<uptr>(Ptr) +
98 Chunk::getHeaderSize());
99 Chunk::UnpackedHeader Header = {};
100 Header.ClassId = QuarantineClassId & Chunk::ClassIdMask;
101 Header.SizeOrUnusedBytes = sizeof(QuarantineBatch);
102 Header.State = Chunk::State::Allocated;
103 Chunk::storeHeader(Cookie: Allocator.Cookie, Ptr, NewUnpackedHeader: &Header);
104
105 // Reset tag to 0 as this chunk may have been previously used for a tagged
106 // user allocation.
107 if (UNLIKELY(useMemoryTagging<AllocatorConfig>(
108 Allocator.Primary.Options.load())))
109 storeTags(Begin: reinterpret_cast<uptr>(Ptr),
110 End: reinterpret_cast<uptr>(Ptr) + sizeof(QuarantineBatch));
111
112 return Ptr;
113 }
114
115 void deallocate(void *Ptr) {
116 const uptr QuarantineClassId = SizeClassMap::getClassIdBySize(
117 sizeof(QuarantineBatch) + Chunk::getHeaderSize());
118 Chunk::UnpackedHeader Header;
119 Chunk::loadHeader(Cookie: Allocator.Cookie, Ptr, NewUnpackedHeader: &Header);
120
121 if (UNLIKELY(Header.State != Chunk::State::Allocated))
122 reportInvalidChunkState(Action: AllocatorAction::Deallocating, Ptr);
123 DCHECK_EQ(Header.ClassId, QuarantineClassId);
124 DCHECK_EQ(Header.Offset, 0);
125 DCHECK_EQ(Header.SizeOrUnusedBytes, sizeof(QuarantineBatch));
126
127 Header.State = Chunk::State::Available;
128 Chunk::storeHeader(Cookie: Allocator.Cookie, Ptr, NewUnpackedHeader: &Header);
129 Cache.deallocate(QuarantineClassId,
130 reinterpret_cast<void *>(reinterpret_cast<uptr>(Ptr) -
131 Chunk::getHeaderSize()));
132 }
133
134 private:
135 ThisT &Allocator;
136 CacheT &Cache;
137 };
138
139 typedef GlobalQuarantine<QuarantineCallback, void> QuarantineT;
140 typedef typename QuarantineT::CacheT QuarantineCacheT;
141
142 void init() {
143 performSanityChecks();
144
145 // Check if hardware CRC32 is supported in the binary and by the platform,
146 // if so, opt for the CRC32 hardware version of the checksum.
147 if (&computeHardwareCRC32 && hasHardwareCRC32())
148 HashAlgorithm = Checksum::HardwareCRC32;
149
150 if (UNLIKELY(!getRandom(&Cookie, sizeof(Cookie))))
151 Cookie = static_cast<u32>(getMonotonicTime() ^
152 (reinterpret_cast<uptr>(this) >> 4));
153
154 initFlags();
155 reportUnrecognizedFlags();
156
157 // Store some flags locally.
158 if (getFlags()->may_return_null)
159 Primary.Options.set(OptionBit::MayReturnNull);
160 if (getFlags()->zero_contents)
161 Primary.Options.setFillContentsMode(ZeroFill);
162 else if (getFlags()->pattern_fill_contents)
163 Primary.Options.setFillContentsMode(PatternOrZeroFill);
164 if (getFlags()->dealloc_type_mismatch)
165 Primary.Options.set(OptionBit::DeallocTypeMismatch);
166 if (getFlags()->delete_size_mismatch)
167 Primary.Options.set(OptionBit::DeleteSizeMismatch);
168 if (allocatorSupportsMemoryTagging<AllocatorConfig>() &&
169 systemSupportsMemoryTagging())
170 Primary.Options.set(OptionBit::UseMemoryTagging);
171
172 QuarantineMaxChunkSize =
173 static_cast<u32>(getFlags()->quarantine_max_chunk_size);
174
175 Stats.init();
176 // TODO(chiahungduan): Given that we support setting the default value in
177 // the PrimaryConfig and CacheConfig, consider to deprecate the use of
178 // `release_to_os_interval_ms` flag.
179 const s32 ReleaseToOsIntervalMs = getFlags()->release_to_os_interval_ms;
180 Primary.init(ReleaseToOsIntervalMs);
181 Secondary.init(&Stats, ReleaseToOsIntervalMs);
182 Quarantine.init(
183 static_cast<uptr>(getFlags()->quarantine_size_kb << 10),
184 static_cast<uptr>(getFlags()->thread_local_quarantine_size_kb << 10));
185 }
186
187 void enableRingBuffer() NO_THREAD_SAFETY_ANALYSIS {
188 AllocationRingBuffer *RB = getRingBuffer();
189 if (RB)
190 RB->Depot->enable();
191 RingBufferInitLock.unlock();
192 }
193
194 void disableRingBuffer() NO_THREAD_SAFETY_ANALYSIS {
195 RingBufferInitLock.lock();
196 AllocationRingBuffer *RB = getRingBuffer();
197 if (RB)
198 RB->Depot->disable();
199 }
200
201 // Initialize the embedded GWP-ASan instance. Requires the main allocator to
202 // be functional, best called from PostInitCallback.
203 void initGwpAsan() {
204#ifdef GWP_ASAN_HOOKS
205 gwp_asan::options::Options Opt;
206 Opt.Enabled = getFlags()->GWP_ASAN_Enabled;
207 Opt.MaxSimultaneousAllocations =
208 getFlags()->GWP_ASAN_MaxSimultaneousAllocations;
209 Opt.SampleRate = getFlags()->GWP_ASAN_SampleRate;
210 Opt.InstallSignalHandlers = getFlags()->GWP_ASAN_InstallSignalHandlers;
211 Opt.Recoverable = getFlags()->GWP_ASAN_Recoverable;
212 // Embedded GWP-ASan is locked through the Scudo atfork handler (via
213 // Allocator::disable calling GWPASan.disable). Disable GWP-ASan's atfork
214 // handler.
215 Opt.InstallForkHandlers = false;
216 Opt.Backtrace = gwp_asan::backtrace::getBacktraceFunction();
217 GuardedAlloc.init(Opt);
218
219 if (Opt.InstallSignalHandlers)
220 gwp_asan::segv_handler::installSignalHandlers(
221 &GuardedAlloc, Printf,
222 gwp_asan::backtrace::getPrintBacktraceFunction(),
223 gwp_asan::backtrace::getSegvBacktraceFunction(),
224 Opt.Recoverable);
225
226 GuardedAllocSlotSize =
227 GuardedAlloc.getAllocatorState()->maximumAllocationSize();
228 Stats.add(StatFree, static_cast<uptr>(Opt.MaxSimultaneousAllocations) *
229 GuardedAllocSlotSize);
230#endif // GWP_ASAN_HOOKS
231 }
232
233#ifdef GWP_ASAN_HOOKS
234 const gwp_asan::AllocationMetadata *getGwpAsanAllocationMetadata() {
235 return GuardedAlloc.getMetadataRegion();
236 }
237
238 const gwp_asan::AllocatorState *getGwpAsanAllocatorState() {
239 return GuardedAlloc.getAllocatorState();
240 }
241#endif // GWP_ASAN_HOOKS
242
243 ALWAYS_INLINE void initThreadMaybe(bool MinimalInit = false) {
244 TSDRegistry.initThreadMaybe(this, MinimalInit);
245 }
246
247 void unmapTestOnly() {
248 unmapRingBuffer();
249 TSDRegistry.unmapTestOnly(this);
250 Primary.unmapTestOnly();
251 Secondary.unmapTestOnly();
252#ifdef GWP_ASAN_HOOKS
253 if (getFlags()->GWP_ASAN_InstallSignalHandlers)
254 gwp_asan::segv_handler::uninstallSignalHandlers();
255 GuardedAlloc.uninitTestOnly();
256#endif // GWP_ASAN_HOOKS
257 }
258
259 TSDRegistryT *getTSDRegistry() { return &TSDRegistry; }
260 QuarantineT *getQuarantine() { return &Quarantine; }
261
262 // The Cache must be provided zero-initialized.
263 void initCache(CacheT *Cache) { Cache->init(&Stats, &Primary); }
264
265 // Release the resources used by a TSD, which involves:
266 // - draining the local quarantine cache to the global quarantine;
267 // - releasing the cached pointers back to the Primary;
268 // - unlinking the local stats from the global ones (destroying the cache does
269 // the last two items).
270 void commitBack(TSD<ThisT> *TSD) {
271 TSD->assertLocked(/*BypassCheck=*/true);
272 Quarantine.drain(&TSD->getQuarantineCache(),
273 QuarantineCallback(*this, TSD->getCache()));
274 TSD->getCache().destroy(&Stats);
275 }
276
277 void drainCache(TSD<ThisT> *TSD) {
278 TSD->assertLocked(/*BypassCheck=*/true);
279 Quarantine.drainAndRecycle(&TSD->getQuarantineCache(),
280 QuarantineCallback(*this, TSD->getCache()));
281 TSD->getCache().drain();
282 }
283 void drainCaches() { TSDRegistry.drainCaches(this); }
284
285 ALWAYS_INLINE void *getHeaderTaggedPointer(void *Ptr) {
286 if (!allocatorSupportsMemoryTagging<AllocatorConfig>())
287 return Ptr;
288 auto UntaggedPtr = untagPointer(Ptr);
289 if (UntaggedPtr != Ptr)
290 return UntaggedPtr;
291 // Secondary, or pointer allocated while memory tagging is unsupported or
292 // disabled. The tag mismatch is okay in the latter case because tags will
293 // not be checked.
294 return addHeaderTag(Ptr);
295 }
296
297 ALWAYS_INLINE uptr addHeaderTag(uptr Ptr) {
298 if (!allocatorSupportsMemoryTagging<AllocatorConfig>())
299 return Ptr;
300 return addFixedTag(Ptr, Tag: 2);
301 }
302
303 ALWAYS_INLINE void *addHeaderTag(void *Ptr) {
304 return reinterpret_cast<void *>(addHeaderTag(reinterpret_cast<uptr>(Ptr)));
305 }
306
307 NOINLINE u32 collectStackTrace(UNUSED StackDepot *Depot) {
308#ifdef HAVE_ANDROID_UNSAFE_FRAME_POINTER_CHASE
309 // Discard collectStackTrace() frame and allocator function frame.
310 constexpr uptr DiscardFrames = 2;
311 uptr Stack[MaxTraceSize + DiscardFrames];
312 uptr Size =
313 android_unsafe_frame_pointer_chase(Stack, MaxTraceSize + DiscardFrames);
314 Size = Min<uptr>(Size, MaxTraceSize + DiscardFrames);
315 return Depot->insert(Stack + Min<uptr>(DiscardFrames, Size), Stack + Size);
316#else
317 return 0;
318#endif
319 }
320
321 uptr computeOddEvenMaskForPointerMaybe(const Options &Options, uptr Ptr,
322 uptr ClassId) {
323 if (!Options.get(Opt: OptionBit::UseOddEvenTags))
324 return 0;
325
326 // If a chunk's tag is odd, we want the tags of the surrounding blocks to be
327 // even, and vice versa. Blocks are laid out Size bytes apart, and adding
328 // Size to Ptr will flip the least significant set bit of Size in Ptr, so
329 // that bit will have the pattern 010101... for consecutive blocks, which we
330 // can use to determine which tag mask to use.
331 return 0x5555U << ((Ptr >> SizeClassMap::getSizeLSBByClassId(ClassId)) & 1);
332 }
333
334 NOINLINE void *allocate(uptr Size, Chunk::Origin Origin,
335 uptr Alignment = MinAlignment,
336 bool ZeroContents = false) NO_THREAD_SAFETY_ANALYSIS {
337 initThreadMaybe();
338
339 const Options Options = Primary.Options.load();
340 if (UNLIKELY(Alignment > MaxAlignment)) {
341 if (Options.get(Opt: OptionBit::MayReturnNull))
342 return nullptr;
343 reportAlignmentTooBig(Alignment, MaxAlignment);
344 }
345 if (Alignment < MinAlignment)
346 Alignment = MinAlignment;
347
348#ifdef GWP_ASAN_HOOKS
349 if (UNLIKELY(GuardedAlloc.shouldSample())) {
350 if (void *Ptr = GuardedAlloc.allocate(Size, Alignment)) {
351 Stats.lock();
352 Stats.add(StatAllocated, GuardedAllocSlotSize);
353 Stats.sub(StatFree, GuardedAllocSlotSize);
354 Stats.unlock();
355 return Ptr;
356 }
357 }
358#endif // GWP_ASAN_HOOKS
359
360 const FillContentsMode FillContents = ZeroContents ? ZeroFill
361 : TSDRegistry.getDisableMemInit()
362 ? NoFill
363 : Options.getFillContentsMode();
364
365 // If the requested size happens to be 0 (more common than you might think),
366 // allocate MinAlignment bytes on top of the header. Then add the extra
367 // bytes required to fulfill the alignment requirements: we allocate enough
368 // to be sure that there will be an address in the block that will satisfy
369 // the alignment.
370 const uptr NeededSize =
371 roundUp(X: Size, Boundary: MinAlignment) +
372 ((Alignment > MinAlignment) ? Alignment : Chunk::getHeaderSize());
373
374 // Takes care of extravagantly large sizes as well as integer overflows.
375 static_assert(MaxAllowedMallocSize < UINTPTR_MAX - MaxAlignment, "");
376 if (UNLIKELY(Size >= MaxAllowedMallocSize)) {
377 if (Options.get(Opt: OptionBit::MayReturnNull))
378 return nullptr;
379 reportAllocationSizeTooBig(UserSize: Size, TotalSize: NeededSize, MaxSize: MaxAllowedMallocSize);
380 }
381 DCHECK_LE(Size, NeededSize);
382
383 void *Block = nullptr;
384 uptr ClassId = 0;
385 uptr SecondaryBlockEnd = 0;
386 if (LIKELY(PrimaryT::canAllocate(NeededSize))) {
387 ClassId = SizeClassMap::getClassIdBySize(NeededSize);
388 DCHECK_NE(ClassId, 0U);
389 typename TSDRegistryT::ScopedTSD TSD(TSDRegistry);
390 Block = TSD->getCache().allocate(ClassId);
391 // If the allocation failed, retry in each successively larger class until
392 // it fits. If it fails to fit in the largest class, fallback to the
393 // Secondary.
394 if (UNLIKELY(!Block)) {
395 while (ClassId < SizeClassMap::LargestClassId && !Block)
396 Block = TSD->getCache().allocate(++ClassId);
397 if (!Block)
398 ClassId = 0;
399 }
400 }
401 if (UNLIKELY(ClassId == 0)) {
402 Block = Secondary.allocate(Options, Size, Alignment, &SecondaryBlockEnd,
403 FillContents);
404 }
405
406 if (UNLIKELY(!Block)) {
407 if (Options.get(Opt: OptionBit::MayReturnNull))
408 return nullptr;
409 printStats();
410 reportOutOfMemory(RequestedSize: NeededSize);
411 }
412
413 const uptr UserPtr = roundUp(
414 X: reinterpret_cast<uptr>(Block) + Chunk::getHeaderSize(), Boundary: Alignment);
415 const uptr SizeOrUnusedBytes =
416 ClassId ? Size : SecondaryBlockEnd - (UserPtr + Size);
417
418 if (LIKELY(!useMemoryTagging<AllocatorConfig>(Options))) {
419 return initChunk(ClassId, Origin, Block, UserPtr, SizeOrUnusedBytes,
420 FillContents);
421 }
422
423 return initChunkWithMemoryTagging(ClassId, Origin, Block, UserPtr, Size,
424 SizeOrUnusedBytes, FillContents);
425 }
426
427 NOINLINE void deallocate(void *Ptr, Chunk::Origin Origin, uptr DeleteSize = 0,
428 UNUSED uptr Alignment = MinAlignment) {
429 if (UNLIKELY(!Ptr))
430 return;
431
432 // For a deallocation, we only ensure minimal initialization, meaning thread
433 // local data will be left uninitialized for now (when using ELF TLS). The
434 // fallback cache will be used instead. This is a workaround for a situation
435 // where the only heap operation performed in a thread would be a free past
436 // the TLS destructors, ending up in initialized thread specific data never
437 // being destroyed properly. Any other heap operation will do a full init.
438 initThreadMaybe(/*MinimalInit=*/MinimalInit: true);
439
440#ifdef GWP_ASAN_HOOKS
441 if (UNLIKELY(GuardedAlloc.pointerIsMine(Ptr))) {
442 GuardedAlloc.deallocate(Ptr);
443 Stats.lock();
444 Stats.add(StatFree, GuardedAllocSlotSize);
445 Stats.sub(StatAllocated, GuardedAllocSlotSize);
446 Stats.unlock();
447 return;
448 }
449#endif // GWP_ASAN_HOOKS
450
451 if (UNLIKELY(!isAligned(reinterpret_cast<uptr>(Ptr), MinAlignment)))
452 reportMisalignedPointer(Action: AllocatorAction::Deallocating, Ptr);
453
454 void *TaggedPtr = Ptr;
455 Ptr = getHeaderTaggedPointer(Ptr);
456
457 Chunk::UnpackedHeader Header;
458 Chunk::loadHeader(Cookie, Ptr, NewUnpackedHeader: &Header);
459
460 if (UNLIKELY(Header.State != Chunk::State::Allocated))
461 reportInvalidChunkState(Action: AllocatorAction::Deallocating, Ptr);
462
463 const Options Options = Primary.Options.load();
464 if (Options.get(Opt: OptionBit::DeallocTypeMismatch)) {
465 if (UNLIKELY(Header.OriginOrWasZeroed != Origin)) {
466 // With the exception of memalign'd chunks, that can be still be free'd.
467 if (Header.OriginOrWasZeroed != Chunk::Origin::Memalign ||
468 Origin != Chunk::Origin::Malloc)
469 reportDeallocTypeMismatch(Action: AllocatorAction::Deallocating, Ptr,
470 TypeA: Header.OriginOrWasZeroed, TypeB: Origin);
471 }
472 }
473
474 const uptr Size = getSize(Ptr, Header: &Header);
475 if (DeleteSize && Options.get(Opt: OptionBit::DeleteSizeMismatch)) {
476 if (UNLIKELY(DeleteSize != Size))
477 reportDeleteSizeMismatch(Ptr, Size: DeleteSize, ExpectedSize: Size);
478 }
479
480 quarantineOrDeallocateChunk(Options, TaggedPtr, Header: &Header, Size);
481 }
482
483 void *reallocate(void *OldPtr, uptr NewSize, uptr Alignment = MinAlignment) {
484 initThreadMaybe();
485
486 const Options Options = Primary.Options.load();
487 if (UNLIKELY(NewSize >= MaxAllowedMallocSize)) {
488 if (Options.get(Opt: OptionBit::MayReturnNull))
489 return nullptr;
490 reportAllocationSizeTooBig(UserSize: NewSize, TotalSize: 0, MaxSize: MaxAllowedMallocSize);
491 }
492
493 // The following cases are handled by the C wrappers.
494 DCHECK_NE(OldPtr, nullptr);
495 DCHECK_NE(NewSize, 0);
496
497#ifdef GWP_ASAN_HOOKS
498 if (UNLIKELY(GuardedAlloc.pointerIsMine(OldPtr))) {
499 uptr OldSize = GuardedAlloc.getSize(OldPtr);
500 void *NewPtr = allocate(NewSize, Chunk::Origin::Malloc, Alignment);
501 if (NewPtr)
502 memcpy(NewPtr, OldPtr, (NewSize < OldSize) ? NewSize : OldSize);
503 GuardedAlloc.deallocate(OldPtr);
504 Stats.lock();
505 Stats.add(StatFree, GuardedAllocSlotSize);
506 Stats.sub(StatAllocated, GuardedAllocSlotSize);
507 Stats.unlock();
508 return NewPtr;
509 }
510#endif // GWP_ASAN_HOOKS
511
512 void *OldTaggedPtr = OldPtr;
513 OldPtr = getHeaderTaggedPointer(Ptr: OldPtr);
514
515 if (UNLIKELY(!isAligned(reinterpret_cast<uptr>(OldPtr), MinAlignment)))
516 reportMisalignedPointer(Action: AllocatorAction::Reallocating, Ptr: OldPtr);
517
518 Chunk::UnpackedHeader Header;
519 Chunk::loadHeader(Cookie, Ptr: OldPtr, NewUnpackedHeader: &Header);
520
521 if (UNLIKELY(Header.State != Chunk::State::Allocated))
522 reportInvalidChunkState(Action: AllocatorAction::Reallocating, Ptr: OldPtr);
523
524 // Pointer has to be allocated with a malloc-type function. Some
525 // applications think that it is OK to realloc a memalign'ed pointer, which
526 // will trigger this check. It really isn't.
527 if (Options.get(Opt: OptionBit::DeallocTypeMismatch)) {
528 if (UNLIKELY(Header.OriginOrWasZeroed != Chunk::Origin::Malloc))
529 reportDeallocTypeMismatch(Action: AllocatorAction::Reallocating, Ptr: OldPtr,
530 TypeA: Header.OriginOrWasZeroed,
531 TypeB: Chunk::Origin::Malloc);
532 }
533
534 void *BlockBegin = getBlockBegin(Ptr: OldTaggedPtr, Header: &Header);
535 uptr BlockEnd;
536 uptr OldSize;
537 const uptr ClassId = Header.ClassId;
538 if (LIKELY(ClassId)) {
539 BlockEnd = reinterpret_cast<uptr>(BlockBegin) +
540 SizeClassMap::getSizeByClassId(ClassId);
541 OldSize = Header.SizeOrUnusedBytes;
542 } else {
543 BlockEnd = SecondaryT::getBlockEnd(BlockBegin);
544 OldSize = BlockEnd - (reinterpret_cast<uptr>(OldTaggedPtr) +
545 Header.SizeOrUnusedBytes);
546 }
547 // If the new chunk still fits in the previously allocated block (with a
548 // reasonable delta), we just keep the old block, and update the chunk
549 // header to reflect the size change.
550 if (reinterpret_cast<uptr>(OldTaggedPtr) + NewSize <= BlockEnd) {
551 if (NewSize > OldSize || (OldSize - NewSize) < getPageSizeCached()) {
552 // If we have reduced the size, set the extra bytes to the fill value
553 // so that we are ready to grow it again in the future.
554 if (NewSize < OldSize) {
555 const FillContentsMode FillContents =
556 TSDRegistry.getDisableMemInit() ? NoFill
557 : Options.getFillContentsMode();
558 if (FillContents != NoFill) {
559 memset(s: reinterpret_cast<char *>(OldTaggedPtr) + NewSize,
560 c: FillContents == ZeroFill ? 0 : PatternFillByte,
561 n: OldSize - NewSize);
562 }
563 }
564
565 Header.SizeOrUnusedBytes =
566 (ClassId ? NewSize
567 : BlockEnd -
568 (reinterpret_cast<uptr>(OldTaggedPtr) + NewSize)) &
569 Chunk::SizeOrUnusedBytesMask;
570 Chunk::storeHeader(Cookie, Ptr: OldPtr, NewUnpackedHeader: &Header);
571 if (UNLIKELY(useMemoryTagging<AllocatorConfig>(Options))) {
572 if (ClassId) {
573 resizeTaggedChunk(OldPtr: reinterpret_cast<uptr>(OldTaggedPtr) + OldSize,
574 NewPtr: reinterpret_cast<uptr>(OldTaggedPtr) + NewSize,
575 NewSize, BlockEnd: untagPointer(Ptr: BlockEnd));
576 storePrimaryAllocationStackMaybe(Options, Ptr: OldPtr);
577 } else {
578 storeSecondaryAllocationStackMaybe(Options, Ptr: OldPtr, Size: NewSize);
579 }
580 }
581 return OldTaggedPtr;
582 }
583 }
584
585 // Otherwise we allocate a new one, and deallocate the old one. Some
586 // allocators will allocate an even larger chunk (by a fixed factor) to
587 // allow for potential further in-place realloc. The gains of such a trick
588 // are currently unclear.
589 void *NewPtr = allocate(Size: NewSize, Origin: Chunk::Origin::Malloc, Alignment);
590 if (LIKELY(NewPtr)) {
591 memcpy(dest: NewPtr, src: OldTaggedPtr, n: Min(A: NewSize, B: OldSize));
592 quarantineOrDeallocateChunk(Options, TaggedPtr: OldTaggedPtr, Header: &Header, Size: OldSize);
593 }
594 return NewPtr;
595 }
596
597 // TODO(kostyak): disable() is currently best-effort. There are some small
598 // windows of time when an allocation could still succeed after
599 // this function finishes. We will revisit that later.
600 void disable() NO_THREAD_SAFETY_ANALYSIS {
601 initThreadMaybe();
602#ifdef GWP_ASAN_HOOKS
603 GuardedAlloc.disable();
604#endif
605 TSDRegistry.disable();
606 Stats.disable();
607 Quarantine.disable();
608 Primary.disable();
609 Secondary.disable();
610 disableRingBuffer();
611 }
612
613 void enable() NO_THREAD_SAFETY_ANALYSIS {
614 initThreadMaybe();
615 enableRingBuffer();
616 Secondary.enable();
617 Primary.enable();
618 Quarantine.enable();
619 Stats.enable();
620 TSDRegistry.enable();
621#ifdef GWP_ASAN_HOOKS
622 GuardedAlloc.enable();
623#endif
624 }
625
626 // The function returns the amount of bytes required to store the statistics,
627 // which might be larger than the amount of bytes provided. Note that the
628 // statistics buffer is not necessarily constant between calls to this
629 // function. This can be called with a null buffer or zero size for buffer
630 // sizing purposes.
631 uptr getStats(char *Buffer, uptr Size) {
632 ScopedString Str;
633 const uptr Length = getStats(&Str) + 1;
634 if (Length < Size)
635 Size = Length;
636 if (Buffer && Size) {
637 memcpy(dest: Buffer, src: Str.data(), n: Size);
638 Buffer[Size - 1] = '\0';
639 }
640 return Length;
641 }
642
643 void printStats() {
644 ScopedString Str;
645 getStats(&Str);
646 Str.output();
647 }
648
649 void printFragmentationInfo() {
650 ScopedString Str;
651 Primary.getFragmentationInfo(&Str);
652 // Secondary allocator dumps the fragmentation data in getStats().
653 Str.output();
654 }
655
656 void releaseToOS(ReleaseToOS ReleaseType) {
657 initThreadMaybe();
658 if (ReleaseType == ReleaseToOS::ForceAll)
659 drainCaches();
660 Primary.releaseToOS(ReleaseType);
661 Secondary.releaseToOS();
662 }
663
664 // Iterate over all chunks and call a callback for all busy chunks located
665 // within the provided memory range. Said callback must not use this allocator
666 // or a deadlock can ensue. This fits Android's malloc_iterate() needs.
667 void iterateOverChunks(uptr Base, uptr Size, iterate_callback Callback,
668 void *Arg) {
669 initThreadMaybe();
670 if (archSupportsMemoryTagging())
671 Base = untagPointer(Ptr: Base);
672 const uptr From = Base;
673 const uptr To = Base + Size;
674 bool MayHaveTaggedPrimary =
675 allocatorSupportsMemoryTagging<AllocatorConfig>() &&
676 systemSupportsMemoryTagging();
677 auto Lambda = [this, From, To, MayHaveTaggedPrimary, Callback,
678 Arg](uptr Block) {
679 if (Block < From || Block >= To)
680 return;
681 uptr Chunk;
682 Chunk::UnpackedHeader Header;
683 if (MayHaveTaggedPrimary) {
684 // A chunk header can either have a zero tag (tagged primary) or the
685 // header tag (secondary, or untagged primary). We don't know which so
686 // try both.
687 ScopedDisableMemoryTagChecks x;
688 if (!getChunkFromBlock(Block, Chunk: &Chunk, Header: &Header) &&
689 !getChunkFromBlock(Block: addHeaderTag(Block), Chunk: &Chunk, Header: &Header))
690 return;
691 } else {
692 if (!getChunkFromBlock(Block: addHeaderTag(Block), Chunk: &Chunk, Header: &Header))
693 return;
694 }
695 if (Header.State == Chunk::State::Allocated) {
696 uptr TaggedChunk = Chunk;
697 if (allocatorSupportsMemoryTagging<AllocatorConfig>())
698 TaggedChunk = untagPointer(Ptr: TaggedChunk);
699 if (useMemoryTagging<AllocatorConfig>(Primary.Options.load()))
700 TaggedChunk = loadTag(Ptr: Chunk);
701 Callback(TaggedChunk, getSize(Ptr: reinterpret_cast<void *>(Chunk), Header: &Header),
702 Arg);
703 }
704 };
705 Primary.iterateOverBlocks(Lambda);
706 Secondary.iterateOverBlocks(Lambda);
707#ifdef GWP_ASAN_HOOKS
708 GuardedAlloc.iterate(reinterpret_cast<void *>(Base), Size, Callback, Arg);
709#endif
710 }
711
712 bool canReturnNull() {
713 initThreadMaybe();
714 return Primary.Options.load().get(OptionBit::MayReturnNull);
715 }
716
717 bool setOption(Option O, sptr Value) {
718 initThreadMaybe();
719 if (O == Option::MemtagTuning) {
720 // Enabling odd/even tags involves a tradeoff between use-after-free
721 // detection and buffer overflow detection. Odd/even tags make it more
722 // likely for buffer overflows to be detected by increasing the size of
723 // the guaranteed "red zone" around the allocation, but on the other hand
724 // use-after-free is less likely to be detected because the tag space for
725 // any particular chunk is cut in half. Therefore we use this tuning
726 // setting to control whether odd/even tags are enabled.
727 if (Value == M_MEMTAG_TUNING_BUFFER_OVERFLOW)
728 Primary.Options.set(OptionBit::UseOddEvenTags);
729 else if (Value == M_MEMTAG_TUNING_UAF)
730 Primary.Options.clear(OptionBit::UseOddEvenTags);
731 return true;
732 } else {
733 // We leave it to the various sub-components to decide whether or not they
734 // want to handle the option, but we do not want to short-circuit
735 // execution if one of the setOption was to return false.
736 const bool PrimaryResult = Primary.setOption(O, Value);
737 const bool SecondaryResult = Secondary.setOption(O, Value);
738 const bool RegistryResult = TSDRegistry.setOption(O, Value);
739 return PrimaryResult && SecondaryResult && RegistryResult;
740 }
741 return false;
742 }
743
744 // Return the usable size for a given chunk. Technically we lie, as we just
745 // report the actual size of a chunk. This is done to counteract code actively
746 // writing past the end of a chunk (like sqlite3) when the usable size allows
747 // for it, which then forces realloc to copy the usable size of a chunk as
748 // opposed to its actual size.
749 uptr getUsableSize(const void *Ptr) {
750 if (UNLIKELY(!Ptr))
751 return 0;
752
753 return getAllocSize(Ptr);
754 }
755
756 uptr getAllocSize(const void *Ptr) {
757 initThreadMaybe();
758
759#ifdef GWP_ASAN_HOOKS
760 if (UNLIKELY(GuardedAlloc.pointerIsMine(Ptr)))
761 return GuardedAlloc.getSize(Ptr);
762#endif // GWP_ASAN_HOOKS
763
764 Ptr = getHeaderTaggedPointer(Ptr: const_cast<void *>(Ptr));
765 Chunk::UnpackedHeader Header;
766 Chunk::loadHeader(Cookie, Ptr, NewUnpackedHeader: &Header);
767
768 // Getting the alloc size of a chunk only makes sense if it's allocated.
769 if (UNLIKELY(Header.State != Chunk::State::Allocated))
770 reportInvalidChunkState(Action: AllocatorAction::Sizing, Ptr: const_cast<void *>(Ptr));
771
772 return getSize(Ptr, Header: &Header);
773 }
774
775 void getStats(StatCounters S) {
776 initThreadMaybe();
777 Stats.get(S);
778 }
779
780 // Returns true if the pointer provided was allocated by the current
781 // allocator instance, which is compliant with tcmalloc's ownership concept.
782 // A corrupted chunk will not be reported as owned, which is WAI.
783 bool isOwned(const void *Ptr) {
784 initThreadMaybe();
785#ifdef GWP_ASAN_HOOKS
786 if (GuardedAlloc.pointerIsMine(Ptr))
787 return true;
788#endif // GWP_ASAN_HOOKS
789 if (!Ptr || !isAligned(X: reinterpret_cast<uptr>(Ptr), Alignment: MinAlignment))
790 return false;
791 Ptr = getHeaderTaggedPointer(Ptr: const_cast<void *>(Ptr));
792 Chunk::UnpackedHeader Header;
793 return Chunk::isValid(Cookie, Ptr, NewUnpackedHeader: &Header) &&
794 Header.State == Chunk::State::Allocated;
795 }
796
797 bool useMemoryTaggingTestOnly() const {
798 return useMemoryTagging<AllocatorConfig>(Primary.Options.load());
799 }
800 void disableMemoryTagging() {
801 // If we haven't been initialized yet, we need to initialize now in order to
802 // prevent a future call to initThreadMaybe() from enabling memory tagging
803 // based on feature detection. But don't call initThreadMaybe() because it
804 // may end up calling the allocator (via pthread_atfork, via the post-init
805 // callback), which may cause mappings to be created with memory tagging
806 // enabled.
807 TSDRegistry.initOnceMaybe(this);
808 if (allocatorSupportsMemoryTagging<AllocatorConfig>()) {
809 Secondary.disableMemoryTagging();
810 Primary.Options.clear(OptionBit::UseMemoryTagging);
811 }
812 }
813
814 void setTrackAllocationStacks(bool Track) {
815 initThreadMaybe();
816 if (getFlags()->allocation_ring_buffer_size <= 0) {
817 DCHECK(!Primary.Options.load().get(OptionBit::TrackAllocationStacks));
818 return;
819 }
820
821 if (Track) {
822 initRingBufferMaybe();
823 Primary.Options.set(OptionBit::TrackAllocationStacks);
824 } else
825 Primary.Options.clear(OptionBit::TrackAllocationStacks);
826 }
827
828 void setFillContents(FillContentsMode FillContents) {
829 initThreadMaybe();
830 Primary.Options.setFillContentsMode(FillContents);
831 }
832
833 void setAddLargeAllocationSlack(bool AddSlack) {
834 initThreadMaybe();
835 if (AddSlack)
836 Primary.Options.set(OptionBit::AddLargeAllocationSlack);
837 else
838 Primary.Options.clear(OptionBit::AddLargeAllocationSlack);
839 }
840
841 const char *getStackDepotAddress() {
842 initThreadMaybe();
843 AllocationRingBuffer *RB = getRingBuffer();
844 return RB ? reinterpret_cast<char *>(RB->Depot) : nullptr;
845 }
846
847 uptr getStackDepotSize() {
848 initThreadMaybe();
849 AllocationRingBuffer *RB = getRingBuffer();
850 return RB ? RB->StackDepotSize : 0;
851 }
852
853 const char *getRegionInfoArrayAddress() const {
854 return Primary.getRegionInfoArrayAddress();
855 }
856
857 static uptr getRegionInfoArraySize() {
858 return PrimaryT::getRegionInfoArraySize();
859 }
860
861 const char *getRingBufferAddress() {
862 initThreadMaybe();
863 return reinterpret_cast<char *>(getRingBuffer());
864 }
865
866 uptr getRingBufferSize() {
867 initThreadMaybe();
868 AllocationRingBuffer *RB = getRingBuffer();
869 return RB && RB->RingBufferElements
870 ? ringBufferSizeInBytes(RingBufferElements: RB->RingBufferElements)
871 : 0;
872 }
873
874 static const uptr MaxTraceSize = 64;
875
876 static void collectTraceMaybe(const StackDepot *Depot,
877 uintptr_t (&Trace)[MaxTraceSize], u32 Hash) {
878 uptr RingPos, Size;
879 if (!Depot->find(Hash, RingPosPtr: &RingPos, SizePtr: &Size))
880 return;
881 for (unsigned I = 0; I != Size && I != MaxTraceSize; ++I)
882 Trace[I] = static_cast<uintptr_t>(Depot->at(RingPos: RingPos + I));
883 }
884
885 static void getErrorInfo(struct scudo_error_info *ErrorInfo,
886 uintptr_t FaultAddr, const char *DepotPtr,
887 size_t DepotSize, const char *RegionInfoPtr,
888 const char *RingBufferPtr, size_t RingBufferSize,
889 const char *Memory, const char *MemoryTags,
890 uintptr_t MemoryAddr, size_t MemorySize) {
891 // N.B. we need to support corrupted data in any of the buffers here. We get
892 // this information from an external process (the crashing process) that
893 // should not be able to crash the crash dumper (crash_dump on Android).
894 // See also the get_error_info_fuzzer.
895 *ErrorInfo = {};
896 if (!allocatorSupportsMemoryTagging<AllocatorConfig>() ||
897 MemoryAddr + MemorySize < MemoryAddr)
898 return;
899
900 const StackDepot *Depot = nullptr;
901 if (DepotPtr) {
902 // check for corrupted StackDepot. First we need to check whether we can
903 // read the metadata, then whether the metadata matches the size.
904 if (DepotSize < sizeof(*Depot))
905 return;
906 Depot = reinterpret_cast<const StackDepot *>(DepotPtr);
907 if (!Depot->isValid(BufSize: DepotSize))
908 return;
909 }
910
911 size_t NextErrorReport = 0;
912
913 // Check for OOB in the current block and the two surrounding blocks. Beyond
914 // that, UAF is more likely.
915 if (extractTag(Ptr: FaultAddr) != 0)
916 getInlineErrorInfo(ErrorInfo, NextErrorReport, FaultAddr, Depot,
917 RegionInfoPtr, Memory, MemoryTags, MemoryAddr,
918 MemorySize, MinDistance: 0, MaxDistance: 2);
919
920 // Check the ring buffer. For primary allocations this will only find UAF;
921 // for secondary allocations we can find either UAF or OOB.
922 getRingBufferErrorInfo(ErrorInfo, NextErrorReport, FaultAddr, Depot,
923 RingBufferPtr, RingBufferSize);
924
925 // Check for OOB in the 28 blocks surrounding the 3 we checked earlier.
926 // Beyond that we are likely to hit false positives.
927 if (extractTag(Ptr: FaultAddr) != 0)
928 getInlineErrorInfo(ErrorInfo, NextErrorReport, FaultAddr, Depot,
929 RegionInfoPtr, Memory, MemoryTags, MemoryAddr,
930 MemorySize, MinDistance: 2, MaxDistance: 16);
931 }
932
933private:
934 typedef typename PrimaryT::SizeClassMap SizeClassMap;
935
936 static const uptr MinAlignmentLog = SCUDO_MIN_ALIGNMENT_LOG;
937 static const uptr MaxAlignmentLog = 24U; // 16 MB seems reasonable.
938 static const uptr MinAlignment = 1UL << MinAlignmentLog;
939 static const uptr MaxAlignment = 1UL << MaxAlignmentLog;
940 static const uptr MaxAllowedMallocSize =
941 FIRST_32_SECOND_64(1UL << 31, 1ULL << 40);
942
943 static_assert(MinAlignment >= sizeof(Chunk::PackedHeader),
944 "Minimal alignment must at least cover a chunk header.");
945 static_assert(!allocatorSupportsMemoryTagging<AllocatorConfig>() ||
946 MinAlignment >= archMemoryTagGranuleSize(),
947 "");
948
949 static const u32 BlockMarker = 0x44554353U;
950
951 // These are indexes into an "array" of 32-bit values that store information
952 // inline with a chunk that is relevant to diagnosing memory tag faults, where
953 // 0 corresponds to the address of the user memory. This means that only
954 // negative indexes may be used. The smallest index that may be used is -2,
955 // which corresponds to 8 bytes before the user memory, because the chunk
956 // header size is 8 bytes and in allocators that support memory tagging the
957 // minimum alignment is at least the tag granule size (16 on aarch64).
958 static const sptr MemTagAllocationTraceIndex = -2;
959 static const sptr MemTagAllocationTidIndex = -1;
960
961 u32 Cookie = 0;
962 u32 QuarantineMaxChunkSize = 0;
963
964 GlobalStats Stats;
965 PrimaryT Primary;
966 SecondaryT Secondary;
967 QuarantineT Quarantine;
968 TSDRegistryT TSDRegistry;
969 pthread_once_t PostInitNonce = PTHREAD_ONCE_INIT;
970
971#ifdef GWP_ASAN_HOOKS
972 gwp_asan::GuardedPoolAllocator GuardedAlloc;
973 uptr GuardedAllocSlotSize = 0;
974#endif // GWP_ASAN_HOOKS
975
976 struct AllocationRingBuffer {
977 struct Entry {
978 atomic_uptr Ptr;
979 atomic_uptr AllocationSize;
980 atomic_u32 AllocationTrace;
981 atomic_u32 AllocationTid;
982 atomic_u32 DeallocationTrace;
983 atomic_u32 DeallocationTid;
984 };
985 StackDepot *Depot = nullptr;
986 uptr StackDepotSize = 0;
987 MemMapT RawRingBufferMap;
988 MemMapT RawStackDepotMap;
989 u32 RingBufferElements = 0;
990 atomic_uptr Pos;
991 // An array of Size (at least one) elements of type Entry is immediately
992 // following to this struct.
993 };
994 static_assert(sizeof(AllocationRingBuffer) %
995 alignof(typename AllocationRingBuffer::Entry) ==
996 0,
997 "invalid alignment");
998
999 // Lock to initialize the RingBuffer
1000 HybridMutex RingBufferInitLock;
1001
1002 // Pointer to memory mapped area starting with AllocationRingBuffer struct,
1003 // and immediately followed by Size elements of type Entry.
1004 atomic_uptr RingBufferAddress = {};
1005
1006 AllocationRingBuffer *getRingBuffer() {
1007 return reinterpret_cast<AllocationRingBuffer *>(
1008 atomic_load(A: &RingBufferAddress, MO: memory_order_acquire));
1009 }
1010
1011 // The following might get optimized out by the compiler.
1012 NOINLINE void performSanityChecks() {
1013 // Verify that the header offset field can hold the maximum offset. In the
1014 // case of the Secondary allocator, it takes care of alignment and the
1015 // offset will always be small. In the case of the Primary, the worst case
1016 // scenario happens in the last size class, when the backend allocation
1017 // would already be aligned on the requested alignment, which would happen
1018 // to be the maximum alignment that would fit in that size class. As a
1019 // result, the maximum offset will be at most the maximum alignment for the
1020 // last size class minus the header size, in multiples of MinAlignment.
1021 Chunk::UnpackedHeader Header = {};
1022 const uptr MaxPrimaryAlignment = 1UL << getMostSignificantSetBitIndex(
1023 SizeClassMap::MaxSize - MinAlignment);
1024 const uptr MaxOffset =
1025 (MaxPrimaryAlignment - Chunk::getHeaderSize()) >> MinAlignmentLog;
1026 Header.Offset = MaxOffset & Chunk::OffsetMask;
1027 if (UNLIKELY(Header.Offset != MaxOffset))
1028 reportSanityCheckError(Field: "offset");
1029
1030 // Verify that we can fit the maximum size or amount of unused bytes in the
1031 // header. Given that the Secondary fits the allocation to a page, the worst
1032 // case scenario happens in the Primary. It will depend on the second to
1033 // last and last class sizes, as well as the dynamic base for the Primary.
1034 // The following is an over-approximation that works for our needs.
1035 const uptr MaxSizeOrUnusedBytes = SizeClassMap::MaxSize - 1;
1036 Header.SizeOrUnusedBytes = MaxSizeOrUnusedBytes;
1037 if (UNLIKELY(Header.SizeOrUnusedBytes != MaxSizeOrUnusedBytes))
1038 reportSanityCheckError(Field: "size (or unused bytes)");
1039
1040 const uptr LargestClassId = SizeClassMap::LargestClassId;
1041 Header.ClassId = LargestClassId;
1042 if (UNLIKELY(Header.ClassId != LargestClassId))
1043 reportSanityCheckError(Field: "class ID");
1044 }
1045
1046 static inline void *getBlockBegin(const void *Ptr,
1047 Chunk::UnpackedHeader *Header) {
1048 return reinterpret_cast<void *>(
1049 reinterpret_cast<uptr>(Ptr) - Chunk::getHeaderSize() -
1050 (static_cast<uptr>(Header->Offset) << MinAlignmentLog));
1051 }
1052
1053 // Return the size of a chunk as requested during its allocation.
1054 inline uptr getSize(const void *Ptr, Chunk::UnpackedHeader *Header) {
1055 const uptr SizeOrUnusedBytes = Header->SizeOrUnusedBytes;
1056 if (LIKELY(Header->ClassId))
1057 return SizeOrUnusedBytes;
1058 if (allocatorSupportsMemoryTagging<AllocatorConfig>())
1059 Ptr = untagPointer(Ptr: const_cast<void *>(Ptr));
1060 return SecondaryT::getBlockEnd(getBlockBegin(Ptr, Header)) -
1061 reinterpret_cast<uptr>(Ptr) - SizeOrUnusedBytes;
1062 }
1063
1064 ALWAYS_INLINE void *initChunk(const uptr ClassId, const Chunk::Origin Origin,
1065 void *Block, const uptr UserPtr,
1066 const uptr SizeOrUnusedBytes,
1067 const FillContentsMode FillContents) {
1068 // Compute the default pointer before adding the header tag
1069 const uptr DefaultAlignedPtr =
1070 reinterpret_cast<uptr>(Block) + Chunk::getHeaderSize();
1071
1072 Block = addHeaderTag(Block);
1073 // Only do content fill when it's from primary allocator because secondary
1074 // allocator has filled the content.
1075 if (ClassId != 0 && UNLIKELY(FillContents != NoFill)) {
1076 // This condition is not necessarily unlikely, but since memset is
1077 // costly, we might as well mark it as such.
1078 memset(Block, FillContents == ZeroFill ? 0 : PatternFillByte,
1079 PrimaryT::getSizeByClassId(ClassId));
1080 }
1081
1082 Chunk::UnpackedHeader Header = {};
1083
1084 if (UNLIKELY(DefaultAlignedPtr != UserPtr)) {
1085 const uptr Offset = UserPtr - DefaultAlignedPtr;
1086 DCHECK_GE(Offset, 2 * sizeof(u32));
1087 // The BlockMarker has no security purpose, but is specifically meant for
1088 // the chunk iteration function that can be used in debugging situations.
1089 // It is the only situation where we have to locate the start of a chunk
1090 // based on its block address.
1091 reinterpret_cast<u32 *>(Block)[0] = BlockMarker;
1092 reinterpret_cast<u32 *>(Block)[1] = static_cast<u32>(Offset);
1093 Header.Offset = (Offset >> MinAlignmentLog) & Chunk::OffsetMask;
1094 }
1095
1096 Header.ClassId = ClassId & Chunk::ClassIdMask;
1097 Header.State = Chunk::State::Allocated;
1098 Header.OriginOrWasZeroed = Origin & Chunk::OriginMask;
1099 Header.SizeOrUnusedBytes = SizeOrUnusedBytes & Chunk::SizeOrUnusedBytesMask;
1100 Chunk::storeHeader(Cookie, Ptr: reinterpret_cast<void *>(addHeaderTag(UserPtr)),
1101 NewUnpackedHeader: &Header);
1102
1103 return reinterpret_cast<void *>(UserPtr);
1104 }
1105
1106 NOINLINE void *
1107 initChunkWithMemoryTagging(const uptr ClassId, const Chunk::Origin Origin,
1108 void *Block, const uptr UserPtr, const uptr Size,
1109 const uptr SizeOrUnusedBytes,
1110 const FillContentsMode FillContents) {
1111 const Options Options = Primary.Options.load();
1112 DCHECK(useMemoryTagging<AllocatorConfig>(Options));
1113
1114 // Compute the default pointer before adding the header tag
1115 const uptr DefaultAlignedPtr =
1116 reinterpret_cast<uptr>(Block) + Chunk::getHeaderSize();
1117
1118 void *Ptr = reinterpret_cast<void *>(UserPtr);
1119 void *TaggedPtr = Ptr;
1120
1121 if (LIKELY(ClassId)) {
1122 // Init the primary chunk.
1123 //
1124 // We only need to zero or tag the contents for Primary backed
1125 // allocations. We only set tags for primary allocations in order to avoid
1126 // faulting potentially large numbers of pages for large secondary
1127 // allocations. We assume that guard pages are enough to protect these
1128 // allocations.
1129 //
1130 // FIXME: When the kernel provides a way to set the background tag of a
1131 // mapping, we should be able to tag secondary allocations as well.
1132 //
1133 // When memory tagging is enabled, zeroing the contents is done as part of
1134 // setting the tag.
1135
1136 Chunk::UnpackedHeader Header;
1137 const uptr BlockSize = PrimaryT::getSizeByClassId(ClassId);
1138 const uptr BlockUptr = reinterpret_cast<uptr>(Block);
1139 const uptr BlockEnd = BlockUptr + BlockSize;
1140 // If possible, try to reuse the UAF tag that was set by deallocate().
1141 // For simplicity, only reuse tags if we have the same start address as
1142 // the previous allocation. This handles the majority of cases since
1143 // most allocations will not be more aligned than the minimum alignment.
1144 //
1145 // We need to handle situations involving reclaimed chunks, and retag
1146 // the reclaimed portions if necessary. In the case where the chunk is
1147 // fully reclaimed, the chunk's header will be zero, which will trigger
1148 // the code path for new mappings and invalid chunks that prepares the
1149 // chunk from scratch. There are three possibilities for partial
1150 // reclaiming:
1151 //
1152 // (1) Header was reclaimed, data was partially reclaimed.
1153 // (2) Header was not reclaimed, all data was reclaimed (e.g. because
1154 // data started on a page boundary).
1155 // (3) Header was not reclaimed, data was partially reclaimed.
1156 //
1157 // Case (1) will be handled in the same way as for full reclaiming,
1158 // since the header will be zero.
1159 //
1160 // We can detect case (2) by loading the tag from the start
1161 // of the chunk. If it is zero, it means that either all data was
1162 // reclaimed (since we never use zero as the chunk tag), or that the
1163 // previous allocation was of size zero. Either way, we need to prepare
1164 // a new chunk from scratch.
1165 //
1166 // We can detect case (3) by moving to the next page (if covered by the
1167 // chunk) and loading the tag of its first granule. If it is zero, it
1168 // means that all following pages may need to be retagged. On the other
1169 // hand, if it is nonzero, we can assume that all following pages are
1170 // still tagged, according to the logic that if any of the pages
1171 // following the next page were reclaimed, the next page would have been
1172 // reclaimed as well.
1173 uptr TaggedUserPtr;
1174 uptr PrevUserPtr;
1175 if (getChunkFromBlock(Block: BlockUptr, Chunk: &PrevUserPtr, Header: &Header) &&
1176 PrevUserPtr == UserPtr &&
1177 (TaggedUserPtr = loadTag(Ptr: UserPtr)) != UserPtr) {
1178 uptr PrevEnd = TaggedUserPtr + Header.SizeOrUnusedBytes;
1179 const uptr NextPage = roundUp(X: TaggedUserPtr, Boundary: getPageSizeCached());
1180 if (NextPage < PrevEnd && loadTag(Ptr: NextPage) != NextPage)
1181 PrevEnd = NextPage;
1182 TaggedPtr = reinterpret_cast<void *>(TaggedUserPtr);
1183 resizeTaggedChunk(OldPtr: PrevEnd, NewPtr: TaggedUserPtr + Size, NewSize: Size, BlockEnd);
1184 if (UNLIKELY(FillContents != NoFill && !Header.OriginOrWasZeroed)) {
1185 // If an allocation needs to be zeroed (i.e. calloc) we can normally
1186 // avoid zeroing the memory now since we can rely on memory having
1187 // been zeroed on free, as this is normally done while setting the
1188 // UAF tag. But if tagging was disabled per-thread when the memory
1189 // was freed, it would not have been retagged and thus zeroed, and
1190 // therefore it needs to be zeroed now.
1191 memset(s: TaggedPtr, c: 0,
1192 n: Min(A: Size, B: roundUp(X: PrevEnd - TaggedUserPtr,
1193 Boundary: archMemoryTagGranuleSize())));
1194 } else if (Size) {
1195 // Clear any stack metadata that may have previously been stored in
1196 // the chunk data.
1197 memset(s: TaggedPtr, c: 0, n: archMemoryTagGranuleSize());
1198 }
1199 } else {
1200 const uptr OddEvenMask =
1201 computeOddEvenMaskForPointerMaybe(Options, Ptr: BlockUptr, ClassId);
1202 TaggedPtr = prepareTaggedChunk(Ptr, Size, ExcludeMask: OddEvenMask, BlockEnd);
1203 }
1204 storePrimaryAllocationStackMaybe(Options, Ptr);
1205 } else {
1206 // Init the secondary chunk.
1207
1208 Block = addHeaderTag(Block);
1209 Ptr = addHeaderTag(Ptr);
1210 storeTags(Begin: reinterpret_cast<uptr>(Block), End: reinterpret_cast<uptr>(Ptr));
1211 storeSecondaryAllocationStackMaybe(Options, Ptr, Size);
1212 }
1213
1214 Chunk::UnpackedHeader Header = {};
1215
1216 if (UNLIKELY(DefaultAlignedPtr != UserPtr)) {
1217 const uptr Offset = UserPtr - DefaultAlignedPtr;
1218 DCHECK_GE(Offset, 2 * sizeof(u32));
1219 // The BlockMarker has no security purpose, but is specifically meant for
1220 // the chunk iteration function that can be used in debugging situations.
1221 // It is the only situation where we have to locate the start of a chunk
1222 // based on its block address.
1223 reinterpret_cast<u32 *>(Block)[0] = BlockMarker;
1224 reinterpret_cast<u32 *>(Block)[1] = static_cast<u32>(Offset);
1225 Header.Offset = (Offset >> MinAlignmentLog) & Chunk::OffsetMask;
1226 }
1227
1228 Header.ClassId = ClassId & Chunk::ClassIdMask;
1229 Header.State = Chunk::State::Allocated;
1230 Header.OriginOrWasZeroed = Origin & Chunk::OriginMask;
1231 Header.SizeOrUnusedBytes = SizeOrUnusedBytes & Chunk::SizeOrUnusedBytesMask;
1232 Chunk::storeHeader(Cookie, Ptr, NewUnpackedHeader: &Header);
1233
1234 return TaggedPtr;
1235 }
1236
1237 void quarantineOrDeallocateChunk(const Options &Options, void *TaggedPtr,
1238 Chunk::UnpackedHeader *Header,
1239 uptr Size) NO_THREAD_SAFETY_ANALYSIS {
1240 void *Ptr = getHeaderTaggedPointer(Ptr: TaggedPtr);
1241 // If the quarantine is disabled, the actual size of a chunk is 0 or larger
1242 // than the maximum allowed, we return a chunk directly to the backend.
1243 // This purposefully underflows for Size == 0.
1244 const bool BypassQuarantine = !Quarantine.getCacheSize() ||
1245 ((Size - 1) >= QuarantineMaxChunkSize) ||
1246 !Header->ClassId;
1247 if (BypassQuarantine)
1248 Header->State = Chunk::State::Available;
1249 else
1250 Header->State = Chunk::State::Quarantined;
1251
1252 void *BlockBegin;
1253 if (LIKELY(!useMemoryTagging<AllocatorConfig>(Options))) {
1254 Header->OriginOrWasZeroed = 0U;
1255 if (BypassQuarantine && allocatorSupportsMemoryTagging<AllocatorConfig>())
1256 Ptr = untagPointer(Ptr);
1257 BlockBegin = getBlockBegin(Ptr, Header);
1258 } else {
1259 Header->OriginOrWasZeroed =
1260 Header->ClassId && !TSDRegistry.getDisableMemInit();
1261 BlockBegin =
1262 retagBlock(Options, TaggedPtr, Ptr, Header, Size, BypassQuarantine);
1263 }
1264
1265 Chunk::storeHeader(Cookie, Ptr, NewUnpackedHeader: Header);
1266
1267 if (BypassQuarantine) {
1268 const uptr ClassId = Header->ClassId;
1269 if (LIKELY(ClassId)) {
1270 bool CacheDrained;
1271 {
1272 typename TSDRegistryT::ScopedTSD TSD(TSDRegistry);
1273 CacheDrained = TSD->getCache().deallocate(ClassId, BlockBegin);
1274 }
1275 // When we have drained some blocks back to the Primary from TSD, that
1276 // implies that we may have the chance to release some pages as well.
1277 // Note that in order not to block other thread's accessing the TSD,
1278 // release the TSD first then try the page release.
1279 if (CacheDrained)
1280 Primary.tryReleaseToOS(ClassId, ReleaseToOS::Normal);
1281 } else {
1282 Secondary.deallocate(Options, BlockBegin);
1283 }
1284 } else {
1285 typename TSDRegistryT::ScopedTSD TSD(TSDRegistry);
1286 Quarantine.put(&TSD->getQuarantineCache(),
1287 QuarantineCallback(*this, TSD->getCache()), Ptr, Size);
1288 }
1289 }
1290
1291 NOINLINE void *retagBlock(const Options &Options, void *TaggedPtr, void *&Ptr,
1292 Chunk::UnpackedHeader *Header, const uptr Size,
1293 bool BypassQuarantine) {
1294 DCHECK(useMemoryTagging<AllocatorConfig>(Options));
1295
1296 const u8 PrevTag = extractTag(Ptr: reinterpret_cast<uptr>(TaggedPtr));
1297 storeDeallocationStackMaybe(Options, Ptr, PrevTag, Size);
1298 if (Header->ClassId && !TSDRegistry.getDisableMemInit()) {
1299 uptr TaggedBegin, TaggedEnd;
1300 const uptr OddEvenMask = computeOddEvenMaskForPointerMaybe(
1301 Options, Ptr: reinterpret_cast<uptr>(getBlockBegin(Ptr, Header)),
1302 ClassId: Header->ClassId);
1303 // Exclude the previous tag so that immediate use after free is
1304 // detected 100% of the time.
1305 setRandomTag(Ptr, Size, ExcludeMask: OddEvenMask | (1UL << PrevTag), TaggedBegin: &TaggedBegin,
1306 TaggedEnd: &TaggedEnd);
1307 }
1308
1309 Ptr = untagPointer(Ptr);
1310 void *BlockBegin = getBlockBegin(Ptr, Header);
1311 if (BypassQuarantine && !Header->ClassId) {
1312 storeTags(Begin: reinterpret_cast<uptr>(BlockBegin),
1313 End: reinterpret_cast<uptr>(Ptr));
1314 }
1315
1316 return BlockBegin;
1317 }
1318
1319 bool getChunkFromBlock(uptr Block, uptr *Chunk,
1320 Chunk::UnpackedHeader *Header) {
1321 *Chunk =
1322 Block + getChunkOffsetFromBlock(Block: reinterpret_cast<const char *>(Block));
1323 return Chunk::isValid(Cookie, Ptr: reinterpret_cast<void *>(*Chunk), NewUnpackedHeader: Header);
1324 }
1325
1326 static uptr getChunkOffsetFromBlock(const char *Block) {
1327 u32 Offset = 0;
1328 if (reinterpret_cast<const u32 *>(Block)[0] == BlockMarker)
1329 Offset = reinterpret_cast<const u32 *>(Block)[1];
1330 return Offset + Chunk::getHeaderSize();
1331 }
1332
1333 // Set the tag of the granule past the end of the allocation to 0, to catch
1334 // linear overflows even if a previous larger allocation used the same block
1335 // and tag. Only do this if the granule past the end is in our block, because
1336 // this would otherwise lead to a SEGV if the allocation covers the entire
1337 // block and our block is at the end of a mapping. The tag of the next block's
1338 // header granule will be set to 0, so it will serve the purpose of catching
1339 // linear overflows in this case.
1340 //
1341 // For allocations of size 0 we do not end up storing the address tag to the
1342 // memory tag space, which getInlineErrorInfo() normally relies on to match
1343 // address tags against chunks. To allow matching in this case we store the
1344 // address tag in the first byte of the chunk.
1345 void storeEndMarker(uptr End, uptr Size, uptr BlockEnd) {
1346 DCHECK_EQ(BlockEnd, untagPointer(BlockEnd));
1347 uptr UntaggedEnd = untagPointer(Ptr: End);
1348 if (UntaggedEnd != BlockEnd) {
1349 storeTag(Ptr: UntaggedEnd);
1350 if (Size == 0)
1351 *reinterpret_cast<u8 *>(UntaggedEnd) = extractTag(Ptr: End);
1352 }
1353 }
1354
1355 void *prepareTaggedChunk(void *Ptr, uptr Size, uptr ExcludeMask,
1356 uptr BlockEnd) {
1357 // Prepare the granule before the chunk to store the chunk header by setting
1358 // its tag to 0. Normally its tag will already be 0, but in the case where a
1359 // chunk holding a low alignment allocation is reused for a higher alignment
1360 // allocation, the chunk may already have a non-zero tag from the previous
1361 // allocation.
1362 storeTag(Ptr: reinterpret_cast<uptr>(Ptr) - archMemoryTagGranuleSize());
1363
1364 uptr TaggedBegin, TaggedEnd;
1365 setRandomTag(Ptr, Size, ExcludeMask, TaggedBegin: &TaggedBegin, TaggedEnd: &TaggedEnd);
1366
1367 storeEndMarker(End: TaggedEnd, Size, BlockEnd);
1368 return reinterpret_cast<void *>(TaggedBegin);
1369 }
1370
1371 void resizeTaggedChunk(uptr OldPtr, uptr NewPtr, uptr NewSize,
1372 uptr BlockEnd) {
1373 uptr RoundOldPtr = roundUp(X: OldPtr, Boundary: archMemoryTagGranuleSize());
1374 uptr RoundNewPtr;
1375 if (RoundOldPtr >= NewPtr) {
1376 // If the allocation is shrinking we just need to set the tag past the end
1377 // of the allocation to 0. See explanation in storeEndMarker() above.
1378 RoundNewPtr = roundUp(X: NewPtr, Boundary: archMemoryTagGranuleSize());
1379 } else {
1380 // Set the memory tag of the region
1381 // [RoundOldPtr, roundUp(NewPtr, archMemoryTagGranuleSize()))
1382 // to the pointer tag stored in OldPtr.
1383 RoundNewPtr = storeTags(Begin: RoundOldPtr, End: NewPtr);
1384 }
1385 storeEndMarker(End: RoundNewPtr, Size: NewSize, BlockEnd);
1386 }
1387
1388 void storePrimaryAllocationStackMaybe(const Options &Options, void *Ptr) {
1389 if (!UNLIKELY(Options.get(OptionBit::TrackAllocationStacks)))
1390 return;
1391 AllocationRingBuffer *RB = getRingBuffer();
1392 if (!RB)
1393 return;
1394 auto *Ptr32 = reinterpret_cast<u32 *>(Ptr);
1395 Ptr32[MemTagAllocationTraceIndex] = collectStackTrace(Depot: RB->Depot);
1396 Ptr32[MemTagAllocationTidIndex] = getThreadID();
1397 }
1398
1399 void storeRingBufferEntry(AllocationRingBuffer *RB, void *Ptr,
1400 u32 AllocationTrace, u32 AllocationTid,
1401 uptr AllocationSize, u32 DeallocationTrace,
1402 u32 DeallocationTid) {
1403 uptr Pos = atomic_fetch_add(&RB->Pos, 1, memory_order_relaxed);
1404 typename AllocationRingBuffer::Entry *Entry =
1405 getRingBufferEntry(RB, Pos % RB->RingBufferElements);
1406
1407 // First invalidate our entry so that we don't attempt to interpret a
1408 // partially written state in getSecondaryErrorInfo(). The fences below
1409 // ensure that the compiler does not move the stores to Ptr in between the
1410 // stores to the other fields.
1411 atomic_store_relaxed(&Entry->Ptr, 0);
1412
1413 __atomic_signal_fence(__ATOMIC_SEQ_CST);
1414 atomic_store_relaxed(&Entry->AllocationTrace, AllocationTrace);
1415 atomic_store_relaxed(&Entry->AllocationTid, AllocationTid);
1416 atomic_store_relaxed(&Entry->AllocationSize, AllocationSize);
1417 atomic_store_relaxed(&Entry->DeallocationTrace, DeallocationTrace);
1418 atomic_store_relaxed(&Entry->DeallocationTid, DeallocationTid);
1419 __atomic_signal_fence(__ATOMIC_SEQ_CST);
1420
1421 atomic_store_relaxed(&Entry->Ptr, reinterpret_cast<uptr>(Ptr));
1422 }
1423
1424 void storeSecondaryAllocationStackMaybe(const Options &Options, void *Ptr,
1425 uptr Size) {
1426 if (!UNLIKELY(Options.get(OptionBit::TrackAllocationStacks)))
1427 return;
1428 AllocationRingBuffer *RB = getRingBuffer();
1429 if (!RB)
1430 return;
1431 u32 Trace = collectStackTrace(Depot: RB->Depot);
1432 u32 Tid = getThreadID();
1433
1434 auto *Ptr32 = reinterpret_cast<u32 *>(Ptr);
1435 Ptr32[MemTagAllocationTraceIndex] = Trace;
1436 Ptr32[MemTagAllocationTidIndex] = Tid;
1437
1438 storeRingBufferEntry(RB, Ptr: untagPointer(Ptr), AllocationTrace: Trace, AllocationTid: Tid, AllocationSize: Size, DeallocationTrace: 0, DeallocationTid: 0);
1439 }
1440
1441 void storeDeallocationStackMaybe(const Options &Options, void *Ptr,
1442 u8 PrevTag, uptr Size) {
1443 if (!UNLIKELY(Options.get(OptionBit::TrackAllocationStacks)))
1444 return;
1445 AllocationRingBuffer *RB = getRingBuffer();
1446 if (!RB)
1447 return;
1448 auto *Ptr32 = reinterpret_cast<u32 *>(Ptr);
1449 u32 AllocationTrace = Ptr32[MemTagAllocationTraceIndex];
1450 u32 AllocationTid = Ptr32[MemTagAllocationTidIndex];
1451
1452 u32 DeallocationTrace = collectStackTrace(Depot: RB->Depot);
1453 u32 DeallocationTid = getThreadID();
1454
1455 storeRingBufferEntry(RB, Ptr: addFixedTag(Ptr: untagPointer(Ptr), Tag: PrevTag),
1456 AllocationTrace, AllocationTid, AllocationSize: Size,
1457 DeallocationTrace, DeallocationTid);
1458 }
1459
1460 static const size_t NumErrorReports =
1461 sizeof(((scudo_error_info *)nullptr)->reports) /
1462 sizeof(((scudo_error_info *)nullptr)->reports[0]);
1463
1464 static void getInlineErrorInfo(struct scudo_error_info *ErrorInfo,
1465 size_t &NextErrorReport, uintptr_t FaultAddr,
1466 const StackDepot *Depot,
1467 const char *RegionInfoPtr, const char *Memory,
1468 const char *MemoryTags, uintptr_t MemoryAddr,
1469 size_t MemorySize, size_t MinDistance,
1470 size_t MaxDistance) {
1471 uptr UntaggedFaultAddr = untagPointer(Ptr: FaultAddr);
1472 u8 FaultAddrTag = extractTag(Ptr: FaultAddr);
1473 BlockInfo Info =
1474 PrimaryT::findNearestBlock(RegionInfoPtr, UntaggedFaultAddr);
1475
1476 auto GetGranule = [&](uptr Addr, const char **Data, uint8_t *Tag) -> bool {
1477 if (Addr < MemoryAddr || Addr + archMemoryTagGranuleSize() < Addr ||
1478 Addr + archMemoryTagGranuleSize() > MemoryAddr + MemorySize)
1479 return false;
1480 *Data = &Memory[Addr - MemoryAddr];
1481 *Tag = static_cast<u8>(
1482 MemoryTags[(Addr - MemoryAddr) / archMemoryTagGranuleSize()]);
1483 return true;
1484 };
1485
1486 auto ReadBlock = [&](uptr Addr, uptr *ChunkAddr,
1487 Chunk::UnpackedHeader *Header, const u32 **Data,
1488 u8 *Tag) {
1489 const char *BlockBegin;
1490 u8 BlockBeginTag;
1491 if (!GetGranule(Addr, &BlockBegin, &BlockBeginTag))
1492 return false;
1493 uptr ChunkOffset = getChunkOffsetFromBlock(Block: BlockBegin);
1494 *ChunkAddr = Addr + ChunkOffset;
1495
1496 const char *ChunkBegin;
1497 if (!GetGranule(*ChunkAddr, &ChunkBegin, Tag))
1498 return false;
1499 *Header = *reinterpret_cast<const Chunk::UnpackedHeader *>(
1500 ChunkBegin - Chunk::getHeaderSize());
1501 *Data = reinterpret_cast<const u32 *>(ChunkBegin);
1502
1503 // Allocations of size 0 will have stashed the tag in the first byte of
1504 // the chunk, see storeEndMarker().
1505 if (Header->SizeOrUnusedBytes == 0)
1506 *Tag = static_cast<u8>(*ChunkBegin);
1507
1508 return true;
1509 };
1510
1511 if (NextErrorReport == NumErrorReports)
1512 return;
1513
1514 auto CheckOOB = [&](uptr BlockAddr) {
1515 if (BlockAddr < Info.RegionBegin || BlockAddr >= Info.RegionEnd)
1516 return false;
1517
1518 uptr ChunkAddr;
1519 Chunk::UnpackedHeader Header;
1520 const u32 *Data;
1521 uint8_t Tag;
1522 if (!ReadBlock(BlockAddr, &ChunkAddr, &Header, &Data, &Tag) ||
1523 Header.State != Chunk::State::Allocated || Tag != FaultAddrTag)
1524 return false;
1525
1526 auto *R = &ErrorInfo->reports[NextErrorReport++];
1527 R->error_type =
1528 UntaggedFaultAddr < ChunkAddr ? BUFFER_UNDERFLOW : BUFFER_OVERFLOW;
1529 R->allocation_address = ChunkAddr;
1530 R->allocation_size = Header.SizeOrUnusedBytes;
1531 if (Depot) {
1532 collectTraceMaybe(Depot, Trace&: R->allocation_trace,
1533 Hash: Data[MemTagAllocationTraceIndex]);
1534 }
1535 R->allocation_tid = Data[MemTagAllocationTidIndex];
1536 return NextErrorReport == NumErrorReports;
1537 };
1538
1539 if (MinDistance == 0 && CheckOOB(Info.BlockBegin))
1540 return;
1541
1542 for (size_t I = Max<size_t>(A: MinDistance, B: 1); I != MaxDistance; ++I)
1543 if (CheckOOB(Info.BlockBegin + I * Info.BlockSize) ||
1544 CheckOOB(Info.BlockBegin - I * Info.BlockSize))
1545 return;
1546 }
1547
1548 static void getRingBufferErrorInfo(struct scudo_error_info *ErrorInfo,
1549 size_t &NextErrorReport,
1550 uintptr_t FaultAddr,
1551 const StackDepot *Depot,
1552 const char *RingBufferPtr,
1553 size_t RingBufferSize) {
1554 auto *RingBuffer =
1555 reinterpret_cast<const AllocationRingBuffer *>(RingBufferPtr);
1556 size_t RingBufferElements = ringBufferElementsFromBytes(Bytes: RingBufferSize);
1557 if (!RingBuffer || RingBufferElements == 0 || !Depot)
1558 return;
1559 uptr Pos = atomic_load_relaxed(&RingBuffer->Pos);
1560
1561 for (uptr I = Pos - 1; I != Pos - 1 - RingBufferElements &&
1562 NextErrorReport != NumErrorReports;
1563 --I) {
1564 auto *Entry = getRingBufferEntry(RingBuffer, I % RingBufferElements);
1565 uptr EntryPtr = atomic_load_relaxed(&Entry->Ptr);
1566 if (!EntryPtr)
1567 continue;
1568
1569 uptr UntaggedEntryPtr = untagPointer(Ptr: EntryPtr);
1570 uptr EntrySize = atomic_load_relaxed(&Entry->AllocationSize);
1571 u32 AllocationTrace = atomic_load_relaxed(&Entry->AllocationTrace);
1572 u32 AllocationTid = atomic_load_relaxed(&Entry->AllocationTid);
1573 u32 DeallocationTrace = atomic_load_relaxed(&Entry->DeallocationTrace);
1574 u32 DeallocationTid = atomic_load_relaxed(&Entry->DeallocationTid);
1575
1576 if (DeallocationTid) {
1577 // For UAF we only consider in-bounds fault addresses because
1578 // out-of-bounds UAF is rare and attempting to detect it is very likely
1579 // to result in false positives.
1580 if (FaultAddr < EntryPtr || FaultAddr >= EntryPtr + EntrySize)
1581 continue;
1582 } else {
1583 // Ring buffer OOB is only possible with secondary allocations. In this
1584 // case we are guaranteed a guard region of at least a page on either
1585 // side of the allocation (guard page on the right, guard page + tagged
1586 // region on the left), so ignore any faults outside of that range.
1587 if (FaultAddr < EntryPtr - getPageSizeCached() ||
1588 FaultAddr >= EntryPtr + EntrySize + getPageSizeCached())
1589 continue;
1590
1591 // For UAF the ring buffer will contain two entries, one for the
1592 // allocation and another for the deallocation. Don't report buffer
1593 // overflow/underflow using the allocation entry if we have already
1594 // collected a report from the deallocation entry.
1595 bool Found = false;
1596 for (uptr J = 0; J != NextErrorReport; ++J) {
1597 if (ErrorInfo->reports[J].allocation_address == UntaggedEntryPtr) {
1598 Found = true;
1599 break;
1600 }
1601 }
1602 if (Found)
1603 continue;
1604 }
1605
1606 auto *R = &ErrorInfo->reports[NextErrorReport++];
1607 if (DeallocationTid)
1608 R->error_type = USE_AFTER_FREE;
1609 else if (FaultAddr < EntryPtr)
1610 R->error_type = BUFFER_UNDERFLOW;
1611 else
1612 R->error_type = BUFFER_OVERFLOW;
1613
1614 R->allocation_address = UntaggedEntryPtr;
1615 R->allocation_size = EntrySize;
1616 collectTraceMaybe(Depot, Trace&: R->allocation_trace, Hash: AllocationTrace);
1617 R->allocation_tid = AllocationTid;
1618 collectTraceMaybe(Depot, Trace&: R->deallocation_trace, Hash: DeallocationTrace);
1619 R->deallocation_tid = DeallocationTid;
1620 }
1621 }
1622
1623 uptr getStats(ScopedString *Str) {
1624 Primary.getStats(Str);
1625 Secondary.getStats(Str);
1626 Quarantine.getStats(Str);
1627 TSDRegistry.getStats(Str);
1628 return Str->length();
1629 }
1630
1631 static typename AllocationRingBuffer::Entry *
1632 getRingBufferEntry(AllocationRingBuffer *RB, uptr N) {
1633 char *RBEntryStart =
1634 &reinterpret_cast<char *>(RB)[sizeof(AllocationRingBuffer)];
1635 return &reinterpret_cast<typename AllocationRingBuffer::Entry *>(
1636 RBEntryStart)[N];
1637 }
1638 static const typename AllocationRingBuffer::Entry *
1639 getRingBufferEntry(const AllocationRingBuffer *RB, uptr N) {
1640 const char *RBEntryStart =
1641 &reinterpret_cast<const char *>(RB)[sizeof(AllocationRingBuffer)];
1642 return &reinterpret_cast<const typename AllocationRingBuffer::Entry *>(
1643 RBEntryStart)[N];
1644 }
1645
1646 void initRingBufferMaybe() {
1647 ScopedLock L(RingBufferInitLock);
1648 if (getRingBuffer() != nullptr)
1649 return;
1650
1651 int ring_buffer_size = getFlags()->allocation_ring_buffer_size;
1652 if (ring_buffer_size <= 0)
1653 return;
1654
1655 u32 AllocationRingBufferSize = static_cast<u32>(ring_buffer_size);
1656
1657 // We store alloc and free stacks for each entry.
1658 constexpr u32 kStacksPerRingBufferEntry = 2;
1659 constexpr u32 kMaxU32Pow2 = ~(UINT32_MAX >> 1);
1660 static_assert(isPowerOfTwo(X: kMaxU32Pow2));
1661 // On Android we always have 3 frames at the bottom: __start_main,
1662 // __libc_init, main, and 3 at the top: malloc, scudo_malloc and
1663 // Allocator::allocate. This leaves 10 frames for the user app. The next
1664 // smallest power of two (8) would only leave 2, which is clearly too
1665 // little.
1666 constexpr u32 kFramesPerStack = 16;
1667 static_assert(isPowerOfTwo(X: kFramesPerStack));
1668
1669 if (AllocationRingBufferSize > kMaxU32Pow2 / kStacksPerRingBufferEntry)
1670 return;
1671 u32 TabSize = static_cast<u32>(roundUpPowerOfTwo(Size: kStacksPerRingBufferEntry *
1672 AllocationRingBufferSize));
1673 if (TabSize > UINT32_MAX / kFramesPerStack)
1674 return;
1675 u32 RingSize = static_cast<u32>(TabSize * kFramesPerStack);
1676
1677 uptr StackDepotSize = sizeof(StackDepot) + sizeof(atomic_u64) * RingSize +
1678 sizeof(atomic_u32) * TabSize;
1679 MemMapT DepotMap;
1680 DepotMap.map(
1681 /*Addr=*/Addr: 0U, Size: roundUp(X: StackDepotSize, Boundary: getPageSizeCached()),
1682 Name: "scudo:stack_depot");
1683 auto *Depot = reinterpret_cast<StackDepot *>(DepotMap.getBase());
1684 Depot->init(RingSz: RingSize, TabSz: TabSize);
1685
1686 MemMapT MemMap;
1687 MemMap.map(
1688 /*Addr=*/Addr: 0U,
1689 Size: roundUp(X: ringBufferSizeInBytes(RingBufferElements: AllocationRingBufferSize),
1690 Boundary: getPageSizeCached()),
1691 Name: "scudo:ring_buffer");
1692 auto *RB = reinterpret_cast<AllocationRingBuffer *>(MemMap.getBase());
1693 RB->RawRingBufferMap = MemMap;
1694 RB->RingBufferElements = AllocationRingBufferSize;
1695 RB->Depot = Depot;
1696 RB->StackDepotSize = StackDepotSize;
1697 RB->RawStackDepotMap = DepotMap;
1698
1699 atomic_store(A: &RingBufferAddress, V: reinterpret_cast<uptr>(RB),
1700 MO: memory_order_release);
1701 }
1702
1703 void unmapRingBuffer() {
1704 AllocationRingBuffer *RB = getRingBuffer();
1705 if (RB == nullptr)
1706 return;
1707 // N.B. because RawStackDepotMap is part of RawRingBufferMap, the order
1708 // is very important.
1709 RB->RawStackDepotMap.unmap(RB->RawStackDepotMap.getBase(),
1710 RB->RawStackDepotMap.getCapacity());
1711 // Note that the `RB->RawRingBufferMap` is stored on the pages managed by
1712 // itself. Take over the ownership before calling unmap() so that any
1713 // operation along with unmap() won't touch inaccessible pages.
1714 MemMapT RawRingBufferMap = RB->RawRingBufferMap;
1715 RawRingBufferMap.unmap(Addr: RawRingBufferMap.getBase(),
1716 Size: RawRingBufferMap.getCapacity());
1717 atomic_store(A: &RingBufferAddress, V: 0, MO: memory_order_release);
1718 }
1719
1720 static constexpr size_t ringBufferSizeInBytes(u32 RingBufferElements) {
1721 return sizeof(AllocationRingBuffer) +
1722 RingBufferElements * sizeof(typename AllocationRingBuffer::Entry);
1723 }
1724
1725 static constexpr size_t ringBufferElementsFromBytes(size_t Bytes) {
1726 if (Bytes < sizeof(AllocationRingBuffer)) {
1727 return 0;
1728 }
1729 return (Bytes - sizeof(AllocationRingBuffer)) /
1730 sizeof(typename AllocationRingBuffer::Entry);
1731 }
1732};
1733
1734} // namespace scudo
1735
1736#endif // SCUDO_COMBINED_H_
1737