1//===-- asan_allocator.cpp ------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is a part of AddressSanitizer, an address sanity checker.
10//
11// Implementation of ASan's memory allocator, 2-nd version.
12// This variant uses the allocator from sanitizer_common, i.e. the one shared
13// with ThreadSanitizer and MemorySanitizer.
14//
15//===----------------------------------------------------------------------===//
16
17#include "asan_allocator.h"
18
19#include "asan_internal.h"
20#include "asan_mapping.h"
21#include "asan_poisoning.h"
22#include "asan_report.h"
23#include "asan_stack.h"
24#include "asan_suppressions.h"
25#include "asan_thread.h"
26#include "lsan/lsan_common.h"
27#include "sanitizer_common/sanitizer_allocator_checks.h"
28#include "sanitizer_common/sanitizer_allocator_interface.h"
29#include "sanitizer_common/sanitizer_common.h"
30#include "sanitizer_common/sanitizer_errno.h"
31#include "sanitizer_common/sanitizer_flags.h"
32#include "sanitizer_common/sanitizer_internal_defs.h"
33#include "sanitizer_common/sanitizer_list.h"
34#include "sanitizer_common/sanitizer_quarantine.h"
35#include "sanitizer_common/sanitizer_stackdepot.h"
36
37namespace __asan {
38
39// Valid redzone sizes are 16, 32, 64, ... 2048, so we encode them in 3 bits.
40// We use adaptive redzones: for larger allocation larger redzones are used.
41static u32 RZLog2Size(u32 rz_log) {
42 CHECK_LT(rz_log, 8);
43 return 16 << rz_log;
44}
45
46static u32 RZSize2Log(u32 rz_size) {
47 CHECK_GE(rz_size, 16);
48 CHECK_LE(rz_size, 2048);
49 CHECK(IsPowerOfTwo(rz_size));
50 u32 res = Log2(x: rz_size) - 4;
51 CHECK_EQ(rz_size, RZLog2Size(res));
52 return res;
53}
54
55static AsanAllocator &get_allocator();
56
57static void AtomicContextStore(volatile atomic_uint64_t *atomic_context,
58 u32 tid, u32 stack) {
59 u64 context = tid;
60 context <<= 32;
61 context += stack;
62 atomic_store(a: atomic_context, v: context, mo: memory_order_relaxed);
63}
64
65static void AtomicContextLoad(const volatile atomic_uint64_t *atomic_context,
66 u32 &tid, u32 &stack) {
67 u64 context = atomic_load(a: atomic_context, mo: memory_order_relaxed);
68 stack = context;
69 context >>= 32;
70 tid = context;
71}
72
73// The memory chunk allocated from the underlying allocator looks like this:
74// L L L L L L H H U U U U U U R R
75// L -- left redzone words (0 or more bytes)
76// H -- ChunkHeader (16 bytes), which is also a part of the left redzone.
77// U -- user memory.
78// R -- right redzone (0 or more bytes)
79// ChunkBase consists of ChunkHeader and other bytes that overlap with user
80// memory.
81
82// If the left redzone is greater than the ChunkHeader size we store a magic
83// value in the first uptr word of the memory block and store the address of
84// ChunkBase in the next uptr.
85// M B L L L L L L L L L H H U U U U U U
86// | ^
87// ---------------------|
88// M -- magic value kAllocBegMagic
89// B -- address of ChunkHeader pointing to the first 'H'
90
91class ChunkHeader {
92 public:
93 atomic_uint8_t chunk_state;
94 u8 alloc_type : 2;
95 u8 lsan_tag : 2;
96#if SANITIZER_WINDOWS
97 // True if this was a zero-size allocation upgraded to size 1.
98 // Used to report the original size (0) to the user via HeapSize/RtlSizeHeap.
99 u8 from_zero_alloc : 1;
100#endif
101
102 // align < 8 -> 0
103 // else -> log2(min(align, 512)) - 2
104 u8 user_requested_alignment_log : 3;
105
106 private:
107 u16 user_requested_size_hi;
108 u32 user_requested_size_lo;
109 atomic_uint64_t alloc_context_id;
110
111 public:
112 uptr UsedSize() const {
113 static_assert(sizeof(user_requested_size_lo) == 4,
114 "Expression below requires this");
115 return FIRST_32_SECOND_64(0, ((uptr)user_requested_size_hi << 32)) +
116 user_requested_size_lo;
117 }
118
119 void SetUsedSize(uptr size) {
120 user_requested_size_lo = size;
121 static_assert(sizeof(user_requested_size_lo) == 4,
122 "Expression below requires this");
123 user_requested_size_hi = FIRST_32_SECOND_64(0, size >> 32);
124 CHECK_EQ(UsedSize(), size);
125 }
126
127 void SetAllocContext(u32 tid, u32 stack) {
128 AtomicContextStore(atomic_context: &alloc_context_id, tid, stack);
129 }
130
131 void GetAllocContext(u32 &tid, u32 &stack) const {
132 AtomicContextLoad(atomic_context: &alloc_context_id, tid, stack);
133 }
134};
135
136class ChunkBase : public ChunkHeader {
137 atomic_uint64_t free_context_id;
138
139 public:
140 void SetFreeContext(u32 tid, u32 stack) {
141 AtomicContextStore(atomic_context: &free_context_id, tid, stack);
142 }
143
144 void GetFreeContext(u32 &tid, u32 &stack) const {
145 AtomicContextLoad(atomic_context: &free_context_id, tid, stack);
146 }
147};
148
149static const uptr kChunkHeaderSize = sizeof(ChunkHeader);
150static const uptr kChunkHeader2Size = sizeof(ChunkBase) - kChunkHeaderSize;
151COMPILER_CHECK(kChunkHeaderSize == 16);
152COMPILER_CHECK(kChunkHeader2Size <= 16);
153
154enum {
155 // Either just allocated by underlying allocator, but AsanChunk is not yet
156 // ready, or almost returned to undelying allocator and AsanChunk is already
157 // meaningless.
158 CHUNK_INVALID = 0,
159 // The chunk is allocated and not yet freed.
160 CHUNK_ALLOCATED = 2,
161 // The chunk was freed and put into quarantine zone.
162 CHUNK_QUARANTINE = 3,
163};
164
165class AsanChunk : public ChunkBase {
166 public:
167 uptr Beg() { return reinterpret_cast<uptr>(this) + kChunkHeaderSize; }
168 bool AddrIsInside(uptr addr) {
169 return (addr >= Beg()) && (addr < Beg() + UsedSize());
170 }
171};
172
173class LargeChunkHeader {
174 static constexpr uptr kAllocBegMagic =
175 FIRST_32_SECOND_64(0xCC6E96B9, 0xCC6E96B9CC6E96B9ULL);
176 atomic_uintptr_t magic;
177 AsanChunk *chunk_header;
178
179 public:
180 AsanChunk *Get() const {
181 return atomic_load(a: &magic, mo: memory_order_acquire) == kAllocBegMagic
182 ? chunk_header
183 : nullptr;
184 }
185
186 void Set(AsanChunk *p) {
187 if (p) {
188 chunk_header = p;
189 atomic_store(a: &magic, v: kAllocBegMagic, mo: memory_order_release);
190 return;
191 }
192
193 uptr old = kAllocBegMagic;
194 if (!atomic_compare_exchange_strong(a: &magic, cmp: &old, xchg: 0,
195 mo: memory_order_release)) {
196 CHECK_EQ(old, kAllocBegMagic);
197 }
198 }
199};
200
201static void FillChunk(AsanChunk *m) {
202 // FIXME: Use ReleaseMemoryPagesToOS.
203 Flags &fl = *flags();
204
205 if (fl.max_free_fill_size > 0) {
206 // We have to skip the chunk header, it contains free_context_id.
207 uptr scribble_start = (uptr)m + kChunkHeaderSize + kChunkHeader2Size;
208 if (m->UsedSize() >= kChunkHeader2Size) { // Skip Header2 in user area.
209 uptr size_to_fill = m->UsedSize() - kChunkHeader2Size;
210 size_to_fill = Min(a: size_to_fill, b: (uptr)fl.max_free_fill_size);
211 REAL(memset)((void *)scribble_start, fl.free_fill_byte, size_to_fill);
212 }
213 }
214}
215
216struct QuarantineCallback {
217 QuarantineCallback(AllocatorCache *cache, BufferedStackTrace *stack)
218 : cache_(cache),
219 stack_(stack) {
220 }
221
222 void PreQuarantine(AsanChunk *m) const {
223 FillChunk(m);
224 // Poison the region.
225 PoisonShadow(addr: m->Beg(), size: RoundUpTo(size: m->UsedSize(), ASAN_SHADOW_GRANULARITY),
226 value: kAsanHeapFreeMagic);
227 }
228
229 void Recycle(AsanChunk *m) const {
230 void *p = get_allocator().GetBlockBegin(p: m);
231
232 // The secondary will immediately unpoison and unmap the memory, so this
233 // branch is unnecessary.
234 if (get_allocator().FromPrimary(p)) {
235 if (p != m) {
236 // Clear the magic value, as allocator internals may overwrite the
237 // contents of deallocated chunk, confusing GetAsanChunk lookup.
238 reinterpret_cast<LargeChunkHeader *>(p)->Set(nullptr);
239 }
240
241 u8 old_chunk_state = CHUNK_QUARANTINE;
242 if (!atomic_compare_exchange_strong(a: &m->chunk_state, cmp: &old_chunk_state,
243 xchg: CHUNK_INVALID,
244 mo: memory_order_acquire)) {
245 CHECK_EQ(old_chunk_state, CHUNK_QUARANTINE);
246 }
247
248 PoisonShadow(addr: m->Beg(), size: RoundUpTo(size: m->UsedSize(), ASAN_SHADOW_GRANULARITY),
249 value: kAsanHeapLeftRedzoneMagic);
250 }
251
252 // Statistics.
253 AsanStats &thread_stats = GetCurrentThreadStats();
254 thread_stats.real_frees++;
255 thread_stats.really_freed += m->UsedSize();
256
257 get_allocator().Deallocate(cache: cache_, p);
258 }
259
260 void RecyclePassThrough(AsanChunk *m) const {
261 // Recycle for the secondary will immediately unpoison and unmap the
262 // memory, so quarantine preparation is unnecessary.
263 if (get_allocator().FromPrimary(p: m)) {
264 // The primary allocation may need pattern fill if enabled.
265 FillChunk(m);
266 }
267 Recycle(m);
268 }
269
270 void *Allocate(uptr size) const {
271 void *res = get_allocator().Allocate(cache: cache_, size, alignment: 1);
272 // TODO(alekseys): Consider making quarantine OOM-friendly.
273 if (UNLIKELY(!res))
274 ReportOutOfMemory(requested_size: size, stack: stack_);
275 return res;
276 }
277
278 void Deallocate(void *p) const { get_allocator().Deallocate(cache: cache_, p); }
279
280 private:
281 AllocatorCache* const cache_;
282 BufferedStackTrace* const stack_;
283};
284
285typedef Quarantine<QuarantineCallback, AsanChunk> AsanQuarantine;
286typedef AsanQuarantine::Cache QuarantineCache;
287
288void AsanMapUnmapCallback::OnMap(uptr p, uptr size) const {
289 PoisonShadow(addr: p, size, value: kAsanHeapLeftRedzoneMagic);
290 // Statistics.
291 AsanStats &thread_stats = GetCurrentThreadStats();
292 thread_stats.mmaps++;
293 thread_stats.mmaped += size;
294}
295
296void AsanMapUnmapCallback::OnMapSecondary(uptr p, uptr size, uptr user_begin,
297 uptr user_size) const {
298 uptr user_end = RoundDownTo(x: user_begin + user_size, ASAN_SHADOW_GRANULARITY);
299 user_begin = RoundUpTo(size: user_begin, ASAN_SHADOW_GRANULARITY);
300 // The secondary mapping will be immediately returned to user, no value
301 // poisoning that with non-zero just before unpoisoning by Allocate(). So just
302 // poison head/tail invisible to Allocate().
303 PoisonShadow(addr: p, size: user_begin - p, value: kAsanHeapLeftRedzoneMagic);
304 PoisonShadow(addr: user_end, size: size - (user_end - p), value: kAsanHeapLeftRedzoneMagic);
305 // Statistics.
306 AsanStats &thread_stats = GetCurrentThreadStats();
307 thread_stats.mmaps++;
308 thread_stats.mmaped += size;
309}
310
311void AsanMapUnmapCallback::OnUnmap(uptr p, uptr size) const {
312 PoisonShadow(addr: p, size, value: 0);
313 // We are about to unmap a chunk of user memory.
314 // Mark the corresponding shadow memory as not needed.
315 FlushUnneededASanShadowMemory(p, size);
316 // Statistics.
317 AsanStats &thread_stats = GetCurrentThreadStats();
318 thread_stats.munmaps++;
319 thread_stats.munmaped += size;
320}
321
322// We can not use THREADLOCAL because it is not supported on some of the
323// platforms we care about (OSX 10.6, Android).
324// static THREADLOCAL AllocatorCache cache;
325AllocatorCache *GetAllocatorCache(AsanThreadLocalMallocStorage *ms) {
326 CHECK(ms);
327 return &ms->allocator_cache;
328}
329
330QuarantineCache *GetQuarantineCache(AsanThreadLocalMallocStorage *ms) {
331 CHECK(ms);
332 CHECK_LE(sizeof(QuarantineCache), sizeof(ms->quarantine_cache));
333 return reinterpret_cast<QuarantineCache *>(ms->quarantine_cache);
334}
335
336void AllocatorOptions::SetFrom(const Flags *f, const CommonFlags *cf) {
337 quarantine_size_mb = f->quarantine_size_mb;
338 thread_local_quarantine_size_kb = f->thread_local_quarantine_size_kb;
339 min_redzone = f->redzone;
340 max_redzone = f->max_redzone;
341 may_return_null = cf->allocator_may_return_null;
342 alloc_dealloc_mismatch = f->alloc_dealloc_mismatch;
343 release_to_os_interval_ms = cf->allocator_release_to_os_interval_ms;
344}
345
346void AllocatorOptions::CopyTo(Flags *f, CommonFlags *cf) {
347 f->quarantine_size_mb = quarantine_size_mb;
348 f->thread_local_quarantine_size_kb = thread_local_quarantine_size_kb;
349 f->redzone = min_redzone;
350 f->max_redzone = max_redzone;
351 cf->allocator_may_return_null = may_return_null;
352 f->alloc_dealloc_mismatch = alloc_dealloc_mismatch;
353 cf->allocator_release_to_os_interval_ms = release_to_os_interval_ms;
354}
355
356struct Allocator {
357 static const uptr kMaxAllowedMallocSize =
358 FIRST_32_SECOND_64(3UL << 30, 1ULL << 40);
359
360 AsanAllocator allocator;
361 AsanQuarantine quarantine;
362 StaticSpinMutex fallback_mutex;
363 AllocatorCache fallback_allocator_cache;
364 QuarantineCache fallback_quarantine_cache;
365
366 uptr max_user_defined_malloc_size;
367
368 // ------------------- Options --------------------------
369 atomic_uint16_t min_redzone;
370 atomic_uint16_t max_redzone;
371 atomic_uint8_t alloc_dealloc_mismatch;
372
373 // ------------------- Initialization ------------------------
374 explicit Allocator(LinkerInitialized)
375 : quarantine(LINKER_INITIALIZED),
376 fallback_quarantine_cache(LINKER_INITIALIZED) {}
377
378 void CheckOptions(const AllocatorOptions &options) const {
379 CHECK_GE(options.min_redzone, 16);
380 CHECK_GE(options.max_redzone, options.min_redzone);
381 CHECK_LE(options.max_redzone, 2048);
382 CHECK(IsPowerOfTwo(options.min_redzone));
383 CHECK(IsPowerOfTwo(options.max_redzone));
384 }
385
386 void SharedInitCode(const AllocatorOptions &options) {
387 CheckOptions(options);
388 quarantine.Init(size: (uptr)options.quarantine_size_mb << 20,
389 cache_size: (uptr)options.thread_local_quarantine_size_kb << 10);
390 atomic_store(a: &alloc_dealloc_mismatch, v: options.alloc_dealloc_mismatch,
391 mo: memory_order_release);
392 atomic_store(a: &min_redzone, v: options.min_redzone, mo: memory_order_release);
393 atomic_store(a: &max_redzone, v: options.max_redzone, mo: memory_order_release);
394 }
395
396 void InitLinkerInitialized(const AllocatorOptions &options) {
397 SetAllocatorMayReturnNull(options.may_return_null);
398 allocator.InitLinkerInitialized(release_to_os_interval_ms: options.release_to_os_interval_ms);
399 SharedInitCode(options);
400 max_user_defined_malloc_size = common_flags()->max_allocation_size_mb
401 ? common_flags()->max_allocation_size_mb
402 << 20
403 : kMaxAllowedMallocSize;
404 }
405
406 void RePoisonChunk(uptr chunk) {
407 // This could be a user-facing chunk (with redzones), or some internal
408 // housekeeping chunk, like TransferBatch. Start by assuming the former.
409 AsanChunk *ac = GetAsanChunk(alloc_beg: (void *)chunk);
410 uptr allocated_size = allocator.GetActuallyAllocatedSize(p: (void *)chunk);
411 if (ac && atomic_load(a: &ac->chunk_state, mo: memory_order_acquire) ==
412 CHUNK_ALLOCATED) {
413 uptr beg = ac->Beg();
414 uptr end = ac->Beg() + ac->UsedSize();
415 uptr chunk_end = chunk + allocated_size;
416 if (chunk < beg && beg < end && end <= chunk_end) {
417 // Looks like a valid AsanChunk in use, poison redzones only.
418 PoisonShadow(addr: chunk, size: beg - chunk, value: kAsanHeapLeftRedzoneMagic);
419 uptr end_aligned_down = RoundDownTo(x: end, ASAN_SHADOW_GRANULARITY);
420 FastPoisonShadowPartialRightRedzone(
421 aligned_addr: end_aligned_down, size: end - end_aligned_down,
422 redzone_size: chunk_end - end_aligned_down, value: kAsanHeapLeftRedzoneMagic);
423 return;
424 }
425 }
426
427 // This is either not an AsanChunk or freed or quarantined AsanChunk.
428 // In either case, poison everything.
429 PoisonShadow(addr: chunk, size: allocated_size, value: kAsanHeapLeftRedzoneMagic);
430 }
431
432 // Apply provided AllocatorOptions to an Allocator
433 void ApplyOptions(const AllocatorOptions &options) {
434 SetAllocatorMayReturnNull(options.may_return_null);
435 allocator.SetReleaseToOSIntervalMs(options.release_to_os_interval_ms);
436 SharedInitCode(options);
437 }
438
439 void ReInitialize(const AllocatorOptions &options) {
440 ApplyOptions(options);
441
442 // Poison all existing allocation's redzones.
443 if (CanPoisonMemory()) {
444 allocator.ForceLock();
445 allocator.ForEachChunk(
446 callback: [](uptr chunk, void *alloc) {
447 ((Allocator *)alloc)->RePoisonChunk(chunk);
448 },
449 arg: this);
450 allocator.ForceUnlock();
451 }
452 }
453
454 void GetOptions(AllocatorOptions *options) const {
455 options->quarantine_size_mb = quarantine.GetMaxSize() >> 20;
456 options->thread_local_quarantine_size_kb =
457 quarantine.GetMaxCacheSize() >> 10;
458 options->min_redzone = atomic_load(a: &min_redzone, mo: memory_order_acquire);
459 options->max_redzone = atomic_load(a: &max_redzone, mo: memory_order_acquire);
460 options->may_return_null = AllocatorMayReturnNull();
461 options->alloc_dealloc_mismatch =
462 atomic_load(a: &alloc_dealloc_mismatch, mo: memory_order_acquire);
463 options->release_to_os_interval_ms = allocator.ReleaseToOSIntervalMs();
464 }
465
466 // -------------------- Helper methods. -------------------------
467 uptr ComputeRZLog(uptr user_requested_size) {
468 u32 rz_log = user_requested_size <= 64 - 16 ? 0
469 : user_requested_size <= 128 - 32 ? 1
470 : user_requested_size <= 512 - 64 ? 2
471 : user_requested_size <= 4096 - 128 ? 3
472 : user_requested_size <= (1 << 14) - 256 ? 4
473 : user_requested_size <= (1 << 15) - 512 ? 5
474 : user_requested_size <= (1 << 16) - 1024 ? 6
475 : 7;
476 u32 hdr_log = RZSize2Log(rz_size: RoundUpToPowerOfTwo(size: sizeof(ChunkHeader)));
477 u32 min_log = RZSize2Log(rz_size: atomic_load(a: &min_redzone, mo: memory_order_acquire));
478 u32 max_log = RZSize2Log(rz_size: atomic_load(a: &max_redzone, mo: memory_order_acquire));
479 return Min(a: Max(a: rz_log, b: Max(a: min_log, b: hdr_log)), b: Max(a: max_log, b: hdr_log));
480 }
481
482 static uptr ComputeUserRequestedAlignmentLog(uptr user_requested_alignment) {
483 if (user_requested_alignment < 8)
484 return 0;
485 if (user_requested_alignment > 512)
486 user_requested_alignment = 512;
487 return Log2(x: user_requested_alignment) - 2;
488 }
489
490 static uptr ComputeUserAlignment(uptr user_requested_alignment_log) {
491 if (user_requested_alignment_log == 0)
492 return 0;
493 return 1LL << (user_requested_alignment_log + 2);
494 }
495
496 // We have an address between two chunks, and we want to report just one.
497 AsanChunk *ChooseChunk(uptr addr, AsanChunk *left_chunk,
498 AsanChunk *right_chunk) {
499 if (!left_chunk)
500 return right_chunk;
501 if (!right_chunk)
502 return left_chunk;
503 // Prefer an allocated chunk over freed chunk and freed chunk
504 // over available chunk.
505 u8 left_state = atomic_load(a: &left_chunk->chunk_state, mo: memory_order_relaxed);
506 u8 right_state =
507 atomic_load(a: &right_chunk->chunk_state, mo: memory_order_relaxed);
508 if (left_state != right_state) {
509 if (left_state == CHUNK_ALLOCATED)
510 return left_chunk;
511 if (right_state == CHUNK_ALLOCATED)
512 return right_chunk;
513 if (left_state == CHUNK_QUARANTINE)
514 return left_chunk;
515 if (right_state == CHUNK_QUARANTINE)
516 return right_chunk;
517 }
518 // Same chunk_state: choose based on offset.
519 sptr l_offset = 0, r_offset = 0;
520 CHECK(AsanChunkView(left_chunk).AddrIsAtRight(addr, 1, &l_offset));
521 CHECK(AsanChunkView(right_chunk).AddrIsAtLeft(addr, 1, &r_offset));
522 if (l_offset < r_offset)
523 return left_chunk;
524 return right_chunk;
525 }
526
527 bool UpdateAllocationStack(uptr addr, BufferedStackTrace *stack) {
528 AsanChunk *m = GetAsanChunkByAddr(p: addr);
529 if (!m) return false;
530 if (atomic_load(a: &m->chunk_state, mo: memory_order_acquire) != CHUNK_ALLOCATED)
531 return false;
532 if (m->Beg() != addr) return false;
533 AsanThread *t = GetCurrentThread();
534 m->SetAllocContext(tid: t ? t->tid() : kMainTid, stack: StackDepotPut(stack: *stack));
535 return true;
536 }
537
538 // -------------------- Allocation/Deallocation routines ---------------
539 void *Allocate(uptr size, uptr alignment, BufferedStackTrace *stack,
540 AllocType alloc_type, bool can_fill) {
541 if (UNLIKELY(!AsanInited()))
542 AsanInitFromRtl();
543 if (UNLIKELY(IsRssLimitExceeded())) {
544 if (AllocatorMayReturnNull())
545 return nullptr;
546 ReportRssLimitExceeded(stack);
547 }
548 Flags &fl = *flags();
549 CHECK(stack);
550 const uptr min_alignment = ASAN_SHADOW_GRANULARITY;
551 const uptr user_requested_alignment_log =
552 ComputeUserRequestedAlignmentLog(user_requested_alignment: alignment);
553 if (alignment < min_alignment)
554 alignment = min_alignment;
555 bool upgraded_from_zero = false;
556 if (size == 0) {
557 // We'd be happy to avoid allocating memory for zero-size requests, but
558 // some programs/tests depend on this behavior and assume that malloc
559 // would not return NULL even for zero-size allocations. Moreover, it
560 // looks like operator new should never return NULL, and results of
561 // consecutive "new" calls must be different even if the allocated size
562 // is zero.
563 size = 1;
564 upgraded_from_zero = true;
565 }
566 CHECK(IsPowerOfTwo(alignment));
567 uptr rz_log = ComputeRZLog(user_requested_size: size);
568 uptr rz_size = RZLog2Size(rz_log);
569 uptr rounded_size = RoundUpTo(size: Max(a: size, b: kChunkHeader2Size), boundary: alignment);
570 uptr needed_size = rounded_size + rz_size;
571 if (alignment > min_alignment)
572 needed_size += alignment;
573 bool from_primary = PrimaryAllocator::CanAllocate(size: needed_size, alignment);
574 // If we are allocating from the secondary allocator, there will be no
575 // automatic right redzone, so add the right redzone manually.
576 if (!from_primary)
577 needed_size += rz_size;
578 CHECK(IsAligned(needed_size, min_alignment));
579 if (size > kMaxAllowedMallocSize || needed_size > kMaxAllowedMallocSize ||
580 size > max_user_defined_malloc_size) {
581 if (AllocatorMayReturnNull()) {
582 Report(format: "WARNING: AddressSanitizer failed to allocate 0x%zx bytes\n",
583 size);
584 return nullptr;
585 }
586 uptr malloc_limit =
587 Min(a: kMaxAllowedMallocSize, b: max_user_defined_malloc_size);
588 ReportAllocationSizeTooBig(user_size: size, total_size: needed_size, max_size: malloc_limit, stack);
589 }
590
591 AsanThread *t = GetCurrentThread();
592 void *allocated;
593 if (t) {
594 AllocatorCache *cache = GetAllocatorCache(ms: &t->malloc_storage());
595 allocated = allocator.Allocate(cache, size: needed_size, alignment: 8);
596 } else {
597 SpinMutexLock l(&fallback_mutex);
598 AllocatorCache *cache = &fallback_allocator_cache;
599 allocated = allocator.Allocate(cache, size: needed_size, alignment: 8);
600 }
601 if (UNLIKELY(!allocated)) {
602 SetAllocatorOutOfMemory();
603 if (AllocatorMayReturnNull())
604 return nullptr;
605 ReportOutOfMemory(requested_size: size, stack);
606 }
607
608 uptr alloc_beg = reinterpret_cast<uptr>(allocated);
609 uptr alloc_end = alloc_beg + needed_size;
610 uptr user_beg = alloc_beg + rz_size;
611 if (!IsAligned(a: user_beg, alignment))
612 user_beg = RoundUpTo(size: user_beg, boundary: alignment);
613 uptr user_end = user_beg + size;
614 CHECK_LE(user_end, alloc_end);
615 uptr chunk_beg = user_beg - kChunkHeaderSize;
616 AsanChunk *m = reinterpret_cast<AsanChunk *>(chunk_beg);
617 m->alloc_type = alloc_type;
618#if SANITIZER_WINDOWS
619 m->from_zero_alloc = upgraded_from_zero;
620#endif
621 CHECK(size);
622 m->SetUsedSize(size);
623 m->user_requested_alignment_log = user_requested_alignment_log;
624
625 m->SetAllocContext(tid: t ? t->tid() : kMainTid, stack: StackDepotPut(stack: *stack));
626
627 if (!from_primary || *(u8 *)MEM_TO_SHADOW((uptr)allocated) == 0) {
628 // The allocator provides an unpoisoned chunk. This is possible for the
629 // secondary allocator, or if CanPoisonMemory() was false for some time,
630 // for example, due to flags()->start_disabled. Anyway, poison left and
631 // right of the block before using it for anything else.
632 uptr tail_beg = RoundUpTo(size: user_end, ASAN_SHADOW_GRANULARITY);
633 uptr tail_end = alloc_beg + allocator.GetActuallyAllocatedSize(p: allocated);
634 PoisonShadow(addr: alloc_beg, size: user_beg - alloc_beg, value: kAsanHeapLeftRedzoneMagic);
635 PoisonShadow(addr: tail_beg, size: tail_end - tail_beg, value: kAsanHeapLeftRedzoneMagic);
636 }
637
638 uptr size_rounded_down_to_granularity =
639 RoundDownTo(x: size, ASAN_SHADOW_GRANULARITY);
640 // Unpoison the bulk of the memory region.
641 if (size_rounded_down_to_granularity)
642 PoisonShadow(addr: user_beg, size: size_rounded_down_to_granularity, value: 0);
643 // Deal with the end of the region if size is not aligned to granularity.
644 if (size != size_rounded_down_to_granularity && CanPoisonMemory()) {
645 u8 *shadow =
646 (u8 *)MemToShadow(p: user_beg + size_rounded_down_to_granularity);
647 *shadow = fl.poison_partial ? (size & (ASAN_SHADOW_GRANULARITY - 1)) : 0;
648 }
649
650 if (upgraded_from_zero)
651 PoisonShadow(addr: user_beg, ASAN_SHADOW_GRANULARITY,
652 value: kAsanHeapLeftRedzoneMagic);
653
654 AsanStats &thread_stats = GetCurrentThreadStats();
655 thread_stats.mallocs++;
656 thread_stats.malloced += size;
657 thread_stats.malloced_redzones += needed_size - size;
658 if (needed_size > SizeClassMap::kMaxSize)
659 thread_stats.malloc_large++;
660 else
661 thread_stats.malloced_by_size[SizeClassMap::ClassID(size: needed_size)]++;
662
663 void *res = reinterpret_cast<void *>(user_beg);
664 if (can_fill && fl.max_malloc_fill_size) {
665 uptr fill_size = Min(a: size, b: (uptr)fl.max_malloc_fill_size);
666 REAL(memset)(res, fl.malloc_fill_byte, fill_size);
667 }
668#if CAN_SANITIZE_LEAKS
669 m->lsan_tag = __lsan::DisabledInThisThread() ? __lsan::kIgnored
670 : __lsan::kDirectlyLeaked;
671#endif
672 // Must be the last mutation of metadata in this function.
673 atomic_store(a: &m->chunk_state, v: CHUNK_ALLOCATED, mo: memory_order_release);
674 if (alloc_beg != chunk_beg) {
675 CHECK_LE(alloc_beg + sizeof(LargeChunkHeader), chunk_beg);
676 reinterpret_cast<LargeChunkHeader *>(alloc_beg)->Set(m);
677 }
678 RunMallocHooks(ptr: res, size);
679 return res;
680 }
681
682 // Set quarantine flag if chunk is allocated, issue ASan error report on
683 // available and quarantined chunks. Return true on success, false otherwise.
684 bool AtomicallySetQuarantineFlagIfAllocated(AsanChunk *m, void *ptr,
685 BufferedStackTrace *stack) {
686 u8 old_chunk_state = CHUNK_ALLOCATED;
687 // Flip the chunk_state atomically to avoid race on double-free.
688 if (!atomic_compare_exchange_strong(a: &m->chunk_state, cmp: &old_chunk_state,
689 xchg: CHUNK_QUARANTINE,
690 mo: memory_order_acquire)) {
691 ReportInvalidFree(ptr, chunk_state: old_chunk_state, stack);
692 // It's not safe to push a chunk in quarantine on invalid free.
693 return false;
694 }
695 CHECK_EQ(CHUNK_ALLOCATED, old_chunk_state);
696 // It was a user data.
697 m->SetFreeContext(tid: kInvalidTid, stack: 0);
698 return true;
699 }
700
701 // Expects the chunk to already be marked as quarantined by using
702 // AtomicallySetQuarantineFlagIfAllocated.
703 void QuarantineChunk(AsanChunk *m, void *ptr, BufferedStackTrace *stack) {
704 CHECK_EQ(atomic_load(&m->chunk_state, memory_order_relaxed),
705 CHUNK_QUARANTINE);
706 AsanThread *t = GetCurrentThread();
707 m->SetFreeContext(tid: t ? t->tid() : 0, stack: StackDepotPut(stack: *stack));
708
709 // Push into quarantine.
710 if (t) {
711 AsanThreadLocalMallocStorage *ms = &t->malloc_storage();
712 AllocatorCache *ac = GetAllocatorCache(ms);
713 quarantine.Put(c: GetQuarantineCache(ms), cb: QuarantineCallback(ac, stack), ptr: m,
714 size: m->UsedSize());
715 } else {
716 SpinMutexLock l(&fallback_mutex);
717 AllocatorCache *ac = &fallback_allocator_cache;
718 quarantine.Put(c: &fallback_quarantine_cache, cb: QuarantineCallback(ac, stack),
719 ptr: m, size: m->UsedSize());
720 }
721 }
722
723 void Deallocate(void *ptr, uptr delete_size, uptr delete_alignment,
724 BufferedStackTrace *stack, AllocType alloc_type) {
725 uptr p = reinterpret_cast<uptr>(ptr);
726 if (p == 0) return;
727
728 uptr chunk_beg = p - kChunkHeaderSize;
729 AsanChunk *m = reinterpret_cast<AsanChunk *>(chunk_beg);
730
731 // On Windows, uninstrumented DLLs may allocate memory before ASan hooks
732 // malloc. Don't report an invalid free in this case.
733 if (SANITIZER_WINDOWS &&
734 !get_allocator().PointerIsMine(p: ptr)) {
735 if (!IsSystemHeapAddress(addr: p))
736 ReportFreeNotMalloced(addr: p, free_stack: stack);
737 return;
738 }
739
740 if (RunFreeHooks(ptr)) {
741 // Someone used __sanitizer_ignore_free_hook() and decided that they
742 // didn't want the memory to __sanitizer_ignore_free_hook freed right now.
743 // When they call free() on this pointer again at a later time, we should
744 // ignore the alloc-type mismatch and allow them to deallocate the pointer
745 // through free(), rather than the initial alloc type.
746 m->alloc_type = FROM_MALLOC;
747 return;
748 }
749
750 // Must mark the chunk as quarantined before any changes to its metadata.
751 // Do not quarantine given chunk if we failed to set CHUNK_QUARANTINE flag.
752 if (!AtomicallySetQuarantineFlagIfAllocated(m, ptr, stack)) return;
753
754 if (m->alloc_type != alloc_type) {
755 if (atomic_load(a: &alloc_dealloc_mismatch, mo: memory_order_acquire) &&
756 !IsAllocDeallocMismatchSuppressed(stack)) {
757 ReportAllocTypeMismatch(addr: (uptr)ptr, free_stack: stack, alloc_type: (AllocType)m->alloc_type,
758 dealloc_type: (AllocType)alloc_type);
759 }
760 } else {
761 switch (alloc_type) {
762 case FROM_NEW:
763 case FROM_NEW_BR:
764 if (flags()->new_delete_type_mismatch &&
765 ((delete_size && delete_size != m->UsedSize()) ||
766 ComputeUserRequestedAlignmentLog(user_requested_alignment: delete_alignment) !=
767 m->user_requested_alignment_log)) {
768 ReportNewDeleteTypeMismatch(addr: p, delete_size, delete_alignment,
769 free_stack: stack);
770 }
771 break;
772 case FROM_MALLOC:
773 if (flags()->free_size_mismatch &&
774 ((delete_size && delete_size != m->UsedSize()) ||
775 (delete_alignment &&
776 ComputeUserRequestedAlignmentLog(user_requested_alignment: delete_alignment) !=
777 m->user_requested_alignment_log))) {
778 ReportFreeSizeMismatch(addr: p, delete_size, delete_alignment, free_stack: stack);
779 }
780 break;
781 }
782 }
783
784 AsanStats &thread_stats = GetCurrentThreadStats();
785 thread_stats.frees++;
786 thread_stats.freed += m->UsedSize();
787
788 QuarantineChunk(m, ptr, stack);
789 }
790
791 void *Reallocate(void *old_ptr, uptr new_size, BufferedStackTrace *stack) {
792 CHECK(old_ptr && new_size);
793 uptr p = reinterpret_cast<uptr>(old_ptr);
794 uptr chunk_beg = p - kChunkHeaderSize;
795 AsanChunk *m = reinterpret_cast<AsanChunk *>(chunk_beg);
796
797 AsanStats &thread_stats = GetCurrentThreadStats();
798 thread_stats.reallocs++;
799 thread_stats.realloced += new_size;
800
801 void *new_ptr = Allocate(size: new_size, alignment: 8, stack, alloc_type: FROM_MALLOC, can_fill: true);
802 if (new_ptr) {
803 u8 chunk_state = atomic_load(a: &m->chunk_state, mo: memory_order_acquire);
804 if (chunk_state != CHUNK_ALLOCATED)
805 ReportInvalidFree(ptr: old_ptr, chunk_state, stack);
806 CHECK_NE(REAL(memcpy), nullptr);
807 uptr memcpy_size = Min(a: new_size, b: m->UsedSize());
808 // If realloc() races with free(), we may start copying freed memory.
809 // However, we will report racy double-free later anyway.
810 REAL(memcpy)(new_ptr, old_ptr, memcpy_size);
811 Deallocate(ptr: old_ptr, delete_size: 0, delete_alignment: 0, stack, alloc_type: FROM_MALLOC);
812 }
813 return new_ptr;
814 }
815
816 void* Calloc(uptr nmemb, uptr size, BufferedStackTrace* stack,
817 uptr align = 8) {
818 if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
819 if (AllocatorMayReturnNull())
820 return nullptr;
821 ReportCallocOverflow(count: nmemb, size, stack);
822 }
823 void* ptr = Allocate(size: nmemb * size, alignment: align, stack, alloc_type: FROM_MALLOC, can_fill: false);
824 // If the memory comes from the secondary allocator no need to clear it
825 // as it comes directly from mmap.
826 if (ptr && allocator.FromPrimary(p: ptr))
827 REAL(memset)(ptr, 0, nmemb * size);
828 return ptr;
829 }
830
831 void ReportInvalidFree(void *ptr, u8 chunk_state, BufferedStackTrace *stack) {
832 if (chunk_state == CHUNK_QUARANTINE)
833 ReportDoubleFree(addr: (uptr)ptr, free_stack: stack);
834 else
835 ReportFreeNotMalloced(addr: (uptr)ptr, free_stack: stack);
836 }
837
838 void CommitBack(AsanThreadLocalMallocStorage *ms, BufferedStackTrace *stack) {
839 AllocatorCache *ac = GetAllocatorCache(ms);
840 quarantine.Drain(c: GetQuarantineCache(ms), cb: QuarantineCallback(ac, stack));
841 allocator.SwallowCache(cache: ac);
842 }
843
844 // -------------------------- Chunk lookup ----------------------
845
846 // Assumes alloc_beg == allocator.GetBlockBegin(alloc_beg).
847 // Returns nullptr if AsanChunk is not yet initialized just after
848 // get_allocator().Allocate(), or is being destroyed just before
849 // get_allocator().Deallocate().
850 AsanChunk *GetAsanChunk(void *alloc_beg) {
851 if (!alloc_beg)
852 return nullptr;
853 AsanChunk *p = reinterpret_cast<LargeChunkHeader *>(alloc_beg)->Get();
854 if (!p) {
855 if (!allocator.FromPrimary(p: alloc_beg))
856 return nullptr;
857 p = reinterpret_cast<AsanChunk *>(alloc_beg);
858 }
859 u8 state = atomic_load(a: &p->chunk_state, mo: memory_order_relaxed);
860 // It does not guaranty that Chunk is initialized, but it's
861 // definitely not for any other value.
862 if (state == CHUNK_ALLOCATED || state == CHUNK_QUARANTINE)
863 return p;
864 return nullptr;
865 }
866
867 AsanChunk *GetAsanChunkByAddr(uptr p) {
868 void *alloc_beg = allocator.GetBlockBegin(p: reinterpret_cast<void *>(p));
869 return GetAsanChunk(alloc_beg);
870 }
871
872 // Allocator must be locked when this function is called.
873 AsanChunk *GetAsanChunkByAddrFastLocked(uptr p) {
874 void *alloc_beg =
875 allocator.GetBlockBeginFastLocked(p: reinterpret_cast<void *>(p));
876 return GetAsanChunk(alloc_beg);
877 }
878
879 uptr AllocationSize(uptr p) {
880 AsanChunk *m = GetAsanChunkByAddr(p);
881 if (!m) return 0;
882 if (atomic_load(a: &m->chunk_state, mo: memory_order_acquire) != CHUNK_ALLOCATED)
883 return 0;
884 if (m->Beg() != p) return 0;
885 return m->UsedSize();
886 }
887
888#if SANITIZER_WINDOWS
889 // Returns true if the allocation at p was a zero-size request that was
890 // internally upgraded to size 1.
891 bool FromZeroAllocation(uptr p) {
892 return reinterpret_cast<AsanChunk*>(p - kChunkHeaderSize)->from_zero_alloc;
893 }
894
895 // Marks an existing size 1 allocation as having originally been zero-size.
896 // Used by SharedReAlloc which augments size 0 to 1 before calling
897 // asan_realloc, bypassing Allocate's own zero-size tracking.
898 void MarkAsZeroAllocation(uptr p) {
899 AsanChunk* m = reinterpret_cast<AsanChunk*>(p - kChunkHeaderSize);
900 m->from_zero_alloc = 1;
901 PoisonShadow(p, ASAN_SHADOW_GRANULARITY, kAsanHeapLeftRedzoneMagic);
902 }
903#endif
904
905 uptr AllocationSizeFast(uptr p) {
906 return reinterpret_cast<AsanChunk *>(p - kChunkHeaderSize)->UsedSize();
907 }
908
909 AsanChunkView FindHeapChunkByAddress(uptr addr) {
910 AsanChunk *m1 = GetAsanChunkByAddr(p: addr);
911 sptr offset = 0;
912 if (!m1 || AsanChunkView(m1).AddrIsAtLeft(addr, access_size: 1, offset: &offset)) {
913 // The address is in the chunk's left redzone, so maybe it is actually
914 // a right buffer overflow from the other chunk before.
915 // Search a bit before to see if there is another chunk.
916 AsanChunk *m2 = nullptr;
917 for (uptr l = 1; l < GetPageSizeCached(); l++) {
918 m2 = GetAsanChunkByAddr(p: addr - l);
919 if (m2 == m1) continue; // Still the same chunk.
920 break;
921 }
922 if (m2 && AsanChunkView(m2).AddrIsAtRight(addr, access_size: 1, offset: &offset))
923 m1 = ChooseChunk(addr, left_chunk: m2, right_chunk: m1);
924 }
925 return AsanChunkView(m1);
926 }
927
928 void Purge(BufferedStackTrace *stack) {
929 AsanThread *t = GetCurrentThread();
930 if (t) {
931 AsanThreadLocalMallocStorage *ms = &t->malloc_storage();
932 quarantine.DrainAndRecycle(c: GetQuarantineCache(ms),
933 cb: QuarantineCallback(GetAllocatorCache(ms),
934 stack));
935 }
936 {
937 SpinMutexLock l(&fallback_mutex);
938 quarantine.DrainAndRecycle(c: &fallback_quarantine_cache,
939 cb: QuarantineCallback(&fallback_allocator_cache,
940 stack));
941 }
942
943 allocator.ForceReleaseToOS();
944 }
945
946 void PrintStats() {
947 allocator.PrintStats();
948 quarantine.PrintStats();
949 }
950
951 void ForceLock() SANITIZER_ACQUIRE(fallback_mutex) {
952 allocator.ForceLock();
953 fallback_mutex.Lock();
954 }
955
956 void ForceUnlock() SANITIZER_RELEASE(fallback_mutex) {
957 fallback_mutex.Unlock();
958 allocator.ForceUnlock();
959 }
960};
961
962static Allocator instance(LINKER_INITIALIZED);
963
964static AsanAllocator &get_allocator() {
965 return instance.allocator;
966}
967
968bool AsanChunkView::IsValid() const {
969 return chunk_ && atomic_load(a: &chunk_->chunk_state, mo: memory_order_relaxed) !=
970 CHUNK_INVALID;
971}
972bool AsanChunkView::IsAllocated() const {
973 return chunk_ && atomic_load(a: &chunk_->chunk_state, mo: memory_order_relaxed) ==
974 CHUNK_ALLOCATED;
975}
976bool AsanChunkView::IsQuarantined() const {
977 return chunk_ && atomic_load(a: &chunk_->chunk_state, mo: memory_order_relaxed) ==
978 CHUNK_QUARANTINE;
979}
980uptr AsanChunkView::Beg() const { return chunk_->Beg(); }
981uptr AsanChunkView::End() const { return Beg() + UsedSize(); }
982uptr AsanChunkView::UsedSize() const { return chunk_->UsedSize(); }
983u32 AsanChunkView::UserRequestedAlignment() const {
984 return Allocator::ComputeUserAlignment(user_requested_alignment_log: chunk_->user_requested_alignment_log);
985}
986
987uptr AsanChunkView::AllocTid() const {
988 u32 tid = 0;
989 u32 stack = 0;
990 chunk_->GetAllocContext(tid, stack);
991 return tid;
992}
993
994uptr AsanChunkView::FreeTid() const {
995 if (!IsQuarantined())
996 return kInvalidTid;
997 u32 tid = 0;
998 u32 stack = 0;
999 chunk_->GetFreeContext(tid, stack);
1000 return tid;
1001}
1002
1003AllocType AsanChunkView::GetAllocType() const {
1004 return (AllocType)chunk_->alloc_type;
1005}
1006
1007u32 AsanChunkView::GetAllocStackId() const {
1008 u32 tid = 0;
1009 u32 stack = 0;
1010 chunk_->GetAllocContext(tid, stack);
1011 return stack;
1012}
1013
1014u32 AsanChunkView::GetFreeStackId() const {
1015 if (!IsQuarantined())
1016 return 0;
1017 u32 tid = 0;
1018 u32 stack = 0;
1019 chunk_->GetFreeContext(tid, stack);
1020 return stack;
1021}
1022
1023void InitializeAllocator(const AllocatorOptions &options) {
1024 instance.InitLinkerInitialized(options);
1025}
1026
1027void ReInitializeAllocator(const AllocatorOptions &options) {
1028 instance.ReInitialize(options);
1029}
1030
1031// Apply provided AllocatorOptions to an Allocator
1032void ApplyAllocatorOptions(const AllocatorOptions &options) {
1033 instance.ApplyOptions(options);
1034}
1035
1036void GetAllocatorOptions(AllocatorOptions *options) {
1037 instance.GetOptions(options);
1038}
1039
1040AsanChunkView FindHeapChunkByAddress(uptr addr) {
1041 return instance.FindHeapChunkByAddress(addr);
1042}
1043AsanChunkView FindHeapChunkByAllocBeg(uptr addr) {
1044 return AsanChunkView(instance.GetAsanChunk(alloc_beg: reinterpret_cast<void*>(addr)));
1045}
1046
1047void AsanThreadLocalMallocStorage::CommitBack() {
1048 GET_STACK_TRACE_MALLOC;
1049 instance.CommitBack(ms: this, stack: &stack);
1050}
1051
1052void PrintInternalAllocatorStats() {
1053 instance.PrintStats();
1054}
1055
1056void asan_free(void *ptr, BufferedStackTrace *stack) {
1057 instance.Deallocate(ptr, delete_size: 0, delete_alignment: 0, stack, alloc_type: FROM_MALLOC);
1058}
1059
1060void asan_free_sized(void* ptr, uptr size, BufferedStackTrace* stack) {
1061 instance.Deallocate(ptr, delete_size: size, /*delete_alignment=*/0, stack, alloc_type: FROM_MALLOC);
1062}
1063
1064void asan_free_aligned_sized(void* ptr, uptr alignment, uptr size,
1065 BufferedStackTrace* stack) {
1066 instance.Deallocate(ptr, delete_size: size, delete_alignment: alignment, stack, alloc_type: FROM_MALLOC);
1067}
1068
1069void *asan_malloc(uptr size, BufferedStackTrace *stack) {
1070 return SetErrnoOnNull(instance.Allocate(size, alignment: 8, stack, alloc_type: FROM_MALLOC, can_fill: true));
1071}
1072
1073void *asan_calloc(uptr nmemb, uptr size, BufferedStackTrace *stack) {
1074 return SetErrnoOnNull(instance.Calloc(nmemb, size, stack));
1075}
1076
1077#if SANITIZER_AIX
1078void* asan_vec_malloc(uptr size, BufferedStackTrace* stack) {
1079 return SetErrnoOnNull(instance.Allocate(size, 16, stack, FROM_MALLOC, true));
1080}
1081
1082void* asan_vec_calloc(uptr nmemb, uptr size, BufferedStackTrace* stack) {
1083 return SetErrnoOnNull(instance.Calloc(nmemb, size, stack, 16));
1084}
1085#endif
1086
1087void *asan_reallocarray(void *p, uptr nmemb, uptr size,
1088 BufferedStackTrace *stack) {
1089 if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
1090 errno = errno_ENOMEM;
1091 if (AllocatorMayReturnNull())
1092 return nullptr;
1093 ReportReallocArrayOverflow(count: nmemb, size, stack);
1094 }
1095 return asan_realloc(p, size: nmemb * size, stack);
1096}
1097
1098void *asan_realloc(void *p, uptr size, BufferedStackTrace *stack) {
1099 if (!p)
1100 return SetErrnoOnNull(instance.Allocate(size, alignment: 8, stack, alloc_type: FROM_MALLOC, can_fill: true));
1101 if (size == 0) {
1102 if (flags()->allocator_frees_and_returns_null_on_realloc_zero) {
1103 instance.Deallocate(ptr: p, delete_size: 0, delete_alignment: 0, stack, alloc_type: FROM_MALLOC);
1104 return nullptr;
1105 }
1106 // Allocate a size of 1 if we shouldn't free() on Realloc to 0
1107 size = 1;
1108 }
1109 return SetErrnoOnNull(instance.Reallocate(old_ptr: p, new_size: size, stack));
1110}
1111
1112void *asan_valloc(uptr size, BufferedStackTrace *stack) {
1113 return SetErrnoOnNull(
1114 instance.Allocate(size, alignment: GetPageSizeCached(), stack, alloc_type: FROM_MALLOC, can_fill: true));
1115}
1116
1117void *asan_pvalloc(uptr size, BufferedStackTrace *stack) {
1118 uptr PageSize = GetPageSizeCached();
1119 if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) {
1120 errno = errno_ENOMEM;
1121 if (AllocatorMayReturnNull())
1122 return nullptr;
1123 ReportPvallocOverflow(size, stack);
1124 }
1125 // pvalloc(0) should allocate one page.
1126 size = size ? RoundUpTo(size, boundary: PageSize) : PageSize;
1127 return SetErrnoOnNull(
1128 instance.Allocate(size, alignment: PageSize, stack, alloc_type: FROM_MALLOC, can_fill: true));
1129}
1130
1131void *asan_memalign(uptr alignment, uptr size, BufferedStackTrace *stack) {
1132 if (UNLIKELY(!IsPowerOfTwo(alignment))) {
1133 errno = errno_EINVAL;
1134 if (AllocatorMayReturnNull())
1135 return nullptr;
1136 ReportInvalidAllocationAlignment(alignment, stack);
1137 }
1138 return SetErrnoOnNull(
1139 instance.Allocate(size, alignment, stack, alloc_type: FROM_MALLOC, can_fill: true));
1140}
1141
1142void *asan_aligned_alloc(uptr alignment, uptr size, BufferedStackTrace *stack) {
1143 if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) {
1144 errno = errno_EINVAL;
1145 if (AllocatorMayReturnNull())
1146 return nullptr;
1147 ReportInvalidAlignedAllocAlignment(size, alignment, stack);
1148 }
1149 return SetErrnoOnNull(
1150 instance.Allocate(size, alignment, stack, alloc_type: FROM_MALLOC, can_fill: true));
1151}
1152
1153int asan_posix_memalign(void **memptr, uptr alignment, uptr size,
1154 BufferedStackTrace *stack) {
1155 if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) {
1156 if (AllocatorMayReturnNull())
1157 return errno_EINVAL;
1158 ReportInvalidPosixMemalignAlignment(alignment, stack);
1159 }
1160 void *ptr = instance.Allocate(size, alignment, stack, alloc_type: FROM_MALLOC, can_fill: true);
1161 if (UNLIKELY(!ptr))
1162 // OOM error is already taken care of by Allocate.
1163 return errno_ENOMEM;
1164 CHECK(IsAligned((uptr)ptr, alignment));
1165 *memptr = ptr;
1166 return 0;
1167}
1168
1169uptr asan_malloc_usable_size(const void *ptr, uptr pc, uptr bp) {
1170 if (!ptr) return 0;
1171 uptr usable_size = instance.AllocationSize(p: reinterpret_cast<uptr>(ptr));
1172 if (flags()->check_malloc_usable_size && (usable_size == 0)) {
1173 GET_STACK_TRACE_FATAL(pc, bp);
1174 ReportMallocUsableSizeNotOwned(addr: (uptr)ptr, stack: &stack);
1175 }
1176#if SANITIZER_WINDOWS
1177 // Zero-size allocations are internally upgraded to size 1 so that
1178 // malloc(0)/new(0) return unique non-NULL pointers as required by the
1179 // standard. Windows heap APIs (HeapSize, RtlSizeHeap, _msize) should still
1180 // report the originally requested size (0).
1181 if (usable_size > 0 &&
1182 instance.FromZeroAllocation(reinterpret_cast<uptr>(ptr))) {
1183 DCHECK(usable_size == 1);
1184 return 0;
1185 }
1186#endif
1187 return usable_size;
1188}
1189
1190namespace {
1191
1192void *asan_new(uptr size, BufferedStackTrace *stack, bool array) {
1193 return SetErrnoOnNull(
1194 instance.Allocate(size, alignment: 0, stack, alloc_type: array ? FROM_NEW_BR : FROM_NEW, can_fill: true));
1195}
1196
1197void *asan_new_aligned(uptr size, uptr alignment, BufferedStackTrace *stack,
1198 bool array) {
1199 if (UNLIKELY(alignment == 0 || !IsPowerOfTwo(alignment))) {
1200 errno = errno_EINVAL;
1201 if (AllocatorMayReturnNull())
1202 return nullptr;
1203 ReportInvalidAllocationAlignment(alignment, stack);
1204 }
1205 return SetErrnoOnNull(instance.Allocate(
1206 size, alignment, stack, alloc_type: array ? FROM_NEW_BR : FROM_NEW, can_fill: true));
1207}
1208
1209void asan_delete(void *ptr, BufferedStackTrace *stack, bool array) {
1210 instance.Deallocate(ptr, delete_size: 0, delete_alignment: 0, stack, alloc_type: array ? FROM_NEW_BR : FROM_NEW);
1211}
1212
1213void asan_delete_aligned(void *ptr, uptr alignment, BufferedStackTrace *stack,
1214 bool array) {
1215 instance.Deallocate(ptr, delete_size: 0, delete_alignment: alignment, stack, alloc_type: array ? FROM_NEW_BR : FROM_NEW);
1216}
1217
1218void asan_delete_sized(void *ptr, uptr size, BufferedStackTrace *stack,
1219 bool array) {
1220 instance.Deallocate(ptr, delete_size: size, delete_alignment: 0, stack, alloc_type: array ? FROM_NEW_BR : FROM_NEW);
1221}
1222
1223void asan_delete_sized_aligned(void *ptr, uptr size, uptr alignment,
1224 BufferedStackTrace *stack, bool array) {
1225 instance.Deallocate(ptr, delete_size: size, delete_alignment: alignment, stack,
1226 alloc_type: array ? FROM_NEW_BR : FROM_NEW);
1227}
1228
1229} // namespace
1230
1231void *asan_new(uptr size, BufferedStackTrace *stack) {
1232 return asan_new(size, stack, /*array=*/false);
1233}
1234
1235void *asan_new_aligned(uptr size, uptr alignment, BufferedStackTrace *stack) {
1236 return asan_new_aligned(size, alignment, stack, /*array=*/false);
1237}
1238
1239void *asan_new_array(uptr size, BufferedStackTrace *stack) {
1240 return asan_new(size, stack, /*array=*/true);
1241}
1242
1243void *asan_new_array_aligned(uptr size, uptr alignment,
1244 BufferedStackTrace *stack) {
1245 return asan_new_aligned(size, alignment, stack, /*array=*/true);
1246}
1247
1248void asan_delete(void *ptr, BufferedStackTrace *stack) {
1249 asan_delete(ptr, stack, /*array=*/false);
1250}
1251
1252void asan_delete_aligned(void *ptr, uptr alignment, BufferedStackTrace *stack) {
1253 asan_delete_aligned(ptr, alignment, stack, /*array=*/false);
1254}
1255
1256void asan_delete_sized(void *ptr, uptr size, BufferedStackTrace *stack) {
1257 asan_delete_sized(ptr, size, stack, /*array=*/false);
1258}
1259
1260void asan_delete_sized_aligned(void *ptr, uptr size, uptr alignment,
1261 BufferedStackTrace *stack) {
1262 asan_delete_sized_aligned(ptr, size, alignment, stack, /*array=*/false);
1263}
1264
1265void asan_delete_array(void *ptr, BufferedStackTrace *stack) {
1266 asan_delete(ptr, stack, /*array=*/true);
1267}
1268
1269void asan_delete_array_aligned(void *ptr, uptr alignment,
1270 BufferedStackTrace *stack) {
1271 asan_delete_aligned(ptr, alignment, stack, /*array=*/true);
1272}
1273
1274void asan_delete_array_sized(void *ptr, uptr size, BufferedStackTrace *stack) {
1275 asan_delete_sized(ptr, size, stack, /*array=*/true);
1276}
1277
1278void asan_delete_array_sized_aligned(void *ptr, uptr size, uptr alignment,
1279 BufferedStackTrace *stack) {
1280 asan_delete_sized_aligned(ptr, size, alignment, stack, /*array=*/true);
1281}
1282
1283uptr asan_mz_size(const void* ptr) {
1284 uptr size = instance.AllocationSize(p: reinterpret_cast<uptr>(ptr));
1285
1286#if SANITIZER_WINDOWS
1287 if (size > 0 && instance.FromZeroAllocation(reinterpret_cast<uptr>(ptr))) {
1288 DCHECK(size == 1);
1289 return 0;
1290 }
1291#endif
1292
1293 return size;
1294}
1295
1296#if SANITIZER_WINDOWS
1297void asan_mark_zero_allocation(void* ptr) {
1298 instance.MarkAsZeroAllocation(reinterpret_cast<uptr>(ptr));
1299}
1300#endif
1301
1302void asan_mz_force_lock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
1303 instance.ForceLock();
1304}
1305
1306void asan_mz_force_unlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
1307 instance.ForceUnlock();
1308}
1309
1310} // namespace __asan
1311
1312// --- Implementation of LSan-specific functions --- {{{1
1313namespace __lsan {
1314void LockAllocator() {
1315 __asan::get_allocator().ForceLock();
1316}
1317
1318void UnlockAllocator() {
1319 __asan::get_allocator().ForceUnlock();
1320}
1321
1322void GetAllocatorGlobalRange(uptr *begin, uptr *end) {
1323 *begin = (uptr)&__asan::get_allocator();
1324 *end = *begin + sizeof(__asan::get_allocator());
1325}
1326
1327uptr PointsIntoChunk(void *p) {
1328 uptr addr = reinterpret_cast<uptr>(p);
1329 __asan::AsanChunk *m = __asan::instance.GetAsanChunkByAddrFastLocked(p: addr);
1330 if (!m || atomic_load(a: &m->chunk_state, mo: memory_order_acquire) !=
1331 __asan::CHUNK_ALLOCATED)
1332 return 0;
1333 uptr chunk = m->Beg();
1334 if (m->AddrIsInside(addr))
1335 return chunk;
1336 if (IsSpecialCaseOfOperatorNew0(chunk_beg: chunk, chunk_size: m->UsedSize(), addr))
1337 return chunk;
1338 return 0;
1339}
1340
1341uptr GetUserBegin(uptr chunk) {
1342 // FIXME: All usecases provide chunk address, GetAsanChunkByAddrFastLocked is
1343 // not needed.
1344 __asan::AsanChunk *m = __asan::instance.GetAsanChunkByAddrFastLocked(p: chunk);
1345 return m ? m->Beg() : 0;
1346}
1347
1348uptr GetUserAddr(uptr chunk) {
1349 return chunk;
1350}
1351
1352LsanMetadata::LsanMetadata(uptr chunk) {
1353 metadata_ = chunk ? reinterpret_cast<void *>(chunk - __asan::kChunkHeaderSize)
1354 : nullptr;
1355}
1356
1357bool LsanMetadata::allocated() const {
1358 if (!metadata_)
1359 return false;
1360 __asan::AsanChunk *m = reinterpret_cast<__asan::AsanChunk *>(metadata_);
1361 return atomic_load(a: &m->chunk_state, mo: memory_order_relaxed) ==
1362 __asan::CHUNK_ALLOCATED;
1363}
1364
1365ChunkTag LsanMetadata::tag() const {
1366 __asan::AsanChunk *m = reinterpret_cast<__asan::AsanChunk *>(metadata_);
1367 return static_cast<ChunkTag>(m->lsan_tag);
1368}
1369
1370void LsanMetadata::set_tag(ChunkTag value) {
1371 __asan::AsanChunk *m = reinterpret_cast<__asan::AsanChunk *>(metadata_);
1372 m->lsan_tag = value;
1373}
1374
1375uptr LsanMetadata::requested_size() const {
1376 __asan::AsanChunk *m = reinterpret_cast<__asan::AsanChunk *>(metadata_);
1377 return m->UsedSize();
1378}
1379
1380u32 LsanMetadata::stack_trace_id() const {
1381 __asan::AsanChunk *m = reinterpret_cast<__asan::AsanChunk *>(metadata_);
1382 u32 tid = 0;
1383 u32 stack = 0;
1384 m->GetAllocContext(tid, stack);
1385 return stack;
1386}
1387
1388void ForEachChunk(ForEachChunkCallback callback, void *arg) {
1389 __asan::get_allocator().ForEachChunk(callback, arg);
1390}
1391
1392IgnoreObjectResult IgnoreObject(const void *p) {
1393 uptr addr = reinterpret_cast<uptr>(p);
1394 __asan::AsanChunk *m = __asan::instance.GetAsanChunkByAddr(p: addr);
1395 if (!m ||
1396 (atomic_load(a: &m->chunk_state, mo: memory_order_acquire) !=
1397 __asan::CHUNK_ALLOCATED) ||
1398 !m->AddrIsInside(addr)) {
1399 return kIgnoreObjectInvalid;
1400 }
1401 if (m->lsan_tag == kIgnored)
1402 return kIgnoreObjectAlreadyIgnored;
1403 m->lsan_tag = __lsan::kIgnored;
1404 return kIgnoreObjectSuccess;
1405}
1406
1407} // namespace __lsan
1408
1409// ---------------------- Interface ---------------- {{{1
1410using namespace __asan;
1411
1412static const void *AllocationBegin(const void *p) {
1413 AsanChunk *m = __asan::instance.GetAsanChunkByAddr(p: (uptr)p);
1414 if (!m)
1415 return nullptr;
1416 if (atomic_load(a: &m->chunk_state, mo: memory_order_acquire) != CHUNK_ALLOCATED)
1417 return nullptr;
1418 if (m->UsedSize() == 0)
1419 return nullptr;
1420 return (const void *)(m->Beg());
1421}
1422
1423// ASan allocator doesn't reserve extra bytes, so normally we would
1424// just return "size". We don't want to expose our redzone sizes, etc here.
1425uptr __sanitizer_get_estimated_allocated_size(uptr size) {
1426 return size;
1427}
1428
1429int __sanitizer_get_ownership(const void *p) {
1430 uptr ptr = reinterpret_cast<uptr>(p);
1431 return instance.AllocationSize(p: ptr) > 0;
1432}
1433
1434uptr __sanitizer_get_allocated_size(const void *p) {
1435 if (!p) return 0;
1436 uptr ptr = reinterpret_cast<uptr>(p);
1437 uptr allocated_size = instance.AllocationSize(p: ptr);
1438 // Die if p is not malloced or if it is already freed.
1439 if (allocated_size == 0) {
1440 GET_STACK_TRACE_FATAL_HERE;
1441 ReportSanitizerGetAllocatedSizeNotOwned(addr: ptr, stack: &stack);
1442 }
1443 return allocated_size;
1444}
1445
1446uptr __sanitizer_get_allocated_size_fast(const void *p) {
1447 DCHECK_EQ(p, __sanitizer_get_allocated_begin(p));
1448 uptr ret = instance.AllocationSizeFast(p: reinterpret_cast<uptr>(p));
1449 DCHECK_EQ(ret, __sanitizer_get_allocated_size(p));
1450 return ret;
1451}
1452
1453const void *__sanitizer_get_allocated_begin(const void *p) {
1454 return AllocationBegin(p);
1455}
1456
1457void __sanitizer_purge_allocator() {
1458 GET_STACK_TRACE_MALLOC;
1459 instance.Purge(stack: &stack);
1460}
1461
1462int __asan_update_allocation_context(void* addr) {
1463 GET_STACK_TRACE_MALLOC;
1464 return instance.UpdateAllocationStack(addr: (uptr)addr, stack: &stack);
1465}
1466