1//===-- asan_allocator.cpp ------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is a part of AddressSanitizer, an address sanity checker.
10//
11// Implementation of ASan's memory allocator, 2-nd version.
12// This variant uses the allocator from sanitizer_common, i.e. the one shared
13// with ThreadSanitizer and MemorySanitizer.
14//
15//===----------------------------------------------------------------------===//
16
17#include "asan_allocator.h"
18
19#include "asan_internal.h"
20#include "asan_mapping.h"
21#include "asan_poisoning.h"
22#include "asan_report.h"
23#include "asan_stack.h"
24#include "asan_suppressions.h"
25#include "asan_thread.h"
26#include "lsan/lsan_common.h"
27#include "sanitizer_common/sanitizer_allocator_checks.h"
28#include "sanitizer_common/sanitizer_allocator_interface.h"
29#include "sanitizer_common/sanitizer_common.h"
30#include "sanitizer_common/sanitizer_errno.h"
31#include "sanitizer_common/sanitizer_flags.h"
32#include "sanitizer_common/sanitizer_internal_defs.h"
33#include "sanitizer_common/sanitizer_list.h"
34#include "sanitizer_common/sanitizer_quarantine.h"
35#include "sanitizer_common/sanitizer_stackdepot.h"
36
37namespace __asan {
38
39// Valid redzone sizes are 16, 32, 64, ... 2048, so we encode them in 3 bits.
40// We use adaptive redzones: for larger allocation larger redzones are used.
41static u32 RZLog2Size(u32 rz_log) {
42 CHECK_LT(rz_log, 8);
43 return 16 << rz_log;
44}
45
46static u32 RZSize2Log(u32 rz_size) {
47 CHECK_GE(rz_size, 16);
48 CHECK_LE(rz_size, 2048);
49 CHECK(IsPowerOfTwo(rz_size));
50 u32 res = Log2(x: rz_size) - 4;
51 CHECK_EQ(rz_size, RZLog2Size(res));
52 return res;
53}
54
55static AsanAllocator &get_allocator();
56
57static void AtomicContextStore(volatile atomic_uint64_t *atomic_context,
58 u32 tid, u32 stack) {
59 u64 context = tid;
60 context <<= 32;
61 context += stack;
62 atomic_store(a: atomic_context, v: context, mo: memory_order_relaxed);
63}
64
65static void AtomicContextLoad(const volatile atomic_uint64_t *atomic_context,
66 u32 &tid, u32 &stack) {
67 u64 context = atomic_load(a: atomic_context, mo: memory_order_relaxed);
68 stack = context;
69 context >>= 32;
70 tid = context;
71}
72
73// The memory chunk allocated from the underlying allocator looks like this:
74// L L L L L L H H U U U U U U R R
75// L -- left redzone words (0 or more bytes)
76// H -- ChunkHeader (16 bytes), which is also a part of the left redzone.
77// U -- user memory.
78// R -- right redzone (0 or more bytes)
79// ChunkBase consists of ChunkHeader and other bytes that overlap with user
80// memory.
81
82// If the left redzone is greater than the ChunkHeader size we store a magic
83// value in the first uptr word of the memory block and store the address of
84// ChunkBase in the next uptr.
85// M B L L L L L L L L L H H U U U U U U
86// | ^
87// ---------------------|
88// M -- magic value kAllocBegMagic
89// B -- address of ChunkHeader pointing to the first 'H'
90
91class ChunkHeader {
92 public:
93 atomic_uint8_t chunk_state;
94 u8 alloc_type : 2;
95 u8 lsan_tag : 2;
96
97 // align < 8 -> 0
98 // else -> log2(min(align, 512)) - 2
99 u8 user_requested_alignment_log : 3;
100
101 private:
102 u16 user_requested_size_hi;
103 u32 user_requested_size_lo;
104 atomic_uint64_t alloc_context_id;
105
106 public:
107 uptr UsedSize() const {
108 static_assert(sizeof(user_requested_size_lo) == 4,
109 "Expression below requires this");
110 return FIRST_32_SECOND_64(0, ((uptr)user_requested_size_hi << 32)) +
111 user_requested_size_lo;
112 }
113
114 void SetUsedSize(uptr size) {
115 user_requested_size_lo = size;
116 static_assert(sizeof(user_requested_size_lo) == 4,
117 "Expression below requires this");
118 user_requested_size_hi = FIRST_32_SECOND_64(0, size >> 32);
119 CHECK_EQ(UsedSize(), size);
120 }
121
122 void SetAllocContext(u32 tid, u32 stack) {
123 AtomicContextStore(atomic_context: &alloc_context_id, tid, stack);
124 }
125
126 void GetAllocContext(u32 &tid, u32 &stack) const {
127 AtomicContextLoad(atomic_context: &alloc_context_id, tid, stack);
128 }
129};
130
131class ChunkBase : public ChunkHeader {
132 atomic_uint64_t free_context_id;
133
134 public:
135 void SetFreeContext(u32 tid, u32 stack) {
136 AtomicContextStore(atomic_context: &free_context_id, tid, stack);
137 }
138
139 void GetFreeContext(u32 &tid, u32 &stack) const {
140 AtomicContextLoad(atomic_context: &free_context_id, tid, stack);
141 }
142};
143
144static const uptr kChunkHeaderSize = sizeof(ChunkHeader);
145static const uptr kChunkHeader2Size = sizeof(ChunkBase) - kChunkHeaderSize;
146COMPILER_CHECK(kChunkHeaderSize == 16);
147COMPILER_CHECK(kChunkHeader2Size <= 16);
148
149enum {
150 // Either just allocated by underlying allocator, but AsanChunk is not yet
151 // ready, or almost returned to undelying allocator and AsanChunk is already
152 // meaningless.
153 CHUNK_INVALID = 0,
154 // The chunk is allocated and not yet freed.
155 CHUNK_ALLOCATED = 2,
156 // The chunk was freed and put into quarantine zone.
157 CHUNK_QUARANTINE = 3,
158};
159
160class AsanChunk : public ChunkBase {
161 public:
162 uptr Beg() { return reinterpret_cast<uptr>(this) + kChunkHeaderSize; }
163 bool AddrIsInside(uptr addr) {
164 return (addr >= Beg()) && (addr < Beg() + UsedSize());
165 }
166};
167
168class LargeChunkHeader {
169 static constexpr uptr kAllocBegMagic =
170 FIRST_32_SECOND_64(0xCC6E96B9, 0xCC6E96B9CC6E96B9ULL);
171 atomic_uintptr_t magic;
172 AsanChunk *chunk_header;
173
174 public:
175 AsanChunk *Get() const {
176 return atomic_load(a: &magic, mo: memory_order_acquire) == kAllocBegMagic
177 ? chunk_header
178 : nullptr;
179 }
180
181 void Set(AsanChunk *p) {
182 if (p) {
183 chunk_header = p;
184 atomic_store(a: &magic, v: kAllocBegMagic, mo: memory_order_release);
185 return;
186 }
187
188 uptr old = kAllocBegMagic;
189 if (!atomic_compare_exchange_strong(a: &magic, cmp: &old, xchg: 0,
190 mo: memory_order_release)) {
191 CHECK_EQ(old, kAllocBegMagic);
192 }
193 }
194};
195
196static void FillChunk(AsanChunk *m) {
197 // FIXME: Use ReleaseMemoryPagesToOS.
198 Flags &fl = *flags();
199
200 if (fl.max_free_fill_size > 0) {
201 // We have to skip the chunk header, it contains free_context_id.
202 uptr scribble_start = (uptr)m + kChunkHeaderSize + kChunkHeader2Size;
203 if (m->UsedSize() >= kChunkHeader2Size) { // Skip Header2 in user area.
204 uptr size_to_fill = m->UsedSize() - kChunkHeader2Size;
205 size_to_fill = Min(a: size_to_fill, b: (uptr)fl.max_free_fill_size);
206 REAL(memset)((void *)scribble_start, fl.free_fill_byte, size_to_fill);
207 }
208 }
209}
210
211struct QuarantineCallback {
212 QuarantineCallback(AllocatorCache *cache, BufferedStackTrace *stack)
213 : cache_(cache),
214 stack_(stack) {
215 }
216
217 void PreQuarantine(AsanChunk *m) const {
218 FillChunk(m);
219 // Poison the region.
220 PoisonShadow(addr: m->Beg(), size: RoundUpTo(size: m->UsedSize(), ASAN_SHADOW_GRANULARITY),
221 value: kAsanHeapFreeMagic);
222 }
223
224 void Recycle(AsanChunk *m) const {
225 void *p = get_allocator().GetBlockBegin(p: m);
226
227 // The secondary will immediately unpoison and unmap the memory, so this
228 // branch is unnecessary.
229 if (get_allocator().FromPrimary(p)) {
230 if (p != m) {
231 // Clear the magic value, as allocator internals may overwrite the
232 // contents of deallocated chunk, confusing GetAsanChunk lookup.
233 reinterpret_cast<LargeChunkHeader *>(p)->Set(nullptr);
234 }
235
236 u8 old_chunk_state = CHUNK_QUARANTINE;
237 if (!atomic_compare_exchange_strong(a: &m->chunk_state, cmp: &old_chunk_state,
238 xchg: CHUNK_INVALID,
239 mo: memory_order_acquire)) {
240 CHECK_EQ(old_chunk_state, CHUNK_QUARANTINE);
241 }
242
243 PoisonShadow(addr: m->Beg(), size: RoundUpTo(size: m->UsedSize(), ASAN_SHADOW_GRANULARITY),
244 value: kAsanHeapLeftRedzoneMagic);
245 }
246
247 // Statistics.
248 AsanStats &thread_stats = GetCurrentThreadStats();
249 thread_stats.real_frees++;
250 thread_stats.really_freed += m->UsedSize();
251
252 get_allocator().Deallocate(cache: cache_, p);
253 }
254
255 void RecyclePassThrough(AsanChunk *m) const {
256 // Recycle for the secondary will immediately unpoison and unmap the
257 // memory, so quarantine preparation is unnecessary.
258 if (get_allocator().FromPrimary(p: m)) {
259 // The primary allocation may need pattern fill if enabled.
260 FillChunk(m);
261 }
262 Recycle(m);
263 }
264
265 void *Allocate(uptr size) const {
266 void *res = get_allocator().Allocate(cache: cache_, size, alignment: 1);
267 // TODO(alekseys): Consider making quarantine OOM-friendly.
268 if (UNLIKELY(!res))
269 ReportOutOfMemory(requested_size: size, stack: stack_);
270 return res;
271 }
272
273 void Deallocate(void *p) const { get_allocator().Deallocate(cache: cache_, p); }
274
275 private:
276 AllocatorCache* const cache_;
277 BufferedStackTrace* const stack_;
278};
279
280typedef Quarantine<QuarantineCallback, AsanChunk> AsanQuarantine;
281typedef AsanQuarantine::Cache QuarantineCache;
282
283void AsanMapUnmapCallback::OnMap(uptr p, uptr size) const {
284 PoisonShadow(addr: p, size, value: kAsanHeapLeftRedzoneMagic);
285 // Statistics.
286 AsanStats &thread_stats = GetCurrentThreadStats();
287 thread_stats.mmaps++;
288 thread_stats.mmaped += size;
289}
290
291void AsanMapUnmapCallback::OnMapSecondary(uptr p, uptr size, uptr user_begin,
292 uptr user_size) const {
293 uptr user_end = RoundDownTo(x: user_begin + user_size, ASAN_SHADOW_GRANULARITY);
294 user_begin = RoundUpTo(size: user_begin, ASAN_SHADOW_GRANULARITY);
295 // The secondary mapping will be immediately returned to user, no value
296 // poisoning that with non-zero just before unpoisoning by Allocate(). So just
297 // poison head/tail invisible to Allocate().
298 PoisonShadow(addr: p, size: user_begin - p, value: kAsanHeapLeftRedzoneMagic);
299 PoisonShadow(addr: user_end, size: size - (user_end - p), value: kAsanHeapLeftRedzoneMagic);
300 // Statistics.
301 AsanStats &thread_stats = GetCurrentThreadStats();
302 thread_stats.mmaps++;
303 thread_stats.mmaped += size;
304}
305
306void AsanMapUnmapCallback::OnUnmap(uptr p, uptr size) const {
307 PoisonShadow(addr: p, size, value: 0);
308 // We are about to unmap a chunk of user memory.
309 // Mark the corresponding shadow memory as not needed.
310 FlushUnneededASanShadowMemory(p, size);
311 // Statistics.
312 AsanStats &thread_stats = GetCurrentThreadStats();
313 thread_stats.munmaps++;
314 thread_stats.munmaped += size;
315}
316
317// We can not use THREADLOCAL because it is not supported on some of the
318// platforms we care about (OSX 10.6, Android).
319// static THREADLOCAL AllocatorCache cache;
320AllocatorCache *GetAllocatorCache(AsanThreadLocalMallocStorage *ms) {
321 CHECK(ms);
322 return &ms->allocator_cache;
323}
324
325QuarantineCache *GetQuarantineCache(AsanThreadLocalMallocStorage *ms) {
326 CHECK(ms);
327 CHECK_LE(sizeof(QuarantineCache), sizeof(ms->quarantine_cache));
328 return reinterpret_cast<QuarantineCache *>(ms->quarantine_cache);
329}
330
331void AllocatorOptions::SetFrom(const Flags *f, const CommonFlags *cf) {
332 quarantine_size_mb = f->quarantine_size_mb;
333 thread_local_quarantine_size_kb = f->thread_local_quarantine_size_kb;
334 min_redzone = f->redzone;
335 max_redzone = f->max_redzone;
336 may_return_null = cf->allocator_may_return_null;
337 alloc_dealloc_mismatch = f->alloc_dealloc_mismatch;
338 release_to_os_interval_ms = cf->allocator_release_to_os_interval_ms;
339}
340
341void AllocatorOptions::CopyTo(Flags *f, CommonFlags *cf) {
342 f->quarantine_size_mb = quarantine_size_mb;
343 f->thread_local_quarantine_size_kb = thread_local_quarantine_size_kb;
344 f->redzone = min_redzone;
345 f->max_redzone = max_redzone;
346 cf->allocator_may_return_null = may_return_null;
347 f->alloc_dealloc_mismatch = alloc_dealloc_mismatch;
348 cf->allocator_release_to_os_interval_ms = release_to_os_interval_ms;
349}
350
351struct Allocator {
352 static const uptr kMaxAllowedMallocSize =
353 FIRST_32_SECOND_64(3UL << 30, 1ULL << 40);
354
355 AsanAllocator allocator;
356 AsanQuarantine quarantine;
357 StaticSpinMutex fallback_mutex;
358 AllocatorCache fallback_allocator_cache;
359 QuarantineCache fallback_quarantine_cache;
360
361 uptr max_user_defined_malloc_size;
362
363 // ------------------- Options --------------------------
364 atomic_uint16_t min_redzone;
365 atomic_uint16_t max_redzone;
366 atomic_uint8_t alloc_dealloc_mismatch;
367
368 // ------------------- Initialization ------------------------
369 explicit Allocator(LinkerInitialized)
370 : quarantine(LINKER_INITIALIZED),
371 fallback_quarantine_cache(LINKER_INITIALIZED) {}
372
373 void CheckOptions(const AllocatorOptions &options) const {
374 CHECK_GE(options.min_redzone, 16);
375 CHECK_GE(options.max_redzone, options.min_redzone);
376 CHECK_LE(options.max_redzone, 2048);
377 CHECK(IsPowerOfTwo(options.min_redzone));
378 CHECK(IsPowerOfTwo(options.max_redzone));
379 }
380
381 void SharedInitCode(const AllocatorOptions &options) {
382 CheckOptions(options);
383 quarantine.Init(size: (uptr)options.quarantine_size_mb << 20,
384 cache_size: (uptr)options.thread_local_quarantine_size_kb << 10);
385 atomic_store(a: &alloc_dealloc_mismatch, v: options.alloc_dealloc_mismatch,
386 mo: memory_order_release);
387 atomic_store(a: &min_redzone, v: options.min_redzone, mo: memory_order_release);
388 atomic_store(a: &max_redzone, v: options.max_redzone, mo: memory_order_release);
389 }
390
391 void InitLinkerInitialized(const AllocatorOptions &options) {
392 SetAllocatorMayReturnNull(options.may_return_null);
393 allocator.InitLinkerInitialized(release_to_os_interval_ms: options.release_to_os_interval_ms);
394 SharedInitCode(options);
395 max_user_defined_malloc_size = common_flags()->max_allocation_size_mb
396 ? common_flags()->max_allocation_size_mb
397 << 20
398 : kMaxAllowedMallocSize;
399 }
400
401 void RePoisonChunk(uptr chunk) {
402 // This could be a user-facing chunk (with redzones), or some internal
403 // housekeeping chunk, like TransferBatch. Start by assuming the former.
404 AsanChunk *ac = GetAsanChunk(alloc_beg: (void *)chunk);
405 uptr allocated_size = allocator.GetActuallyAllocatedSize(p: (void *)chunk);
406 if (ac && atomic_load(a: &ac->chunk_state, mo: memory_order_acquire) ==
407 CHUNK_ALLOCATED) {
408 uptr beg = ac->Beg();
409 uptr end = ac->Beg() + ac->UsedSize();
410 uptr chunk_end = chunk + allocated_size;
411 if (chunk < beg && beg < end && end <= chunk_end) {
412 // Looks like a valid AsanChunk in use, poison redzones only.
413 PoisonShadow(addr: chunk, size: beg - chunk, value: kAsanHeapLeftRedzoneMagic);
414 uptr end_aligned_down = RoundDownTo(x: end, ASAN_SHADOW_GRANULARITY);
415 FastPoisonShadowPartialRightRedzone(
416 aligned_addr: end_aligned_down, size: end - end_aligned_down,
417 redzone_size: chunk_end - end_aligned_down, value: kAsanHeapLeftRedzoneMagic);
418 return;
419 }
420 }
421
422 // This is either not an AsanChunk or freed or quarantined AsanChunk.
423 // In either case, poison everything.
424 PoisonShadow(addr: chunk, size: allocated_size, value: kAsanHeapLeftRedzoneMagic);
425 }
426
427 // Apply provided AllocatorOptions to an Allocator
428 void ApplyOptions(const AllocatorOptions &options) {
429 SetAllocatorMayReturnNull(options.may_return_null);
430 allocator.SetReleaseToOSIntervalMs(options.release_to_os_interval_ms);
431 SharedInitCode(options);
432 }
433
434 void ReInitialize(const AllocatorOptions &options) {
435 ApplyOptions(options);
436
437 // Poison all existing allocation's redzones.
438 if (CanPoisonMemory()) {
439 allocator.ForceLock();
440 allocator.ForEachChunk(
441 callback: [](uptr chunk, void *alloc) {
442 ((Allocator *)alloc)->RePoisonChunk(chunk);
443 },
444 arg: this);
445 allocator.ForceUnlock();
446 }
447 }
448
449 void GetOptions(AllocatorOptions *options) const {
450 options->quarantine_size_mb = quarantine.GetMaxSize() >> 20;
451 options->thread_local_quarantine_size_kb =
452 quarantine.GetMaxCacheSize() >> 10;
453 options->min_redzone = atomic_load(a: &min_redzone, mo: memory_order_acquire);
454 options->max_redzone = atomic_load(a: &max_redzone, mo: memory_order_acquire);
455 options->may_return_null = AllocatorMayReturnNull();
456 options->alloc_dealloc_mismatch =
457 atomic_load(a: &alloc_dealloc_mismatch, mo: memory_order_acquire);
458 options->release_to_os_interval_ms = allocator.ReleaseToOSIntervalMs();
459 }
460
461 // -------------------- Helper methods. -------------------------
462 uptr ComputeRZLog(uptr user_requested_size) {
463 u32 rz_log = user_requested_size <= 64 - 16 ? 0
464 : user_requested_size <= 128 - 32 ? 1
465 : user_requested_size <= 512 - 64 ? 2
466 : user_requested_size <= 4096 - 128 ? 3
467 : user_requested_size <= (1 << 14) - 256 ? 4
468 : user_requested_size <= (1 << 15) - 512 ? 5
469 : user_requested_size <= (1 << 16) - 1024 ? 6
470 : 7;
471 u32 hdr_log = RZSize2Log(rz_size: RoundUpToPowerOfTwo(size: sizeof(ChunkHeader)));
472 u32 min_log = RZSize2Log(rz_size: atomic_load(a: &min_redzone, mo: memory_order_acquire));
473 u32 max_log = RZSize2Log(rz_size: atomic_load(a: &max_redzone, mo: memory_order_acquire));
474 return Min(a: Max(a: rz_log, b: Max(a: min_log, b: hdr_log)), b: Max(a: max_log, b: hdr_log));
475 }
476
477 static uptr ComputeUserRequestedAlignmentLog(uptr user_requested_alignment) {
478 if (user_requested_alignment < 8)
479 return 0;
480 if (user_requested_alignment > 512)
481 user_requested_alignment = 512;
482 return Log2(x: user_requested_alignment) - 2;
483 }
484
485 static uptr ComputeUserAlignment(uptr user_requested_alignment_log) {
486 if (user_requested_alignment_log == 0)
487 return 0;
488 return 1LL << (user_requested_alignment_log + 2);
489 }
490
491 // We have an address between two chunks, and we want to report just one.
492 AsanChunk *ChooseChunk(uptr addr, AsanChunk *left_chunk,
493 AsanChunk *right_chunk) {
494 if (!left_chunk)
495 return right_chunk;
496 if (!right_chunk)
497 return left_chunk;
498 // Prefer an allocated chunk over freed chunk and freed chunk
499 // over available chunk.
500 u8 left_state = atomic_load(a: &left_chunk->chunk_state, mo: memory_order_relaxed);
501 u8 right_state =
502 atomic_load(a: &right_chunk->chunk_state, mo: memory_order_relaxed);
503 if (left_state != right_state) {
504 if (left_state == CHUNK_ALLOCATED)
505 return left_chunk;
506 if (right_state == CHUNK_ALLOCATED)
507 return right_chunk;
508 if (left_state == CHUNK_QUARANTINE)
509 return left_chunk;
510 if (right_state == CHUNK_QUARANTINE)
511 return right_chunk;
512 }
513 // Same chunk_state: choose based on offset.
514 sptr l_offset = 0, r_offset = 0;
515 CHECK(AsanChunkView(left_chunk).AddrIsAtRight(addr, 1, &l_offset));
516 CHECK(AsanChunkView(right_chunk).AddrIsAtLeft(addr, 1, &r_offset));
517 if (l_offset < r_offset)
518 return left_chunk;
519 return right_chunk;
520 }
521
522 bool UpdateAllocationStack(uptr addr, BufferedStackTrace *stack) {
523 AsanChunk *m = GetAsanChunkByAddr(p: addr);
524 if (!m) return false;
525 if (atomic_load(a: &m->chunk_state, mo: memory_order_acquire) != CHUNK_ALLOCATED)
526 return false;
527 if (m->Beg() != addr) return false;
528 AsanThread *t = GetCurrentThread();
529 m->SetAllocContext(tid: t ? t->tid() : kMainTid, stack: StackDepotPut(stack: *stack));
530 return true;
531 }
532
533 // -------------------- Allocation/Deallocation routines ---------------
534 void *Allocate(uptr size, uptr alignment, BufferedStackTrace *stack,
535 AllocType alloc_type, bool can_fill) {
536 if (UNLIKELY(!AsanInited()))
537 AsanInitFromRtl();
538 if (UNLIKELY(IsRssLimitExceeded())) {
539 if (AllocatorMayReturnNull())
540 return nullptr;
541 ReportRssLimitExceeded(stack);
542 }
543 Flags &fl = *flags();
544 CHECK(stack);
545 const uptr min_alignment = ASAN_SHADOW_GRANULARITY;
546 const uptr user_requested_alignment_log =
547 ComputeUserRequestedAlignmentLog(user_requested_alignment: alignment);
548 if (alignment < min_alignment)
549 alignment = min_alignment;
550 bool upgraded_from_zero = false;
551 if (size == 0) {
552 // We'd be happy to avoid allocating memory for zero-size requests, but
553 // some programs/tests depend on this behavior and assume that malloc
554 // would not return NULL even for zero-size allocations. Moreover, it
555 // looks like operator new should never return NULL, and results of
556 // consecutive "new" calls must be different even if the allocated size
557 // is zero.
558 size = 1;
559 upgraded_from_zero = true;
560 }
561 CHECK(IsPowerOfTwo(alignment));
562 uptr rz_log = ComputeRZLog(user_requested_size: size);
563 uptr rz_size = RZLog2Size(rz_log);
564 uptr rounded_size = RoundUpTo(size: Max(a: size, b: kChunkHeader2Size), boundary: alignment);
565 uptr needed_size = rounded_size + rz_size;
566 if (alignment > min_alignment)
567 needed_size += alignment;
568 bool from_primary = PrimaryAllocator::CanAllocate(size: needed_size, alignment);
569 // If we are allocating from the secondary allocator, there will be no
570 // automatic right redzone, so add the right redzone manually.
571 if (!from_primary)
572 needed_size += rz_size;
573 CHECK(IsAligned(needed_size, min_alignment));
574 if (size > kMaxAllowedMallocSize || needed_size > kMaxAllowedMallocSize ||
575 size > max_user_defined_malloc_size) {
576 if (AllocatorMayReturnNull()) {
577 Report(format: "WARNING: AddressSanitizer failed to allocate 0x%zx bytes\n",
578 size);
579 return nullptr;
580 }
581 uptr malloc_limit =
582 Min(a: kMaxAllowedMallocSize, b: max_user_defined_malloc_size);
583 ReportAllocationSizeTooBig(user_size: size, total_size: needed_size, max_size: malloc_limit, stack);
584 }
585
586 AsanThread *t = GetCurrentThread();
587 void *allocated;
588 if (t) {
589 AllocatorCache *cache = GetAllocatorCache(ms: &t->malloc_storage());
590 allocated = allocator.Allocate(cache, size: needed_size, alignment: 8);
591 } else {
592 SpinMutexLock l(&fallback_mutex);
593 AllocatorCache *cache = &fallback_allocator_cache;
594 allocated = allocator.Allocate(cache, size: needed_size, alignment: 8);
595 }
596 if (UNLIKELY(!allocated)) {
597 SetAllocatorOutOfMemory();
598 if (AllocatorMayReturnNull())
599 return nullptr;
600 ReportOutOfMemory(requested_size: size, stack);
601 }
602
603 uptr alloc_beg = reinterpret_cast<uptr>(allocated);
604 uptr alloc_end = alloc_beg + needed_size;
605 uptr user_beg = alloc_beg + rz_size;
606 if (!IsAligned(a: user_beg, alignment))
607 user_beg = RoundUpTo(size: user_beg, boundary: alignment);
608 uptr user_end = user_beg + size;
609 CHECK_LE(user_end, alloc_end);
610 uptr chunk_beg = user_beg - kChunkHeaderSize;
611 AsanChunk *m = reinterpret_cast<AsanChunk *>(chunk_beg);
612 m->alloc_type = alloc_type;
613 CHECK(size);
614 m->SetUsedSize(size);
615 m->user_requested_alignment_log = user_requested_alignment_log;
616
617 m->SetAllocContext(tid: t ? t->tid() : kMainTid, stack: StackDepotPut(stack: *stack));
618
619 if (!from_primary || *(u8 *)MEM_TO_SHADOW((uptr)allocated) == 0) {
620 // The allocator provides an unpoisoned chunk. This is possible for the
621 // secondary allocator, or if CanPoisonMemory() was false for some time,
622 // for example, due to flags()->start_disabled. Anyway, poison left and
623 // right of the block before using it for anything else.
624 uptr tail_beg = RoundUpTo(size: user_end, ASAN_SHADOW_GRANULARITY);
625 uptr tail_end = alloc_beg + allocator.GetActuallyAllocatedSize(p: allocated);
626 PoisonShadow(addr: alloc_beg, size: user_beg - alloc_beg, value: kAsanHeapLeftRedzoneMagic);
627 PoisonShadow(addr: tail_beg, size: tail_end - tail_beg, value: kAsanHeapLeftRedzoneMagic);
628 }
629
630 uptr size_rounded_down_to_granularity =
631 RoundDownTo(x: size, ASAN_SHADOW_GRANULARITY);
632 // Unpoison the bulk of the memory region.
633 if (size_rounded_down_to_granularity)
634 PoisonShadow(addr: user_beg, size: size_rounded_down_to_granularity, value: 0);
635 // Deal with the end of the region if size is not aligned to granularity.
636 if (size != size_rounded_down_to_granularity && CanPoisonMemory()) {
637 u8 *shadow =
638 (u8 *)MemToShadow(p: user_beg + size_rounded_down_to_granularity);
639 *shadow = fl.poison_partial ? (size & (ASAN_SHADOW_GRANULARITY - 1)) : 0;
640 }
641
642 if (upgraded_from_zero)
643 PoisonShadow(addr: user_beg, ASAN_SHADOW_GRANULARITY,
644 value: kAsanHeapLeftRedzoneMagic);
645
646 AsanStats &thread_stats = GetCurrentThreadStats();
647 thread_stats.mallocs++;
648 thread_stats.malloced += size;
649 thread_stats.malloced_redzones += needed_size - size;
650 if (needed_size > SizeClassMap::kMaxSize)
651 thread_stats.malloc_large++;
652 else
653 thread_stats.malloced_by_size[SizeClassMap::ClassID(size: needed_size)]++;
654
655 void *res = reinterpret_cast<void *>(user_beg);
656 if (can_fill && fl.max_malloc_fill_size) {
657 uptr fill_size = Min(a: size, b: (uptr)fl.max_malloc_fill_size);
658 REAL(memset)(res, fl.malloc_fill_byte, fill_size);
659 }
660#if CAN_SANITIZE_LEAKS
661 m->lsan_tag = __lsan::DisabledInThisThread() ? __lsan::kIgnored
662 : __lsan::kDirectlyLeaked;
663#endif
664 // Must be the last mutation of metadata in this function.
665 atomic_store(a: &m->chunk_state, v: CHUNK_ALLOCATED, mo: memory_order_release);
666 if (alloc_beg != chunk_beg) {
667 CHECK_LE(alloc_beg + sizeof(LargeChunkHeader), chunk_beg);
668 reinterpret_cast<LargeChunkHeader *>(alloc_beg)->Set(m);
669 }
670 RunMallocHooks(ptr: res, size);
671 return res;
672 }
673
674 // Set quarantine flag if chunk is allocated, issue ASan error report on
675 // available and quarantined chunks. Return true on success, false otherwise.
676 bool AtomicallySetQuarantineFlagIfAllocated(AsanChunk *m, void *ptr,
677 BufferedStackTrace *stack) {
678 u8 old_chunk_state = CHUNK_ALLOCATED;
679 // Flip the chunk_state atomically to avoid race on double-free.
680 if (!atomic_compare_exchange_strong(a: &m->chunk_state, cmp: &old_chunk_state,
681 xchg: CHUNK_QUARANTINE,
682 mo: memory_order_acquire)) {
683 ReportInvalidFree(ptr, chunk_state: old_chunk_state, stack);
684 // It's not safe to push a chunk in quarantine on invalid free.
685 return false;
686 }
687 CHECK_EQ(CHUNK_ALLOCATED, old_chunk_state);
688 // It was a user data.
689 m->SetFreeContext(tid: kInvalidTid, stack: 0);
690 return true;
691 }
692
693 // Expects the chunk to already be marked as quarantined by using
694 // AtomicallySetQuarantineFlagIfAllocated.
695 void QuarantineChunk(AsanChunk *m, void *ptr, BufferedStackTrace *stack) {
696 CHECK_EQ(atomic_load(&m->chunk_state, memory_order_relaxed),
697 CHUNK_QUARANTINE);
698 AsanThread *t = GetCurrentThread();
699 m->SetFreeContext(tid: t ? t->tid() : 0, stack: StackDepotPut(stack: *stack));
700
701 // Push into quarantine.
702 if (t) {
703 AsanThreadLocalMallocStorage *ms = &t->malloc_storage();
704 AllocatorCache *ac = GetAllocatorCache(ms);
705 quarantine.Put(c: GetQuarantineCache(ms), cb: QuarantineCallback(ac, stack), ptr: m,
706 size: m->UsedSize());
707 } else {
708 SpinMutexLock l(&fallback_mutex);
709 AllocatorCache *ac = &fallback_allocator_cache;
710 quarantine.Put(c: &fallback_quarantine_cache, cb: QuarantineCallback(ac, stack),
711 ptr: m, size: m->UsedSize());
712 }
713 }
714
715 void Deallocate(void *ptr, uptr delete_size, uptr delete_alignment,
716 BufferedStackTrace *stack, AllocType alloc_type) {
717 uptr p = reinterpret_cast<uptr>(ptr);
718 if (p == 0) return;
719
720 uptr chunk_beg = p - kChunkHeaderSize;
721 AsanChunk *m = reinterpret_cast<AsanChunk *>(chunk_beg);
722
723 // On Windows, uninstrumented DLLs may allocate memory before ASan hooks
724 // malloc. Don't report an invalid free in this case.
725 if (SANITIZER_WINDOWS &&
726 !get_allocator().PointerIsMine(p: ptr)) {
727 if (!IsSystemHeapAddress(addr: p))
728 ReportFreeNotMalloced(addr: p, free_stack: stack);
729 return;
730 }
731
732 if (RunFreeHooks(ptr)) {
733 // Someone used __sanitizer_ignore_free_hook() and decided that they
734 // didn't want the memory to __sanitizer_ignore_free_hook freed right now.
735 // When they call free() on this pointer again at a later time, we should
736 // ignore the alloc-type mismatch and allow them to deallocate the pointer
737 // through free(), rather than the initial alloc type.
738 m->alloc_type = FROM_MALLOC;
739 return;
740 }
741
742 // Must mark the chunk as quarantined before any changes to its metadata.
743 // Do not quarantine given chunk if we failed to set CHUNK_QUARANTINE flag.
744 if (!AtomicallySetQuarantineFlagIfAllocated(m, ptr, stack)) return;
745
746 if (m->alloc_type != alloc_type) {
747 if (atomic_load(a: &alloc_dealloc_mismatch, mo: memory_order_acquire) &&
748 !IsAllocDeallocMismatchSuppressed(stack)) {
749 ReportAllocTypeMismatch(addr: (uptr)ptr, free_stack: stack, alloc_type: (AllocType)m->alloc_type,
750 dealloc_type: (AllocType)alloc_type);
751 }
752 } else {
753 if (flags()->new_delete_type_mismatch &&
754 (alloc_type == FROM_NEW || alloc_type == FROM_NEW_BR) &&
755 ((delete_size && delete_size != m->UsedSize()) ||
756 ComputeUserRequestedAlignmentLog(user_requested_alignment: delete_alignment) !=
757 m->user_requested_alignment_log)) {
758 ReportNewDeleteTypeMismatch(addr: p, delete_size, delete_alignment, free_stack: stack);
759 }
760 }
761
762 AsanStats &thread_stats = GetCurrentThreadStats();
763 thread_stats.frees++;
764 thread_stats.freed += m->UsedSize();
765
766 QuarantineChunk(m, ptr, stack);
767 }
768
769 void *Reallocate(void *old_ptr, uptr new_size, BufferedStackTrace *stack) {
770 CHECK(old_ptr && new_size);
771 uptr p = reinterpret_cast<uptr>(old_ptr);
772 uptr chunk_beg = p - kChunkHeaderSize;
773 AsanChunk *m = reinterpret_cast<AsanChunk *>(chunk_beg);
774
775 AsanStats &thread_stats = GetCurrentThreadStats();
776 thread_stats.reallocs++;
777 thread_stats.realloced += new_size;
778
779 void *new_ptr = Allocate(size: new_size, alignment: 8, stack, alloc_type: FROM_MALLOC, can_fill: true);
780 if (new_ptr) {
781 u8 chunk_state = atomic_load(a: &m->chunk_state, mo: memory_order_acquire);
782 if (chunk_state != CHUNK_ALLOCATED)
783 ReportInvalidFree(ptr: old_ptr, chunk_state, stack);
784 CHECK_NE(REAL(memcpy), nullptr);
785 uptr memcpy_size = Min(a: new_size, b: m->UsedSize());
786 // If realloc() races with free(), we may start copying freed memory.
787 // However, we will report racy double-free later anyway.
788 REAL(memcpy)(new_ptr, old_ptr, memcpy_size);
789 Deallocate(ptr: old_ptr, delete_size: 0, delete_alignment: 0, stack, alloc_type: FROM_MALLOC);
790 }
791 return new_ptr;
792 }
793
794 void *Calloc(uptr nmemb, uptr size, BufferedStackTrace *stack) {
795 if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
796 if (AllocatorMayReturnNull())
797 return nullptr;
798 ReportCallocOverflow(count: nmemb, size, stack);
799 }
800 void *ptr = Allocate(size: nmemb * size, alignment: 8, stack, alloc_type: FROM_MALLOC, can_fill: false);
801 // If the memory comes from the secondary allocator no need to clear it
802 // as it comes directly from mmap.
803 if (ptr && allocator.FromPrimary(p: ptr))
804 REAL(memset)(ptr, 0, nmemb * size);
805 return ptr;
806 }
807
808 void ReportInvalidFree(void *ptr, u8 chunk_state, BufferedStackTrace *stack) {
809 if (chunk_state == CHUNK_QUARANTINE)
810 ReportDoubleFree(addr: (uptr)ptr, free_stack: stack);
811 else
812 ReportFreeNotMalloced(addr: (uptr)ptr, free_stack: stack);
813 }
814
815 void CommitBack(AsanThreadLocalMallocStorage *ms, BufferedStackTrace *stack) {
816 AllocatorCache *ac = GetAllocatorCache(ms);
817 quarantine.Drain(c: GetQuarantineCache(ms), cb: QuarantineCallback(ac, stack));
818 allocator.SwallowCache(cache: ac);
819 }
820
821 // -------------------------- Chunk lookup ----------------------
822
823 // Assumes alloc_beg == allocator.GetBlockBegin(alloc_beg).
824 // Returns nullptr if AsanChunk is not yet initialized just after
825 // get_allocator().Allocate(), or is being destroyed just before
826 // get_allocator().Deallocate().
827 AsanChunk *GetAsanChunk(void *alloc_beg) {
828 if (!alloc_beg)
829 return nullptr;
830 AsanChunk *p = reinterpret_cast<LargeChunkHeader *>(alloc_beg)->Get();
831 if (!p) {
832 if (!allocator.FromPrimary(p: alloc_beg))
833 return nullptr;
834 p = reinterpret_cast<AsanChunk *>(alloc_beg);
835 }
836 u8 state = atomic_load(a: &p->chunk_state, mo: memory_order_relaxed);
837 // It does not guaranty that Chunk is initialized, but it's
838 // definitely not for any other value.
839 if (state == CHUNK_ALLOCATED || state == CHUNK_QUARANTINE)
840 return p;
841 return nullptr;
842 }
843
844 AsanChunk *GetAsanChunkByAddr(uptr p) {
845 void *alloc_beg = allocator.GetBlockBegin(p: reinterpret_cast<void *>(p));
846 return GetAsanChunk(alloc_beg);
847 }
848
849 // Allocator must be locked when this function is called.
850 AsanChunk *GetAsanChunkByAddrFastLocked(uptr p) {
851 void *alloc_beg =
852 allocator.GetBlockBeginFastLocked(p: reinterpret_cast<void *>(p));
853 return GetAsanChunk(alloc_beg);
854 }
855
856 uptr AllocationSize(uptr p) {
857 AsanChunk *m = GetAsanChunkByAddr(p);
858 if (!m) return 0;
859 if (atomic_load(a: &m->chunk_state, mo: memory_order_acquire) != CHUNK_ALLOCATED)
860 return 0;
861 if (m->Beg() != p) return 0;
862 return m->UsedSize();
863 }
864
865 uptr AllocationSizeFast(uptr p) {
866 return reinterpret_cast<AsanChunk *>(p - kChunkHeaderSize)->UsedSize();
867 }
868
869 AsanChunkView FindHeapChunkByAddress(uptr addr) {
870 AsanChunk *m1 = GetAsanChunkByAddr(p: addr);
871 sptr offset = 0;
872 if (!m1 || AsanChunkView(m1).AddrIsAtLeft(addr, access_size: 1, offset: &offset)) {
873 // The address is in the chunk's left redzone, so maybe it is actually
874 // a right buffer overflow from the other chunk before.
875 // Search a bit before to see if there is another chunk.
876 AsanChunk *m2 = nullptr;
877 for (uptr l = 1; l < GetPageSizeCached(); l++) {
878 m2 = GetAsanChunkByAddr(p: addr - l);
879 if (m2 == m1) continue; // Still the same chunk.
880 break;
881 }
882 if (m2 && AsanChunkView(m2).AddrIsAtRight(addr, access_size: 1, offset: &offset))
883 m1 = ChooseChunk(addr, left_chunk: m2, right_chunk: m1);
884 }
885 return AsanChunkView(m1);
886 }
887
888 void Purge(BufferedStackTrace *stack) {
889 AsanThread *t = GetCurrentThread();
890 if (t) {
891 AsanThreadLocalMallocStorage *ms = &t->malloc_storage();
892 quarantine.DrainAndRecycle(c: GetQuarantineCache(ms),
893 cb: QuarantineCallback(GetAllocatorCache(ms),
894 stack));
895 }
896 {
897 SpinMutexLock l(&fallback_mutex);
898 quarantine.DrainAndRecycle(c: &fallback_quarantine_cache,
899 cb: QuarantineCallback(&fallback_allocator_cache,
900 stack));
901 }
902
903 allocator.ForceReleaseToOS();
904 }
905
906 void PrintStats() {
907 allocator.PrintStats();
908 quarantine.PrintStats();
909 }
910
911 void ForceLock() SANITIZER_ACQUIRE(fallback_mutex) {
912 allocator.ForceLock();
913 fallback_mutex.Lock();
914 }
915
916 void ForceUnlock() SANITIZER_RELEASE(fallback_mutex) {
917 fallback_mutex.Unlock();
918 allocator.ForceUnlock();
919 }
920};
921
922static Allocator instance(LINKER_INITIALIZED);
923
924static AsanAllocator &get_allocator() {
925 return instance.allocator;
926}
927
928bool AsanChunkView::IsValid() const {
929 return chunk_ && atomic_load(a: &chunk_->chunk_state, mo: memory_order_relaxed) !=
930 CHUNK_INVALID;
931}
932bool AsanChunkView::IsAllocated() const {
933 return chunk_ && atomic_load(a: &chunk_->chunk_state, mo: memory_order_relaxed) ==
934 CHUNK_ALLOCATED;
935}
936bool AsanChunkView::IsQuarantined() const {
937 return chunk_ && atomic_load(a: &chunk_->chunk_state, mo: memory_order_relaxed) ==
938 CHUNK_QUARANTINE;
939}
940uptr AsanChunkView::Beg() const { return chunk_->Beg(); }
941uptr AsanChunkView::End() const { return Beg() + UsedSize(); }
942uptr AsanChunkView::UsedSize() const { return chunk_->UsedSize(); }
943u32 AsanChunkView::UserRequestedAlignment() const {
944 return Allocator::ComputeUserAlignment(user_requested_alignment_log: chunk_->user_requested_alignment_log);
945}
946
947uptr AsanChunkView::AllocTid() const {
948 u32 tid = 0;
949 u32 stack = 0;
950 chunk_->GetAllocContext(tid, stack);
951 return tid;
952}
953
954uptr AsanChunkView::FreeTid() const {
955 if (!IsQuarantined())
956 return kInvalidTid;
957 u32 tid = 0;
958 u32 stack = 0;
959 chunk_->GetFreeContext(tid, stack);
960 return tid;
961}
962
963AllocType AsanChunkView::GetAllocType() const {
964 return (AllocType)chunk_->alloc_type;
965}
966
967u32 AsanChunkView::GetAllocStackId() const {
968 u32 tid = 0;
969 u32 stack = 0;
970 chunk_->GetAllocContext(tid, stack);
971 return stack;
972}
973
974u32 AsanChunkView::GetFreeStackId() const {
975 if (!IsQuarantined())
976 return 0;
977 u32 tid = 0;
978 u32 stack = 0;
979 chunk_->GetFreeContext(tid, stack);
980 return stack;
981}
982
983void InitializeAllocator(const AllocatorOptions &options) {
984 instance.InitLinkerInitialized(options);
985}
986
987void ReInitializeAllocator(const AllocatorOptions &options) {
988 instance.ReInitialize(options);
989}
990
991// Apply provided AllocatorOptions to an Allocator
992void ApplyAllocatorOptions(const AllocatorOptions &options) {
993 instance.ApplyOptions(options);
994}
995
996void GetAllocatorOptions(AllocatorOptions *options) {
997 instance.GetOptions(options);
998}
999
1000AsanChunkView FindHeapChunkByAddress(uptr addr) {
1001 return instance.FindHeapChunkByAddress(addr);
1002}
1003AsanChunkView FindHeapChunkByAllocBeg(uptr addr) {
1004 return AsanChunkView(instance.GetAsanChunk(alloc_beg: reinterpret_cast<void*>(addr)));
1005}
1006
1007void AsanThreadLocalMallocStorage::CommitBack() {
1008 GET_STACK_TRACE_MALLOC;
1009 instance.CommitBack(ms: this, stack: &stack);
1010}
1011
1012void PrintInternalAllocatorStats() {
1013 instance.PrintStats();
1014}
1015
1016void asan_free(void *ptr, BufferedStackTrace *stack) {
1017 instance.Deallocate(ptr, delete_size: 0, delete_alignment: 0, stack, alloc_type: FROM_MALLOC);
1018}
1019
1020void *asan_malloc(uptr size, BufferedStackTrace *stack) {
1021 return SetErrnoOnNull(instance.Allocate(size, alignment: 8, stack, alloc_type: FROM_MALLOC, can_fill: true));
1022}
1023
1024void *asan_calloc(uptr nmemb, uptr size, BufferedStackTrace *stack) {
1025 return SetErrnoOnNull(instance.Calloc(nmemb, size, stack));
1026}
1027
1028void *asan_reallocarray(void *p, uptr nmemb, uptr size,
1029 BufferedStackTrace *stack) {
1030 if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
1031 errno = errno_ENOMEM;
1032 if (AllocatorMayReturnNull())
1033 return nullptr;
1034 ReportReallocArrayOverflow(count: nmemb, size, stack);
1035 }
1036 return asan_realloc(p, size: nmemb * size, stack);
1037}
1038
1039void *asan_realloc(void *p, uptr size, BufferedStackTrace *stack) {
1040 if (!p)
1041 return SetErrnoOnNull(instance.Allocate(size, alignment: 8, stack, alloc_type: FROM_MALLOC, can_fill: true));
1042 if (size == 0) {
1043 if (flags()->allocator_frees_and_returns_null_on_realloc_zero) {
1044 instance.Deallocate(ptr: p, delete_size: 0, delete_alignment: 0, stack, alloc_type: FROM_MALLOC);
1045 return nullptr;
1046 }
1047 // Allocate a size of 1 if we shouldn't free() on Realloc to 0
1048 size = 1;
1049 }
1050 return SetErrnoOnNull(instance.Reallocate(old_ptr: p, new_size: size, stack));
1051}
1052
1053void *asan_valloc(uptr size, BufferedStackTrace *stack) {
1054 return SetErrnoOnNull(
1055 instance.Allocate(size, alignment: GetPageSizeCached(), stack, alloc_type: FROM_MALLOC, can_fill: true));
1056}
1057
1058void *asan_pvalloc(uptr size, BufferedStackTrace *stack) {
1059 uptr PageSize = GetPageSizeCached();
1060 if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) {
1061 errno = errno_ENOMEM;
1062 if (AllocatorMayReturnNull())
1063 return nullptr;
1064 ReportPvallocOverflow(size, stack);
1065 }
1066 // pvalloc(0) should allocate one page.
1067 size = size ? RoundUpTo(size, boundary: PageSize) : PageSize;
1068 return SetErrnoOnNull(
1069 instance.Allocate(size, alignment: PageSize, stack, alloc_type: FROM_MALLOC, can_fill: true));
1070}
1071
1072void *asan_memalign(uptr alignment, uptr size, BufferedStackTrace *stack) {
1073 if (UNLIKELY(!IsPowerOfTwo(alignment))) {
1074 errno = errno_EINVAL;
1075 if (AllocatorMayReturnNull())
1076 return nullptr;
1077 ReportInvalidAllocationAlignment(alignment, stack);
1078 }
1079 return SetErrnoOnNull(
1080 instance.Allocate(size, alignment, stack, alloc_type: FROM_MALLOC, can_fill: true));
1081}
1082
1083void *asan_aligned_alloc(uptr alignment, uptr size, BufferedStackTrace *stack) {
1084 if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) {
1085 errno = errno_EINVAL;
1086 if (AllocatorMayReturnNull())
1087 return nullptr;
1088 ReportInvalidAlignedAllocAlignment(size, alignment, stack);
1089 }
1090 return SetErrnoOnNull(
1091 instance.Allocate(size, alignment, stack, alloc_type: FROM_MALLOC, can_fill: true));
1092}
1093
1094int asan_posix_memalign(void **memptr, uptr alignment, uptr size,
1095 BufferedStackTrace *stack) {
1096 if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) {
1097 if (AllocatorMayReturnNull())
1098 return errno_EINVAL;
1099 ReportInvalidPosixMemalignAlignment(alignment, stack);
1100 }
1101 void *ptr = instance.Allocate(size, alignment, stack, alloc_type: FROM_MALLOC, can_fill: true);
1102 if (UNLIKELY(!ptr))
1103 // OOM error is already taken care of by Allocate.
1104 return errno_ENOMEM;
1105 CHECK(IsAligned((uptr)ptr, alignment));
1106 *memptr = ptr;
1107 return 0;
1108}
1109
1110uptr asan_malloc_usable_size(const void *ptr, uptr pc, uptr bp) {
1111 if (!ptr) return 0;
1112 uptr usable_size = instance.AllocationSize(p: reinterpret_cast<uptr>(ptr));
1113 if (flags()->check_malloc_usable_size && (usable_size == 0)) {
1114 GET_STACK_TRACE_FATAL(pc, bp);
1115 ReportMallocUsableSizeNotOwned(addr: (uptr)ptr, stack: &stack);
1116 }
1117 return usable_size;
1118}
1119
1120namespace {
1121
1122void *asan_new(uptr size, BufferedStackTrace *stack, bool array) {
1123 return SetErrnoOnNull(
1124 instance.Allocate(size, alignment: 0, stack, alloc_type: array ? FROM_NEW_BR : FROM_NEW, can_fill: true));
1125}
1126
1127void *asan_new_aligned(uptr size, uptr alignment, BufferedStackTrace *stack,
1128 bool array) {
1129 if (UNLIKELY(alignment == 0 || !IsPowerOfTwo(alignment))) {
1130 errno = errno_EINVAL;
1131 if (AllocatorMayReturnNull())
1132 return nullptr;
1133 ReportInvalidAllocationAlignment(alignment, stack);
1134 }
1135 return SetErrnoOnNull(instance.Allocate(
1136 size, alignment, stack, alloc_type: array ? FROM_NEW_BR : FROM_NEW, can_fill: true));
1137}
1138
1139void asan_delete(void *ptr, BufferedStackTrace *stack, bool array) {
1140 instance.Deallocate(ptr, delete_size: 0, delete_alignment: 0, stack, alloc_type: array ? FROM_NEW_BR : FROM_NEW);
1141}
1142
1143void asan_delete_aligned(void *ptr, uptr alignment, BufferedStackTrace *stack,
1144 bool array) {
1145 instance.Deallocate(ptr, delete_size: 0, delete_alignment: alignment, stack, alloc_type: array ? FROM_NEW_BR : FROM_NEW);
1146}
1147
1148void asan_delete_sized(void *ptr, uptr size, BufferedStackTrace *stack,
1149 bool array) {
1150 instance.Deallocate(ptr, delete_size: size, delete_alignment: 0, stack, alloc_type: array ? FROM_NEW_BR : FROM_NEW);
1151}
1152
1153void asan_delete_sized_aligned(void *ptr, uptr size, uptr alignment,
1154 BufferedStackTrace *stack, bool array) {
1155 instance.Deallocate(ptr, delete_size: size, delete_alignment: alignment, stack,
1156 alloc_type: array ? FROM_NEW_BR : FROM_NEW);
1157}
1158
1159} // namespace
1160
1161void *asan_new(uptr size, BufferedStackTrace *stack) {
1162 return asan_new(size, stack, /*array=*/false);
1163}
1164
1165void *asan_new_aligned(uptr size, uptr alignment, BufferedStackTrace *stack) {
1166 return asan_new_aligned(size, alignment, stack, /*array=*/false);
1167}
1168
1169void *asan_new_array(uptr size, BufferedStackTrace *stack) {
1170 return asan_new(size, stack, /*array=*/true);
1171}
1172
1173void *asan_new_array_aligned(uptr size, uptr alignment,
1174 BufferedStackTrace *stack) {
1175 return asan_new_aligned(size, alignment, stack, /*array=*/true);
1176}
1177
1178void asan_delete(void *ptr, BufferedStackTrace *stack) {
1179 asan_delete(ptr, stack, /*array=*/false);
1180}
1181
1182void asan_delete_aligned(void *ptr, uptr alignment, BufferedStackTrace *stack) {
1183 asan_delete_aligned(ptr, alignment, stack, /*array=*/false);
1184}
1185
1186void asan_delete_sized(void *ptr, uptr size, BufferedStackTrace *stack) {
1187 asan_delete_sized(ptr, size, stack, /*array=*/false);
1188}
1189
1190void asan_delete_sized_aligned(void *ptr, uptr size, uptr alignment,
1191 BufferedStackTrace *stack) {
1192 asan_delete_sized_aligned(ptr, size, alignment, stack, /*array=*/false);
1193}
1194
1195void asan_delete_array(void *ptr, BufferedStackTrace *stack) {
1196 asan_delete(ptr, stack, /*array=*/true);
1197}
1198
1199void asan_delete_array_aligned(void *ptr, uptr alignment,
1200 BufferedStackTrace *stack) {
1201 asan_delete_aligned(ptr, alignment, stack, /*array=*/true);
1202}
1203
1204void asan_delete_array_sized(void *ptr, uptr size, BufferedStackTrace *stack) {
1205 asan_delete_sized(ptr, size, stack, /*array=*/true);
1206}
1207
1208void asan_delete_array_sized_aligned(void *ptr, uptr size, uptr alignment,
1209 BufferedStackTrace *stack) {
1210 asan_delete_sized_aligned(ptr, size, alignment, stack, /*array=*/true);
1211}
1212
1213uptr asan_mz_size(const void *ptr) {
1214 return instance.AllocationSize(p: reinterpret_cast<uptr>(ptr));
1215}
1216
1217void asan_mz_force_lock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
1218 instance.ForceLock();
1219}
1220
1221void asan_mz_force_unlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
1222 instance.ForceUnlock();
1223}
1224
1225} // namespace __asan
1226
1227// --- Implementation of LSan-specific functions --- {{{1
1228namespace __lsan {
1229void LockAllocator() {
1230 __asan::get_allocator().ForceLock();
1231}
1232
1233void UnlockAllocator() {
1234 __asan::get_allocator().ForceUnlock();
1235}
1236
1237void GetAllocatorGlobalRange(uptr *begin, uptr *end) {
1238 *begin = (uptr)&__asan::get_allocator();
1239 *end = *begin + sizeof(__asan::get_allocator());
1240}
1241
1242uptr PointsIntoChunk(void *p) {
1243 uptr addr = reinterpret_cast<uptr>(p);
1244 __asan::AsanChunk *m = __asan::instance.GetAsanChunkByAddrFastLocked(p: addr);
1245 if (!m || atomic_load(a: &m->chunk_state, mo: memory_order_acquire) !=
1246 __asan::CHUNK_ALLOCATED)
1247 return 0;
1248 uptr chunk = m->Beg();
1249 if (m->AddrIsInside(addr))
1250 return chunk;
1251 if (IsSpecialCaseOfOperatorNew0(chunk_beg: chunk, chunk_size: m->UsedSize(), addr))
1252 return chunk;
1253 return 0;
1254}
1255
1256uptr GetUserBegin(uptr chunk) {
1257 // FIXME: All usecases provide chunk address, GetAsanChunkByAddrFastLocked is
1258 // not needed.
1259 __asan::AsanChunk *m = __asan::instance.GetAsanChunkByAddrFastLocked(p: chunk);
1260 return m ? m->Beg() : 0;
1261}
1262
1263uptr GetUserAddr(uptr chunk) {
1264 return chunk;
1265}
1266
1267LsanMetadata::LsanMetadata(uptr chunk) {
1268 metadata_ = chunk ? reinterpret_cast<void *>(chunk - __asan::kChunkHeaderSize)
1269 : nullptr;
1270}
1271
1272bool LsanMetadata::allocated() const {
1273 if (!metadata_)
1274 return false;
1275 __asan::AsanChunk *m = reinterpret_cast<__asan::AsanChunk *>(metadata_);
1276 return atomic_load(a: &m->chunk_state, mo: memory_order_relaxed) ==
1277 __asan::CHUNK_ALLOCATED;
1278}
1279
1280ChunkTag LsanMetadata::tag() const {
1281 __asan::AsanChunk *m = reinterpret_cast<__asan::AsanChunk *>(metadata_);
1282 return static_cast<ChunkTag>(m->lsan_tag);
1283}
1284
1285void LsanMetadata::set_tag(ChunkTag value) {
1286 __asan::AsanChunk *m = reinterpret_cast<__asan::AsanChunk *>(metadata_);
1287 m->lsan_tag = value;
1288}
1289
1290uptr LsanMetadata::requested_size() const {
1291 __asan::AsanChunk *m = reinterpret_cast<__asan::AsanChunk *>(metadata_);
1292 return m->UsedSize();
1293}
1294
1295u32 LsanMetadata::stack_trace_id() const {
1296 __asan::AsanChunk *m = reinterpret_cast<__asan::AsanChunk *>(metadata_);
1297 u32 tid = 0;
1298 u32 stack = 0;
1299 m->GetAllocContext(tid, stack);
1300 return stack;
1301}
1302
1303void ForEachChunk(ForEachChunkCallback callback, void *arg) {
1304 __asan::get_allocator().ForEachChunk(callback, arg);
1305}
1306
1307IgnoreObjectResult IgnoreObject(const void *p) {
1308 uptr addr = reinterpret_cast<uptr>(p);
1309 __asan::AsanChunk *m = __asan::instance.GetAsanChunkByAddr(p: addr);
1310 if (!m ||
1311 (atomic_load(a: &m->chunk_state, mo: memory_order_acquire) !=
1312 __asan::CHUNK_ALLOCATED) ||
1313 !m->AddrIsInside(addr)) {
1314 return kIgnoreObjectInvalid;
1315 }
1316 if (m->lsan_tag == kIgnored)
1317 return kIgnoreObjectAlreadyIgnored;
1318 m->lsan_tag = __lsan::kIgnored;
1319 return kIgnoreObjectSuccess;
1320}
1321
1322} // namespace __lsan
1323
1324// ---------------------- Interface ---------------- {{{1
1325using namespace __asan;
1326
1327static const void *AllocationBegin(const void *p) {
1328 AsanChunk *m = __asan::instance.GetAsanChunkByAddr(p: (uptr)p);
1329 if (!m)
1330 return nullptr;
1331 if (atomic_load(a: &m->chunk_state, mo: memory_order_acquire) != CHUNK_ALLOCATED)
1332 return nullptr;
1333 if (m->UsedSize() == 0)
1334 return nullptr;
1335 return (const void *)(m->Beg());
1336}
1337
1338// ASan allocator doesn't reserve extra bytes, so normally we would
1339// just return "size". We don't want to expose our redzone sizes, etc here.
1340uptr __sanitizer_get_estimated_allocated_size(uptr size) {
1341 return size;
1342}
1343
1344int __sanitizer_get_ownership(const void *p) {
1345 uptr ptr = reinterpret_cast<uptr>(p);
1346 return instance.AllocationSize(p: ptr) > 0;
1347}
1348
1349uptr __sanitizer_get_allocated_size(const void *p) {
1350 if (!p) return 0;
1351 uptr ptr = reinterpret_cast<uptr>(p);
1352 uptr allocated_size = instance.AllocationSize(p: ptr);
1353 // Die if p is not malloced or if it is already freed.
1354 if (allocated_size == 0) {
1355 GET_STACK_TRACE_FATAL_HERE;
1356 ReportSanitizerGetAllocatedSizeNotOwned(addr: ptr, stack: &stack);
1357 }
1358 return allocated_size;
1359}
1360
1361uptr __sanitizer_get_allocated_size_fast(const void *p) {
1362 DCHECK_EQ(p, __sanitizer_get_allocated_begin(p));
1363 uptr ret = instance.AllocationSizeFast(p: reinterpret_cast<uptr>(p));
1364 DCHECK_EQ(ret, __sanitizer_get_allocated_size(p));
1365 return ret;
1366}
1367
1368const void *__sanitizer_get_allocated_begin(const void *p) {
1369 return AllocationBegin(p);
1370}
1371
1372void __sanitizer_purge_allocator() {
1373 GET_STACK_TRACE_MALLOC;
1374 instance.Purge(stack: &stack);
1375}
1376
1377int __asan_update_allocation_context(void* addr) {
1378 GET_STACK_TRACE_MALLOC;
1379 return instance.UpdateAllocationStack(addr: (uptr)addr, stack: &stack);
1380}
1381