1//===-- asan_allocator.cpp ------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is a part of AddressSanitizer, an address sanity checker.
10//
11// Implementation of ASan's memory allocator, 2-nd version.
12// This variant uses the allocator from sanitizer_common, i.e. the one shared
13// with ThreadSanitizer and MemorySanitizer.
14//
15//===----------------------------------------------------------------------===//
16
17#include "asan_allocator.h"
18
19#include "asan_internal.h"
20#include "asan_mapping.h"
21#include "asan_poisoning.h"
22#include "asan_report.h"
23#include "asan_stack.h"
24#include "asan_suppressions.h"
25#include "asan_thread.h"
26#include "lsan/lsan_common.h"
27#include "sanitizer_common/sanitizer_allocator_checks.h"
28#include "sanitizer_common/sanitizer_allocator_interface.h"
29#include "sanitizer_common/sanitizer_common.h"
30#include "sanitizer_common/sanitizer_errno.h"
31#include "sanitizer_common/sanitizer_flags.h"
32#include "sanitizer_common/sanitizer_internal_defs.h"
33#include "sanitizer_common/sanitizer_list.h"
34#include "sanitizer_common/sanitizer_quarantine.h"
35#include "sanitizer_common/sanitizer_stackdepot.h"
36
37namespace __asan {
38
39// Valid redzone sizes are 16, 32, 64, ... 2048, so we encode them in 3 bits.
40// We use adaptive redzones: for larger allocation larger redzones are used.
41static u32 RZLog2Size(u32 rz_log) {
42 CHECK_LT(rz_log, 8);
43 return 16 << rz_log;
44}
45
46static u32 RZSize2Log(u32 rz_size) {
47 CHECK_GE(rz_size, 16);
48 CHECK_LE(rz_size, 2048);
49 CHECK(IsPowerOfTwo(rz_size));
50 u32 res = Log2(x: rz_size) - 4;
51 CHECK_EQ(rz_size, RZLog2Size(res));
52 return res;
53}
54
55static AsanAllocator& get_allocator();
56
57static void AtomicContextStore(volatile atomic_uint64_t* atomic_context,
58 u32 tid, u32 stack) {
59 u64 context = tid;
60 context <<= 32;
61 context += stack;
62 atomic_store(a: atomic_context, v: context, mo: memory_order_relaxed);
63}
64
65static void AtomicContextLoad(const volatile atomic_uint64_t* atomic_context,
66 u32& tid, u32& stack) {
67 u64 context = atomic_load(a: atomic_context, mo: memory_order_relaxed);
68 stack = context;
69 context >>= 32;
70 tid = context;
71}
72
73// The memory chunk allocated from the underlying allocator looks like this:
74// L L L L L L H H U U U U U U R R
75// L -- left redzone words (0 or more bytes)
76// H -- ChunkHeader (16 bytes), which is also a part of the left redzone.
77// U -- user memory.
78// R -- right redzone (0 or more bytes)
79// ChunkBase consists of ChunkHeader and other bytes that overlap with user
80// memory.
81
82// If the left redzone is greater than the ChunkHeader size we store a magic
83// value in the first uptr word of the memory block and store the address of
84// ChunkBase in the next uptr.
85// M B L L L L L L L L L H H U U U U U U
86// | ^
87// ---------------------|
88// M -- magic value kAllocBegMagic
89// B -- address of ChunkHeader pointing to the first 'H'
90
91class ChunkHeader {
92 public:
93 atomic_uint8_t chunk_state;
94 u8 alloc_type : 2;
95 u8 lsan_tag : 2;
96#if SANITIZER_WINDOWS
97 // True if this was a zero-size allocation upgraded to size 1.
98 // Used to report the original size (0) to the user via HeapSize/RtlSizeHeap.
99 u8 from_zero_alloc : 1;
100#endif
101
102 // align < 8 -> 0
103 // else -> log2(min(align, 512)) - 2
104 u8 user_requested_alignment_log : 3;
105
106 private:
107 u16 user_requested_size_hi;
108 u32 user_requested_size_lo;
109 atomic_uint64_t alloc_context_id;
110
111 public:
112 uptr UsedSize() const {
113 static_assert(sizeof(user_requested_size_lo) == 4,
114 "Expression below requires this");
115 return FIRST_32_SECOND_64(0, ((uptr)user_requested_size_hi << 32)) +
116 user_requested_size_lo;
117 }
118
119 void SetUsedSize(uptr size) {
120 user_requested_size_lo = size;
121 static_assert(sizeof(user_requested_size_lo) == 4,
122 "Expression below requires this");
123 user_requested_size_hi = FIRST_32_SECOND_64(0, size >> 32);
124 CHECK_EQ(UsedSize(), size);
125 }
126
127 void SetAllocContext(u32 tid, u32 stack) {
128 AtomicContextStore(atomic_context: &alloc_context_id, tid, stack);
129 }
130
131 void GetAllocContext(u32& tid, u32& stack) const {
132 AtomicContextLoad(atomic_context: &alloc_context_id, tid, stack);
133 }
134};
135
136class ChunkBase : public ChunkHeader {
137 atomic_uint64_t free_context_id;
138
139 public:
140 void SetFreeContext(u32 tid, u32 stack) {
141 AtomicContextStore(atomic_context: &free_context_id, tid, stack);
142 }
143
144 void GetFreeContext(u32& tid, u32& stack) const {
145 AtomicContextLoad(atomic_context: &free_context_id, tid, stack);
146 }
147};
148
149static const uptr kChunkHeaderSize = sizeof(ChunkHeader);
150static const uptr kChunkHeader2Size = sizeof(ChunkBase) - kChunkHeaderSize;
151COMPILER_CHECK(kChunkHeaderSize == 16);
152COMPILER_CHECK(kChunkHeader2Size <= 16);
153
154enum {
155 // Either just allocated by underlying allocator, but AsanChunk is not yet
156 // ready, or almost returned to undelying allocator and AsanChunk is already
157 // meaningless.
158 CHUNK_INVALID = 0,
159 // The chunk is allocated and not yet freed.
160 CHUNK_ALLOCATED = 2,
161 // The chunk was freed and put into quarantine zone.
162 CHUNK_QUARANTINE = 3,
163};
164
165class AsanChunk : public ChunkBase {
166 public:
167 uptr Beg() { return reinterpret_cast<uptr>(this) + kChunkHeaderSize; }
168 bool AddrIsInside(uptr addr) {
169 return (addr >= Beg()) && (addr < Beg() + UsedSize());
170 }
171};
172
173class LargeChunkHeader {
174 static constexpr uptr kAllocBegMagic =
175 FIRST_32_SECOND_64(0xCC6E96B9, 0xCC6E96B9CC6E96B9ULL);
176 atomic_uintptr_t magic;
177 AsanChunk* chunk_header;
178
179 public:
180 AsanChunk* Get() const {
181 return atomic_load(a: &magic, mo: memory_order_acquire) == kAllocBegMagic
182 ? chunk_header
183 : nullptr;
184 }
185
186 void Set(AsanChunk* p) {
187 if (p) {
188 chunk_header = p;
189 atomic_store(a: &magic, v: kAllocBegMagic, mo: memory_order_release);
190 return;
191 }
192
193 uptr old = kAllocBegMagic;
194 if (!atomic_compare_exchange_strong(a: &magic, cmp: &old, xchg: 0,
195 mo: memory_order_release)) {
196 CHECK_EQ(old, kAllocBegMagic);
197 }
198 }
199};
200
201static void FillChunk(AsanChunk* m) {
202 // FIXME: Use ReleaseMemoryPagesToOS.
203 Flags& fl = *flags();
204
205 if (fl.max_free_fill_size > 0) {
206 // We have to skip the chunk header, it contains free_context_id.
207 uptr scribble_start = (uptr)m + kChunkHeaderSize + kChunkHeader2Size;
208 if (m->UsedSize() >= kChunkHeader2Size) { // Skip Header2 in user area.
209 uptr size_to_fill = m->UsedSize() - kChunkHeader2Size;
210 size_to_fill = Min(a: size_to_fill, b: (uptr)fl.max_free_fill_size);
211 REAL(memset)((void*)scribble_start, fl.free_fill_byte, size_to_fill);
212 }
213 }
214}
215
216struct QuarantineCallback {
217 QuarantineCallback(AllocatorCache* cache, BufferedStackTrace* stack)
218 : cache_(cache), stack_(stack) {}
219
220 void PreQuarantine(AsanChunk* m) const {
221 FillChunk(m);
222 // Poison the region.
223 PoisonShadow(addr: m->Beg(), size: RoundUpTo(size: m->UsedSize(), ASAN_SHADOW_GRANULARITY),
224 value: kAsanHeapFreeMagic);
225 }
226
227 void Recycle(AsanChunk* m) const {
228 void* p = get_allocator().GetBlockBegin(p: m);
229
230 // The secondary will immediately unpoison and unmap the memory, so this
231 // branch is unnecessary.
232 if (get_allocator().FromPrimary(p)) {
233 if (p != m) {
234 // Clear the magic value, as allocator internals may overwrite the
235 // contents of deallocated chunk, confusing GetAsanChunk lookup.
236 reinterpret_cast<LargeChunkHeader*>(p)->Set(nullptr);
237 }
238
239 u8 old_chunk_state = CHUNK_QUARANTINE;
240 if (!atomic_compare_exchange_strong(a: &m->chunk_state, cmp: &old_chunk_state,
241 xchg: CHUNK_INVALID,
242 mo: memory_order_acquire)) {
243 CHECK_EQ(old_chunk_state, CHUNK_QUARANTINE);
244 }
245
246 PoisonShadow(addr: m->Beg(), size: RoundUpTo(size: m->UsedSize(), ASAN_SHADOW_GRANULARITY),
247 value: kAsanHeapLeftRedzoneMagic);
248 }
249
250 // Statistics.
251 AsanStats& thread_stats = GetCurrentThreadStats();
252 thread_stats.real_frees++;
253 thread_stats.really_freed += m->UsedSize();
254
255 get_allocator().Deallocate(cache: cache_, p);
256 }
257
258 void RecyclePassThrough(AsanChunk* m) const {
259 // Recycle for the secondary will immediately unpoison and unmap the
260 // memory, so quarantine preparation is unnecessary.
261 if (get_allocator().FromPrimary(p: m)) {
262 // The primary allocation may need pattern fill if enabled.
263 FillChunk(m);
264 }
265 Recycle(m);
266 }
267
268 void* Allocate(uptr size) const {
269 void* res = get_allocator().Allocate(cache: cache_, size, alignment: 1);
270 // TODO(alekseys): Consider making quarantine OOM-friendly.
271 if (UNLIKELY(!res))
272 ReportOutOfMemory(requested_size: size, stack: stack_);
273 return res;
274 }
275
276 void Deallocate(void* p) const { get_allocator().Deallocate(cache: cache_, p); }
277
278 private:
279 AllocatorCache* const cache_;
280 BufferedStackTrace* const stack_;
281};
282
283typedef Quarantine<QuarantineCallback, AsanChunk> AsanQuarantine;
284typedef AsanQuarantine::Cache QuarantineCache;
285
286void AsanMapUnmapCallback::OnMap(uptr p, uptr size) const {
287 PoisonShadow(addr: p, size, value: kAsanHeapLeftRedzoneMagic);
288 // Statistics.
289 AsanStats& thread_stats = GetCurrentThreadStats();
290 thread_stats.mmaps++;
291 thread_stats.mmaped += size;
292}
293
294void AsanMapUnmapCallback::OnMapSecondary(uptr p, uptr size, uptr user_begin,
295 uptr user_size) const {
296 uptr user_end = RoundDownTo(x: user_begin + user_size, ASAN_SHADOW_GRANULARITY);
297 user_begin = RoundUpTo(size: user_begin, ASAN_SHADOW_GRANULARITY);
298 // The secondary mapping will be immediately returned to user, no value
299 // poisoning that with non-zero just before unpoisoning by Allocate(). So just
300 // poison head/tail invisible to Allocate().
301 PoisonShadow(addr: p, size: user_begin - p, value: kAsanHeapLeftRedzoneMagic);
302 PoisonShadow(addr: user_end, size: size - (user_end - p), value: kAsanHeapLeftRedzoneMagic);
303 // Statistics.
304 AsanStats& thread_stats = GetCurrentThreadStats();
305 thread_stats.mmaps++;
306 thread_stats.mmaped += size;
307}
308
309void AsanMapUnmapCallback::OnUnmap(uptr p, uptr size) const {
310 PoisonShadow(addr: p, size, value: 0);
311 // We are about to unmap a chunk of user memory.
312 // Mark the corresponding shadow memory as not needed.
313 FlushUnneededASanShadowMemory(p, size);
314 // Statistics.
315 AsanStats& thread_stats = GetCurrentThreadStats();
316 thread_stats.munmaps++;
317 thread_stats.munmaped += size;
318}
319
320// We can not use THREADLOCAL because it is not supported on some of the
321// platforms we care about (OSX 10.6, Android).
322// static THREADLOCAL AllocatorCache cache;
323AllocatorCache* GetAllocatorCache(AsanThreadLocalMallocStorage* ms) {
324 CHECK(ms);
325 return &ms->allocator_cache;
326}
327
328QuarantineCache* GetQuarantineCache(AsanThreadLocalMallocStorage* ms) {
329 CHECK(ms);
330 CHECK_LE(sizeof(QuarantineCache), sizeof(ms->quarantine_cache));
331 return reinterpret_cast<QuarantineCache*>(ms->quarantine_cache);
332}
333
334void AllocatorOptions::SetFrom(const Flags* f, const CommonFlags* cf) {
335 quarantine_size_mb = f->quarantine_size_mb;
336 thread_local_quarantine_size_kb = f->thread_local_quarantine_size_kb;
337 min_redzone = f->redzone;
338 max_redzone = f->max_redzone;
339 may_return_null = cf->allocator_may_return_null;
340 alloc_dealloc_mismatch = f->alloc_dealloc_mismatch;
341 release_to_os_interval_ms = cf->allocator_release_to_os_interval_ms;
342}
343
344void AllocatorOptions::CopyTo(Flags* f, CommonFlags* cf) {
345 f->quarantine_size_mb = quarantine_size_mb;
346 f->thread_local_quarantine_size_kb = thread_local_quarantine_size_kb;
347 f->redzone = min_redzone;
348 f->max_redzone = max_redzone;
349 cf->allocator_may_return_null = may_return_null;
350 f->alloc_dealloc_mismatch = alloc_dealloc_mismatch;
351 cf->allocator_release_to_os_interval_ms = release_to_os_interval_ms;
352}
353
354struct Allocator {
355 static const uptr kMaxAllowedMallocSize =
356 FIRST_32_SECOND_64(3UL << 30, 1ULL << 40);
357
358 AsanAllocator allocator;
359 AsanQuarantine quarantine;
360 StaticSpinMutex fallback_mutex;
361 AllocatorCache fallback_allocator_cache;
362 QuarantineCache fallback_quarantine_cache;
363
364 uptr max_user_defined_malloc_size;
365
366 // ------------------- Options --------------------------
367 atomic_uint16_t min_redzone;
368 atomic_uint16_t max_redzone;
369 atomic_uint8_t alloc_dealloc_mismatch;
370
371 // ------------------- Initialization ------------------------
372 explicit Allocator(LinkerInitialized)
373 : quarantine(LINKER_INITIALIZED),
374 fallback_quarantine_cache(LINKER_INITIALIZED) {}
375
376 void CheckOptions(const AllocatorOptions& options) const {
377 CHECK_GE(options.min_redzone, 16);
378 CHECK_GE(options.max_redzone, options.min_redzone);
379 CHECK_LE(options.max_redzone, 2048);
380 CHECK(IsPowerOfTwo(options.min_redzone));
381 CHECK(IsPowerOfTwo(options.max_redzone));
382 }
383
384 void SharedInitCode(const AllocatorOptions& options) {
385 CheckOptions(options);
386 quarantine.Init(size: (uptr)options.quarantine_size_mb << 20,
387 cache_size: (uptr)options.thread_local_quarantine_size_kb << 10);
388 atomic_store(a: &alloc_dealloc_mismatch, v: options.alloc_dealloc_mismatch,
389 mo: memory_order_release);
390 atomic_store(a: &min_redzone, v: options.min_redzone, mo: memory_order_release);
391 atomic_store(a: &max_redzone, v: options.max_redzone, mo: memory_order_release);
392 }
393
394 void InitLinkerInitialized(const AllocatorOptions& options) {
395 SetAllocatorMayReturnNull(options.may_return_null);
396 allocator.InitLinkerInitialized(release_to_os_interval_ms: options.release_to_os_interval_ms);
397 SharedInitCode(options);
398 max_user_defined_malloc_size = common_flags()->max_allocation_size_mb
399 ? common_flags()->max_allocation_size_mb
400 << 20
401 : kMaxAllowedMallocSize;
402 }
403
404 void RePoisonChunk(uptr chunk) {
405 // This could be a user-facing chunk (with redzones), or some internal
406 // housekeeping chunk, like TransferBatch. Start by assuming the former.
407 AsanChunk* ac = GetAsanChunk(alloc_beg: (void*)chunk);
408 uptr allocated_size = allocator.GetActuallyAllocatedSize(p: (void*)chunk);
409 if (ac && atomic_load(a: &ac->chunk_state, mo: memory_order_acquire) ==
410 CHUNK_ALLOCATED) {
411 uptr beg = ac->Beg();
412 uptr end = ac->Beg() + ac->UsedSize();
413 uptr chunk_end = chunk + allocated_size;
414 if (chunk < beg && beg < end && end <= chunk_end) {
415 // Looks like a valid AsanChunk in use, poison redzones only.
416 PoisonShadow(addr: chunk, size: beg - chunk, value: kAsanHeapLeftRedzoneMagic);
417 uptr end_aligned_down = RoundDownTo(x: end, ASAN_SHADOW_GRANULARITY);
418 FastPoisonShadowPartialRightRedzone(
419 aligned_addr: end_aligned_down, size: end - end_aligned_down,
420 redzone_size: chunk_end - end_aligned_down, value: kAsanHeapLeftRedzoneMagic);
421 return;
422 }
423 }
424
425 // This is either not an AsanChunk or freed or quarantined AsanChunk.
426 // In either case, poison everything.
427 PoisonShadow(addr: chunk, size: allocated_size, value: kAsanHeapLeftRedzoneMagic);
428 }
429
430 // Apply provided AllocatorOptions to an Allocator
431 void ApplyOptions(const AllocatorOptions& options) {
432 SetAllocatorMayReturnNull(options.may_return_null);
433 allocator.SetReleaseToOSIntervalMs(options.release_to_os_interval_ms);
434 SharedInitCode(options);
435 }
436
437 void ReInitialize(const AllocatorOptions& options) {
438 ApplyOptions(options);
439
440 // Poison all existing allocation's redzones.
441 if (CanPoisonMemory()) {
442 allocator.ForceLock();
443 allocator.ForEachChunk(
444 callback: [](uptr chunk, void* alloc) {
445 ((Allocator*)alloc)->RePoisonChunk(chunk);
446 },
447 arg: this);
448 allocator.ForceUnlock();
449 }
450 }
451
452 void GetOptions(AllocatorOptions* options) const {
453 options->quarantine_size_mb = quarantine.GetMaxSize() >> 20;
454 options->thread_local_quarantine_size_kb =
455 quarantine.GetMaxCacheSize() >> 10;
456 options->min_redzone = atomic_load(a: &min_redzone, mo: memory_order_acquire);
457 options->max_redzone = atomic_load(a: &max_redzone, mo: memory_order_acquire);
458 options->may_return_null = AllocatorMayReturnNull();
459 options->alloc_dealloc_mismatch =
460 atomic_load(a: &alloc_dealloc_mismatch, mo: memory_order_acquire);
461 options->release_to_os_interval_ms = allocator.ReleaseToOSIntervalMs();
462 }
463
464 // -------------------- Helper methods. -------------------------
465 uptr ComputeRZLog(uptr user_requested_size) {
466 u32 rz_log = user_requested_size <= 64 - 16 ? 0
467 : user_requested_size <= 128 - 32 ? 1
468 : user_requested_size <= 512 - 64 ? 2
469 : user_requested_size <= 4096 - 128 ? 3
470 : user_requested_size <= (1 << 14) - 256 ? 4
471 : user_requested_size <= (1 << 15) - 512 ? 5
472 : user_requested_size <= (1 << 16) - 1024 ? 6
473 : 7;
474 u32 hdr_log = RZSize2Log(rz_size: RoundUpToPowerOfTwo(size: sizeof(ChunkHeader)));
475 u32 min_log = RZSize2Log(rz_size: atomic_load(a: &min_redzone, mo: memory_order_acquire));
476 u32 max_log = RZSize2Log(rz_size: atomic_load(a: &max_redzone, mo: memory_order_acquire));
477 return Min(a: Max(a: rz_log, b: Max(a: min_log, b: hdr_log)), b: Max(a: max_log, b: hdr_log));
478 }
479
480 static uptr ComputeUserRequestedAlignmentLog(uptr user_requested_alignment) {
481 if (user_requested_alignment < 8)
482 return 0;
483 if (user_requested_alignment > 512)
484 user_requested_alignment = 512;
485 return Log2(x: user_requested_alignment) - 2;
486 }
487
488 static uptr ComputeUserAlignment(uptr user_requested_alignment_log) {
489 if (user_requested_alignment_log == 0)
490 return 0;
491 return 1LL << (user_requested_alignment_log + 2);
492 }
493
494 // We have an address between two chunks, and we want to report just one.
495 AsanChunk* ChooseChunk(uptr addr, AsanChunk* left_chunk,
496 AsanChunk* right_chunk) {
497 if (!left_chunk)
498 return right_chunk;
499 if (!right_chunk)
500 return left_chunk;
501 // Prefer an allocated chunk over freed chunk and freed chunk
502 // over available chunk.
503 u8 left_state = atomic_load(a: &left_chunk->chunk_state, mo: memory_order_relaxed);
504 u8 right_state =
505 atomic_load(a: &right_chunk->chunk_state, mo: memory_order_relaxed);
506 if (left_state != right_state) {
507 if (left_state == CHUNK_ALLOCATED)
508 return left_chunk;
509 if (right_state == CHUNK_ALLOCATED)
510 return right_chunk;
511 if (left_state == CHUNK_QUARANTINE)
512 return left_chunk;
513 if (right_state == CHUNK_QUARANTINE)
514 return right_chunk;
515 }
516 // Same chunk_state: choose based on offset.
517 sptr l_offset = 0, r_offset = 0;
518 CHECK(AsanChunkView(left_chunk).AddrIsAtRight(addr, 1, &l_offset));
519 CHECK(AsanChunkView(right_chunk).AddrIsAtLeft(addr, 1, &r_offset));
520 if (l_offset < r_offset)
521 return left_chunk;
522 return right_chunk;
523 }
524
525 bool UpdateAllocationStack(uptr addr, BufferedStackTrace* stack) {
526 AsanChunk* m = GetAsanChunkByAddr(p: addr);
527 if (!m)
528 return false;
529 if (atomic_load(a: &m->chunk_state, mo: memory_order_acquire) != CHUNK_ALLOCATED)
530 return false;
531 if (m->Beg() != addr)
532 return false;
533 AsanThread* t = GetCurrentThread();
534 m->SetAllocContext(tid: t ? t->tid() : kMainTid, stack: StackDepotPut(stack: *stack));
535 return true;
536 }
537
538 // -------------------- Allocation/Deallocation routines ---------------
539 // may_return_null tells AllocateImpl() whether OOM should produce a nullptr
540 // (true) or a fatal Report*+Die() (false).
541 void* AllocateImpl(uptr size, uptr alignment, BufferedStackTrace* stack,
542 AllocType alloc_type, bool can_fill,
543 bool may_return_null) {
544 if (UNLIKELY(!AsanInited()))
545 AsanInitFromRtl();
546 if (UNLIKELY(IsRssLimitExceeded())) {
547 if (may_return_null)
548 return nullptr;
549 ReportRssLimitExceeded(stack);
550 }
551 Flags& fl = *flags();
552 CHECK(stack);
553 const uptr min_alignment = ASAN_SHADOW_GRANULARITY;
554 const uptr user_requested_alignment_log =
555 ComputeUserRequestedAlignmentLog(user_requested_alignment: alignment);
556 if (alignment < min_alignment)
557 alignment = min_alignment;
558 bool upgraded_from_zero = false;
559 if (size == 0) {
560 // We'd be happy to avoid allocating memory for zero-size requests, but
561 // some programs/tests depend on this behavior and assume that malloc
562 // would not return NULL even for zero-size allocations. Moreover, it
563 // looks like operator new should never return NULL, and results of
564 // consecutive "new" calls must be different even if the allocated size
565 // is zero.
566 size = 1;
567 upgraded_from_zero = true;
568 }
569 CHECK(IsPowerOfTwo(alignment));
570 uptr rz_log = ComputeRZLog(user_requested_size: size);
571 uptr rz_size = RZLog2Size(rz_log);
572 uptr rounded_size = RoundUpTo(size: Max(a: size, b: kChunkHeader2Size), boundary: alignment);
573 uptr needed_size = rounded_size + rz_size;
574 if (alignment > min_alignment)
575 needed_size += alignment;
576 bool from_primary = PrimaryAllocator::CanAllocate(size: needed_size, alignment);
577 // If we are allocating from the secondary allocator, there will be no
578 // automatic right redzone, so add the right redzone manually.
579 if (!from_primary)
580 needed_size += rz_size;
581 CHECK(IsAligned(needed_size, min_alignment));
582 if (size > kMaxAllowedMallocSize || needed_size > kMaxAllowedMallocSize ||
583 size > max_user_defined_malloc_size) {
584 if (may_return_null) {
585 Report(format: "WARNING: AddressSanitizer failed to allocate 0x%zx bytes\n",
586 size);
587 return nullptr;
588 }
589 uptr malloc_limit =
590 Min(a: kMaxAllowedMallocSize, b: max_user_defined_malloc_size);
591 ReportAllocationSizeTooBig(user_size: size, total_size: needed_size, max_size: malloc_limit, stack);
592 }
593
594 AsanThread* t = GetCurrentThread();
595 void* allocated;
596 if (t) {
597 AllocatorCache* cache = GetAllocatorCache(ms: &t->malloc_storage());
598 allocated = allocator.Allocate(cache, size: needed_size, alignment: 8);
599 } else {
600 SpinMutexLock l(&fallback_mutex);
601 AllocatorCache* cache = &fallback_allocator_cache;
602 allocated = allocator.Allocate(cache, size: needed_size, alignment: 8);
603 }
604 if (UNLIKELY(!allocated)) {
605 SetAllocatorOutOfMemory();
606 if (may_return_null)
607 return nullptr;
608 ReportOutOfMemory(requested_size: size, stack);
609 }
610
611 uptr alloc_beg = reinterpret_cast<uptr>(allocated);
612 uptr alloc_end = alloc_beg + needed_size;
613 uptr user_beg = alloc_beg + rz_size;
614 if (!IsAligned(a: user_beg, alignment))
615 user_beg = RoundUpTo(size: user_beg, boundary: alignment);
616 uptr user_end = user_beg + size;
617 CHECK_LE(user_end, alloc_end);
618 uptr chunk_beg = user_beg - kChunkHeaderSize;
619 AsanChunk* m = reinterpret_cast<AsanChunk*>(chunk_beg);
620 m->alloc_type = alloc_type;
621#if SANITIZER_WINDOWS
622 m->from_zero_alloc = upgraded_from_zero;
623#endif
624 CHECK(size);
625 m->SetUsedSize(size);
626 m->user_requested_alignment_log = user_requested_alignment_log;
627
628 m->SetAllocContext(tid: t ? t->tid() : kMainTid, stack: StackDepotPut(stack: *stack));
629
630 if (!from_primary || *(u8*)MEM_TO_SHADOW((uptr)allocated) == 0) {
631 // The allocator provides an unpoisoned chunk. This is possible for the
632 // secondary allocator, or if CanPoisonMemory() was false for some time,
633 // for example, due to flags()->start_disabled. Anyway, poison left and
634 // right of the block before using it for anything else.
635 uptr tail_beg = RoundUpTo(size: user_end, ASAN_SHADOW_GRANULARITY);
636 uptr tail_end = alloc_beg + allocator.GetActuallyAllocatedSize(p: allocated);
637 PoisonShadow(addr: alloc_beg, size: user_beg - alloc_beg, value: kAsanHeapLeftRedzoneMagic);
638 PoisonShadow(addr: tail_beg, size: tail_end - tail_beg, value: kAsanHeapLeftRedzoneMagic);
639 }
640
641 uptr size_rounded_down_to_granularity =
642 RoundDownTo(x: size, ASAN_SHADOW_GRANULARITY);
643 // Unpoison the bulk of the memory region.
644 if (size_rounded_down_to_granularity)
645 PoisonShadow(addr: user_beg, size: size_rounded_down_to_granularity, value: 0);
646 // Deal with the end of the region if size is not aligned to granularity.
647 if (size != size_rounded_down_to_granularity && CanPoisonMemory()) {
648 u8* shadow =
649 (u8*)MemToShadow(p: user_beg + size_rounded_down_to_granularity);
650 *shadow = fl.poison_partial ? (size & (ASAN_SHADOW_GRANULARITY - 1)) : 0;
651 }
652
653 if (upgraded_from_zero)
654 PoisonShadow(addr: user_beg, ASAN_SHADOW_GRANULARITY,
655 value: kAsanHeapLeftRedzoneMagic);
656
657 AsanStats& thread_stats = GetCurrentThreadStats();
658 thread_stats.mallocs++;
659 thread_stats.malloced += size;
660 thread_stats.malloced_redzones += needed_size - size;
661 if (needed_size > SizeClassMap::kMaxSize)
662 thread_stats.malloc_large++;
663 else
664 thread_stats.malloced_by_size[SizeClassMap::ClassID(size: needed_size)]++;
665
666 void* res = reinterpret_cast<void*>(user_beg);
667 if (can_fill && fl.max_malloc_fill_size) {
668 uptr fill_size = Min(a: size, b: (uptr)fl.max_malloc_fill_size);
669 REAL(memset)(res, fl.malloc_fill_byte, fill_size);
670 }
671#if CAN_SANITIZE_LEAKS
672 m->lsan_tag = __lsan::DisabledInThisThread() ? __lsan::kIgnored
673 : __lsan::kDirectlyLeaked;
674#endif
675 // Must be the last mutation of metadata in this function.
676 atomic_store(a: &m->chunk_state, v: CHUNK_ALLOCATED, mo: memory_order_release);
677 if (alloc_beg != chunk_beg) {
678 CHECK_LE(alloc_beg + sizeof(LargeChunkHeader), chunk_beg);
679 reinterpret_cast<LargeChunkHeader*>(alloc_beg)->Set(m);
680 }
681 RunMallocHooks(ptr: res, size);
682 return res;
683 }
684
685 // Defer to the global, flag controlled, OOM policy.
686 void* Allocate(uptr size, uptr alignment, BufferedStackTrace* stack,
687 AllocType alloc_type, bool can_fill) {
688 return AllocateImpl(size, alignment, stack, alloc_type, can_fill,
689 may_return_null: AllocatorMayReturnNull());
690 }
691
692 // Set quarantine flag if chunk is allocated, issue ASan error report on
693 // available and quarantined chunks. Return true on success, false otherwise.
694 bool AtomicallySetQuarantineFlagIfAllocated(AsanChunk* m, void* ptr,
695 BufferedStackTrace* stack) {
696 u8 old_chunk_state = CHUNK_ALLOCATED;
697 // Flip the chunk_state atomically to avoid race on double-free.
698 if (!atomic_compare_exchange_strong(a: &m->chunk_state, cmp: &old_chunk_state,
699 xchg: CHUNK_QUARANTINE,
700 mo: memory_order_acquire)) {
701 ReportInvalidFree(ptr, chunk_state: old_chunk_state, stack);
702 // It's not safe to push a chunk in quarantine on invalid free.
703 return false;
704 }
705 CHECK_EQ(CHUNK_ALLOCATED, old_chunk_state);
706 // It was a user data.
707 m->SetFreeContext(tid: kInvalidTid, stack: 0);
708 return true;
709 }
710
711 // Expects the chunk to already be marked as quarantined by using
712 // AtomicallySetQuarantineFlagIfAllocated.
713 void QuarantineChunk(AsanChunk* m, void* ptr, BufferedStackTrace* stack) {
714 CHECK_EQ(atomic_load(&m->chunk_state, memory_order_relaxed),
715 CHUNK_QUARANTINE);
716 AsanThread* t = GetCurrentThread();
717 m->SetFreeContext(tid: t ? t->tid() : 0, stack: StackDepotPut(stack: *stack));
718
719 // Push into quarantine.
720 if (t) {
721 AsanThreadLocalMallocStorage* ms = &t->malloc_storage();
722 AllocatorCache* ac = GetAllocatorCache(ms);
723 quarantine.Put(c: GetQuarantineCache(ms), cb: QuarantineCallback(ac, stack), ptr: m,
724 size: m->UsedSize());
725 } else {
726 SpinMutexLock l(&fallback_mutex);
727 AllocatorCache* ac = &fallback_allocator_cache;
728 quarantine.Put(c: &fallback_quarantine_cache, cb: QuarantineCallback(ac, stack),
729 ptr: m, size: m->UsedSize());
730 }
731 }
732
733 void Deallocate(void* ptr, uptr delete_size, uptr delete_alignment,
734 BufferedStackTrace* stack, AllocType alloc_type) {
735 uptr p = reinterpret_cast<uptr>(ptr);
736 if (p == 0)
737 return;
738
739 uptr chunk_beg = p - kChunkHeaderSize;
740 AsanChunk* m = reinterpret_cast<AsanChunk*>(chunk_beg);
741
742 // On Windows, uninstrumented DLLs may allocate memory before ASan hooks
743 // malloc. Don't report an invalid free in this case.
744 if (SANITIZER_WINDOWS && !get_allocator().PointerIsMine(p: ptr)) {
745 if (!IsSystemHeapAddress(addr: p))
746 ReportFreeNotMalloced(addr: p, free_stack: stack);
747 return;
748 }
749
750 if (RunFreeHooks(ptr)) {
751 // Someone used __sanitizer_ignore_free_hook() and decided that they
752 // didn't want the memory to __sanitizer_ignore_free_hook freed right now.
753 // When they call free() on this pointer again at a later time, we should
754 // ignore the alloc-type mismatch and allow them to deallocate the pointer
755 // through free(), rather than the initial alloc type.
756 m->alloc_type = FROM_MALLOC;
757 return;
758 }
759
760 // Must mark the chunk as quarantined before any changes to its metadata.
761 // Do not quarantine given chunk if we failed to set CHUNK_QUARANTINE flag.
762 if (!AtomicallySetQuarantineFlagIfAllocated(m, ptr, stack))
763 return;
764
765 if (m->alloc_type != alloc_type) {
766 if (atomic_load(a: &alloc_dealloc_mismatch, mo: memory_order_acquire) &&
767 !IsAllocDeallocMismatchSuppressed(stack)) {
768 ReportAllocTypeMismatch(addr: (uptr)ptr, free_stack: stack, alloc_type: (AllocType)m->alloc_type,
769 dealloc_type: (AllocType)alloc_type);
770 }
771 } else {
772 switch (alloc_type) {
773 case FROM_NEW:
774 case FROM_NEW_BR:
775 if (flags()->new_delete_type_mismatch &&
776 ((delete_size && delete_size != m->UsedSize()) ||
777 ComputeUserRequestedAlignmentLog(user_requested_alignment: delete_alignment) !=
778 m->user_requested_alignment_log)) {
779 ReportNewDeleteTypeMismatch(addr: p, delete_size, delete_alignment,
780 free_stack: stack);
781 }
782 break;
783 case FROM_MALLOC:
784 if (flags()->free_size_mismatch &&
785 ((delete_size && delete_size != m->UsedSize()) ||
786 (delete_alignment &&
787 ComputeUserRequestedAlignmentLog(user_requested_alignment: delete_alignment) !=
788 m->user_requested_alignment_log))) {
789 ReportFreeSizeMismatch(addr: p, delete_size, delete_alignment, free_stack: stack);
790 }
791 break;
792 }
793 }
794
795 AsanStats& thread_stats = GetCurrentThreadStats();
796 thread_stats.frees++;
797 thread_stats.freed += m->UsedSize();
798
799 QuarantineChunk(m, ptr, stack);
800 }
801
802 void* Reallocate(void* old_ptr, uptr new_size, BufferedStackTrace* stack) {
803 CHECK(old_ptr && new_size);
804 uptr p = reinterpret_cast<uptr>(old_ptr);
805 uptr chunk_beg = p - kChunkHeaderSize;
806 AsanChunk* m = reinterpret_cast<AsanChunk*>(chunk_beg);
807
808 AsanStats& thread_stats = GetCurrentThreadStats();
809 thread_stats.reallocs++;
810 thread_stats.realloced += new_size;
811
812 void* new_ptr = Allocate(size: new_size, /*alignment=*/8, stack, alloc_type: FROM_MALLOC,
813 /*can_fill=*/true);
814 if (new_ptr) {
815 u8 chunk_state = atomic_load(a: &m->chunk_state, mo: memory_order_acquire);
816 if (chunk_state != CHUNK_ALLOCATED)
817 ReportInvalidFree(ptr: old_ptr, chunk_state, stack);
818 CHECK_NE(REAL(memcpy), nullptr);
819 uptr memcpy_size = Min(a: new_size, b: m->UsedSize());
820 // If realloc() races with free(), we may start copying freed memory.
821 // However, we will report racy double-free later anyway.
822 REAL(memcpy)(new_ptr, old_ptr, memcpy_size);
823 Deallocate(ptr: old_ptr, delete_size: 0, delete_alignment: 0, stack, alloc_type: FROM_MALLOC);
824 }
825 return new_ptr;
826 }
827
828 void* Calloc(uptr nmemb, uptr size, BufferedStackTrace* stack,
829 uptr align = 8) {
830 if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
831 if (AllocatorMayReturnNull())
832 return nullptr;
833 ReportCallocOverflow(count: nmemb, size, stack);
834 }
835 void* ptr = Allocate(size: nmemb * size, alignment: align, stack, alloc_type: FROM_MALLOC,
836 /*can_fill=*/false);
837 // If the memory comes from the secondary allocator no need to clear it
838 // as it comes directly from mmap.
839 if (ptr && allocator.FromPrimary(p: ptr))
840 REAL(memset)(ptr, 0, nmemb * size);
841 return ptr;
842 }
843
844 void ReportInvalidFree(void* ptr, u8 chunk_state, BufferedStackTrace* stack) {
845 if (chunk_state == CHUNK_QUARANTINE)
846 ReportDoubleFree(addr: (uptr)ptr, free_stack: stack);
847 else
848 ReportFreeNotMalloced(addr: (uptr)ptr, free_stack: stack);
849 }
850
851 void CommitBack(AsanThreadLocalMallocStorage* ms, BufferedStackTrace* stack) {
852 AllocatorCache* ac = GetAllocatorCache(ms);
853 quarantine.Drain(c: GetQuarantineCache(ms), cb: QuarantineCallback(ac, stack));
854 allocator.SwallowCache(cache: ac);
855 }
856
857 // -------------------------- Chunk lookup ----------------------
858
859 // Assumes alloc_beg == allocator.GetBlockBegin(alloc_beg).
860 // Returns nullptr if AsanChunk is not yet initialized just after
861 // get_allocator().Allocate(), or is being destroyed just before
862 // get_allocator().Deallocate().
863 AsanChunk* GetAsanChunk(void* alloc_beg) {
864 if (!alloc_beg)
865 return nullptr;
866 AsanChunk* p = reinterpret_cast<LargeChunkHeader*>(alloc_beg)->Get();
867 if (!p) {
868 if (!allocator.FromPrimary(p: alloc_beg))
869 return nullptr;
870 p = reinterpret_cast<AsanChunk*>(alloc_beg);
871 }
872 u8 state = atomic_load(a: &p->chunk_state, mo: memory_order_relaxed);
873 // It does not guaranty that Chunk is initialized, but it's
874 // definitely not for any other value.
875 if (state == CHUNK_ALLOCATED || state == CHUNK_QUARANTINE)
876 return p;
877 return nullptr;
878 }
879
880 AsanChunk* GetAsanChunkByAddr(uptr p) {
881 void* alloc_beg = allocator.GetBlockBegin(p: reinterpret_cast<void*>(p));
882 return GetAsanChunk(alloc_beg);
883 }
884
885 // Allocator must be locked when this function is called.
886 AsanChunk* GetAsanChunkByAddrFastLocked(uptr p) {
887 void* alloc_beg =
888 allocator.GetBlockBeginFastLocked(p: reinterpret_cast<void*>(p));
889 return GetAsanChunk(alloc_beg);
890 }
891
892 uptr AllocationSize(uptr p) {
893 AsanChunk* m = GetAsanChunkByAddr(p);
894 if (!m)
895 return 0;
896 if (atomic_load(a: &m->chunk_state, mo: memory_order_acquire) != CHUNK_ALLOCATED)
897 return 0;
898 if (m->Beg() != p)
899 return 0;
900 return m->UsedSize();
901 }
902
903#if SANITIZER_WINDOWS
904 // Returns true if the allocation at p was a zero-size request that was
905 // internally upgraded to size 1.
906 bool FromZeroAllocation(uptr p) {
907 return reinterpret_cast<AsanChunk*>(p - kChunkHeaderSize)->from_zero_alloc;
908 }
909
910 // Marks an existing size 1 allocation as having originally been zero-size.
911 // Used by SharedReAlloc which augments size 0 to 1 before calling
912 // asan_realloc, bypassing Allocate's own zero-size tracking.
913 void MarkAsZeroAllocation(uptr p) {
914 AsanChunk* m = reinterpret_cast<AsanChunk*>(p - kChunkHeaderSize);
915 m->from_zero_alloc = 1;
916 PoisonShadow(p, ASAN_SHADOW_GRANULARITY, kAsanHeapLeftRedzoneMagic);
917 }
918#endif
919
920 uptr AllocationSizeFast(uptr p) {
921 return reinterpret_cast<AsanChunk*>(p - kChunkHeaderSize)->UsedSize();
922 }
923
924 AsanChunkView FindHeapChunkByAddress(uptr addr) {
925 AsanChunk* m1 = GetAsanChunkByAddr(p: addr);
926 sptr offset = 0;
927 if (!m1 || AsanChunkView(m1).AddrIsAtLeft(addr, access_size: 1, offset: &offset)) {
928 // The address is in the chunk's left redzone, so maybe it is actually
929 // a right buffer overflow from the other chunk before.
930 // Search a bit before to see if there is another chunk.
931 AsanChunk* m2 = nullptr;
932 for (uptr l = 1; l < GetPageSizeCached(); l++) {
933 m2 = GetAsanChunkByAddr(p: addr - l);
934 if (m2 == m1)
935 continue; // Still the same chunk.
936 break;
937 }
938 if (m2 && AsanChunkView(m2).AddrIsAtRight(addr, access_size: 1, offset: &offset))
939 m1 = ChooseChunk(addr, left_chunk: m2, right_chunk: m1);
940 }
941 return AsanChunkView(m1);
942 }
943
944 void Purge(BufferedStackTrace* stack) {
945 AsanThread* t = GetCurrentThread();
946 if (t) {
947 AsanThreadLocalMallocStorage* ms = &t->malloc_storage();
948 quarantine.DrainAndRecycle(
949 c: GetQuarantineCache(ms),
950 cb: QuarantineCallback(GetAllocatorCache(ms), stack));
951 }
952 {
953 SpinMutexLock l(&fallback_mutex);
954 quarantine.DrainAndRecycle(
955 c: &fallback_quarantine_cache,
956 cb: QuarantineCallback(&fallback_allocator_cache, stack));
957 }
958
959 allocator.ForceReleaseToOS();
960 }
961
962 void PrintStats() {
963 allocator.PrintStats();
964 quarantine.PrintStats();
965 }
966
967 void ForceLock() SANITIZER_ACQUIRE(fallback_mutex) {
968 allocator.ForceLock();
969 fallback_mutex.Lock();
970 }
971
972 void ForceUnlock() SANITIZER_RELEASE(fallback_mutex) {
973 fallback_mutex.Unlock();
974 allocator.ForceUnlock();
975 }
976};
977
978static Allocator instance(LINKER_INITIALIZED);
979
980static AsanAllocator& get_allocator() { return instance.allocator; }
981
982bool AsanChunkView::IsValid() const {
983 return chunk_ && atomic_load(a: &chunk_->chunk_state, mo: memory_order_relaxed) !=
984 CHUNK_INVALID;
985}
986bool AsanChunkView::IsAllocated() const {
987 return chunk_ && atomic_load(a: &chunk_->chunk_state, mo: memory_order_relaxed) ==
988 CHUNK_ALLOCATED;
989}
990bool AsanChunkView::IsQuarantined() const {
991 return chunk_ && atomic_load(a: &chunk_->chunk_state, mo: memory_order_relaxed) ==
992 CHUNK_QUARANTINE;
993}
994uptr AsanChunkView::Beg() const { return chunk_->Beg(); }
995uptr AsanChunkView::End() const { return Beg() + UsedSize(); }
996uptr AsanChunkView::UsedSize() const { return chunk_->UsedSize(); }
997u32 AsanChunkView::UserRequestedAlignment() const {
998 return Allocator::ComputeUserAlignment(user_requested_alignment_log: chunk_->user_requested_alignment_log);
999}
1000
1001uptr AsanChunkView::AllocTid() const {
1002 u32 tid = 0;
1003 u32 stack = 0;
1004 chunk_->GetAllocContext(tid, stack);
1005 return tid;
1006}
1007
1008uptr AsanChunkView::FreeTid() const {
1009 if (!IsQuarantined())
1010 return kInvalidTid;
1011 u32 tid = 0;
1012 u32 stack = 0;
1013 chunk_->GetFreeContext(tid, stack);
1014 return tid;
1015}
1016
1017AllocType AsanChunkView::GetAllocType() const {
1018 return (AllocType)chunk_->alloc_type;
1019}
1020
1021u32 AsanChunkView::GetAllocStackId() const {
1022 u32 tid = 0;
1023 u32 stack = 0;
1024 chunk_->GetAllocContext(tid, stack);
1025 return stack;
1026}
1027
1028u32 AsanChunkView::GetFreeStackId() const {
1029 if (!IsQuarantined())
1030 return 0;
1031 u32 tid = 0;
1032 u32 stack = 0;
1033 chunk_->GetFreeContext(tid, stack);
1034 return stack;
1035}
1036
1037void InitializeAllocator(const AllocatorOptions& options) {
1038 instance.InitLinkerInitialized(options);
1039}
1040
1041void ReInitializeAllocator(const AllocatorOptions& options) {
1042 instance.ReInitialize(options);
1043}
1044
1045// Apply provided AllocatorOptions to an Allocator
1046void ApplyAllocatorOptions(const AllocatorOptions& options) {
1047 instance.ApplyOptions(options);
1048}
1049
1050void GetAllocatorOptions(AllocatorOptions* options) {
1051 instance.GetOptions(options);
1052}
1053
1054AsanChunkView FindHeapChunkByAddress(uptr addr) {
1055 return instance.FindHeapChunkByAddress(addr);
1056}
1057AsanChunkView FindHeapChunkByAllocBeg(uptr addr) {
1058 return AsanChunkView(instance.GetAsanChunk(alloc_beg: reinterpret_cast<void*>(addr)));
1059}
1060
1061void AsanThreadLocalMallocStorage::CommitBack() {
1062 GET_STACK_TRACE_MALLOC;
1063 instance.CommitBack(ms: this, stack: &stack);
1064}
1065
1066void PrintInternalAllocatorStats() { instance.PrintStats(); }
1067
1068void asan_free(void* ptr, BufferedStackTrace* stack) {
1069 instance.Deallocate(ptr, delete_size: 0, delete_alignment: 0, stack, alloc_type: FROM_MALLOC);
1070}
1071
1072void asan_free_sized(void* ptr, uptr size, BufferedStackTrace* stack) {
1073 instance.Deallocate(ptr, delete_size: size, /*delete_alignment=*/0, stack, alloc_type: FROM_MALLOC);
1074}
1075
1076void asan_free_aligned_sized(void* ptr, uptr alignment, uptr size,
1077 BufferedStackTrace* stack) {
1078 instance.Deallocate(ptr, delete_size: size, delete_alignment: alignment, stack, alloc_type: FROM_MALLOC);
1079}
1080
1081void* asan_malloc(uptr size, BufferedStackTrace* stack) {
1082 return SetErrnoOnNull(instance.Allocate(size, /*alignment=*/8, stack,
1083 alloc_type: FROM_MALLOC, /*can_fill=*/true));
1084}
1085
1086void* asan_calloc(uptr nmemb, uptr size, BufferedStackTrace* stack) {
1087 return SetErrnoOnNull(instance.Calloc(nmemb, size, stack));
1088}
1089
1090#if SANITIZER_AIX
1091void* asan_vec_malloc(uptr size, BufferedStackTrace* stack) {
1092 return SetErrnoOnNull(instance.Allocate(size, /*alignment=*/16, stack,
1093 FROM_MALLOC, /*can_fill=*/true));
1094}
1095
1096void* asan_vec_calloc(uptr nmemb, uptr size, BufferedStackTrace* stack) {
1097 return SetErrnoOnNull(instance.Calloc(nmemb, size, stack, 16));
1098}
1099#endif
1100
1101void* asan_reallocarray(void* p, uptr nmemb, uptr size,
1102 BufferedStackTrace* stack) {
1103 if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
1104 errno = errno_ENOMEM;
1105 if (AllocatorMayReturnNull())
1106 return nullptr;
1107 ReportReallocArrayOverflow(count: nmemb, size, stack);
1108 }
1109 return asan_realloc(p, size: nmemb * size, stack);
1110}
1111
1112void* asan_realloc(void* p, uptr size, BufferedStackTrace* stack) {
1113 if (!p)
1114 return SetErrnoOnNull(instance.Allocate(size, /*alignment=*/8, stack,
1115 alloc_type: FROM_MALLOC, /*can_fill=*/true));
1116 if (size == 0) {
1117 if (flags()->allocator_frees_and_returns_null_on_realloc_zero) {
1118 instance.Deallocate(ptr: p, delete_size: 0, delete_alignment: 0, stack, alloc_type: FROM_MALLOC);
1119 return nullptr;
1120 }
1121 // Allocate a size of 1 if we shouldn't free() on Realloc to 0
1122 size = 1;
1123 }
1124 return SetErrnoOnNull(instance.Reallocate(old_ptr: p, new_size: size, stack));
1125}
1126
1127void* asan_valloc(uptr size, BufferedStackTrace* stack) {
1128 return SetErrnoOnNull(instance.Allocate(size, alignment: GetPageSizeCached(), stack,
1129 alloc_type: FROM_MALLOC, /*can_fill=*/true));
1130}
1131
1132void* asan_pvalloc(uptr size, BufferedStackTrace* stack) {
1133 uptr PageSize = GetPageSizeCached();
1134 if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) {
1135 errno = errno_ENOMEM;
1136 if (AllocatorMayReturnNull())
1137 return nullptr;
1138 ReportPvallocOverflow(size, stack);
1139 }
1140 // pvalloc(0) should allocate one page.
1141 size = size ? RoundUpTo(size, boundary: PageSize) : PageSize;
1142 return SetErrnoOnNull(
1143 instance.Allocate(size, alignment: PageSize, stack, alloc_type: FROM_MALLOC, /*can_fill=*/true));
1144}
1145
1146void* asan_memalign(uptr alignment, uptr size, BufferedStackTrace* stack) {
1147 if (UNLIKELY(!IsPowerOfTwo(alignment))) {
1148 errno = errno_EINVAL;
1149 if (AllocatorMayReturnNull())
1150 return nullptr;
1151 ReportInvalidAllocationAlignment(alignment, stack);
1152 }
1153 return SetErrnoOnNull(instance.Allocate(size, alignment, stack, alloc_type: FROM_MALLOC,
1154 /*can_fill=*/true));
1155}
1156
1157void* asan_aligned_alloc(uptr alignment, uptr size, BufferedStackTrace* stack) {
1158 if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) {
1159 errno = errno_EINVAL;
1160 if (AllocatorMayReturnNull())
1161 return nullptr;
1162 ReportInvalidAlignedAllocAlignment(size, alignment, stack);
1163 }
1164 return SetErrnoOnNull(instance.Allocate(size, alignment, stack, alloc_type: FROM_MALLOC,
1165 /*can_fill=*/true));
1166}
1167
1168int asan_posix_memalign(void** memptr, uptr alignment, uptr size,
1169 BufferedStackTrace* stack) {
1170 if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) {
1171 if (AllocatorMayReturnNull())
1172 return errno_EINVAL;
1173 ReportInvalidPosixMemalignAlignment(alignment, stack);
1174 }
1175 void* ptr = instance.Allocate(size, alignment, stack, alloc_type: FROM_MALLOC,
1176 /*can_fill=*/true);
1177 if (UNLIKELY(!ptr))
1178 // OOM error is already taken care of by Allocate.
1179 return errno_ENOMEM;
1180 CHECK(IsAligned((uptr)ptr, alignment));
1181 *memptr = ptr;
1182 return 0;
1183}
1184
1185uptr asan_malloc_usable_size(const void* ptr, uptr pc, uptr bp) {
1186 if (!ptr)
1187 return 0;
1188 uptr usable_size = instance.AllocationSize(p: reinterpret_cast<uptr>(ptr));
1189 if (flags()->check_malloc_usable_size && (usable_size == 0)) {
1190 GET_STACK_TRACE_FATAL(pc, bp);
1191 ReportMallocUsableSizeNotOwned(addr: (uptr)ptr, stack: &stack);
1192 }
1193#if SANITIZER_WINDOWS
1194 // Zero-size allocations are internally upgraded to size 1 so that
1195 // malloc(0)/new(0) return unique non-NULL pointers as required by the
1196 // standard. Windows heap APIs (HeapSize, RtlSizeHeap, _msize) should still
1197 // report the originally requested size (0).
1198 if (usable_size > 0 &&
1199 instance.FromZeroAllocation(reinterpret_cast<uptr>(ptr))) {
1200 DCHECK(usable_size == 1);
1201 return 0;
1202 }
1203#endif
1204 return usable_size;
1205}
1206
1207namespace {
1208
1209void* asan_new(uptr size, BufferedStackTrace* stack, bool array) {
1210 return SetErrnoOnNull(instance.Allocate(size, /*alignment=*/0, stack,
1211 alloc_type: array ? FROM_NEW_BR : FROM_NEW,
1212 /*can_fill=*/true));
1213}
1214
1215void* asan_new_aligned(uptr size, uptr alignment, BufferedStackTrace* stack,
1216 bool array) {
1217 if (UNLIKELY(alignment == 0 || !IsPowerOfTwo(alignment))) {
1218 errno = errno_EINVAL;
1219 if (AllocatorMayReturnNull())
1220 return nullptr;
1221 ReportInvalidAllocationAlignment(alignment, stack);
1222 }
1223 return SetErrnoOnNull(instance.Allocate(size, alignment, stack,
1224 alloc_type: array ? FROM_NEW_BR : FROM_NEW,
1225 /*can_fill=*/true));
1226}
1227
1228void asan_delete(void* ptr, BufferedStackTrace* stack, bool array) {
1229 instance.Deallocate(ptr, delete_size: 0, delete_alignment: 0, stack, alloc_type: array ? FROM_NEW_BR : FROM_NEW);
1230}
1231
1232void asan_delete_aligned(void* ptr, uptr alignment, BufferedStackTrace* stack,
1233 bool array) {
1234 instance.Deallocate(ptr, delete_size: 0, delete_alignment: alignment, stack, alloc_type: array ? FROM_NEW_BR : FROM_NEW);
1235}
1236
1237void asan_delete_sized(void* ptr, uptr size, BufferedStackTrace* stack,
1238 bool array) {
1239 instance.Deallocate(ptr, delete_size: size, delete_alignment: 0, stack, alloc_type: array ? FROM_NEW_BR : FROM_NEW);
1240}
1241
1242void asan_delete_sized_aligned(void* ptr, uptr size, uptr alignment,
1243 BufferedStackTrace* stack, bool array) {
1244 instance.Deallocate(ptr, delete_size: size, delete_alignment: alignment, stack,
1245 alloc_type: array ? FROM_NEW_BR : FROM_NEW);
1246}
1247
1248} // namespace
1249
1250void* asan_new(uptr size, BufferedStackTrace* stack) {
1251 return asan_new(size, stack, /*array=*/false);
1252}
1253
1254void* asan_new_aligned(uptr size, uptr alignment, BufferedStackTrace* stack) {
1255 return asan_new_aligned(size, alignment, stack, /*array=*/false);
1256}
1257
1258void* asan_new_array(uptr size, BufferedStackTrace* stack) {
1259 return asan_new(size, stack, /*array=*/true);
1260}
1261
1262void* asan_new_array_aligned(uptr size, uptr alignment,
1263 BufferedStackTrace* stack) {
1264 return asan_new_aligned(size, alignment, stack, /*array=*/true);
1265}
1266
1267void asan_delete(void* ptr, BufferedStackTrace* stack) {
1268 asan_delete(ptr, stack, /*array=*/false);
1269}
1270
1271void asan_delete_aligned(void* ptr, uptr alignment, BufferedStackTrace* stack) {
1272 asan_delete_aligned(ptr, alignment, stack, /*array=*/false);
1273}
1274
1275void asan_delete_sized(void* ptr, uptr size, BufferedStackTrace* stack) {
1276 asan_delete_sized(ptr, size, stack, /*array=*/false);
1277}
1278
1279void asan_delete_sized_aligned(void* ptr, uptr size, uptr alignment,
1280 BufferedStackTrace* stack) {
1281 asan_delete_sized_aligned(ptr, size, alignment, stack, /*array=*/false);
1282}
1283
1284void asan_delete_array(void* ptr, BufferedStackTrace* stack) {
1285 asan_delete(ptr, stack, /*array=*/true);
1286}
1287
1288void asan_delete_array_aligned(void* ptr, uptr alignment,
1289 BufferedStackTrace* stack) {
1290 asan_delete_aligned(ptr, alignment, stack, /*array=*/true);
1291}
1292
1293void asan_delete_array_sized(void* ptr, uptr size, BufferedStackTrace* stack) {
1294 asan_delete_sized(ptr, size, stack, /*array=*/true);
1295}
1296
1297void asan_delete_array_sized_aligned(void* ptr, uptr size, uptr alignment,
1298 BufferedStackTrace* stack) {
1299 asan_delete_sized_aligned(ptr, size, alignment, stack, /*array=*/true);
1300}
1301
1302uptr asan_mz_size(const void* ptr) {
1303 uptr size = instance.AllocationSize(p: reinterpret_cast<uptr>(ptr));
1304
1305#if SANITIZER_WINDOWS
1306 if (size > 0 && instance.FromZeroAllocation(reinterpret_cast<uptr>(ptr))) {
1307 DCHECK(size == 1);
1308 return 0;
1309 }
1310#endif
1311
1312 return size;
1313}
1314
1315#if SANITIZER_WINDOWS
1316void asan_mark_zero_allocation(void* ptr) {
1317 instance.MarkAsZeroAllocation(reinterpret_cast<uptr>(ptr));
1318}
1319#endif
1320
1321void asan_mz_force_lock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
1322 instance.ForceLock();
1323}
1324
1325void asan_mz_force_unlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
1326 instance.ForceUnlock();
1327}
1328
1329} // namespace __asan
1330
1331// --- Implementation of LSan-specific functions --- {{{1
1332namespace __lsan {
1333void LockAllocator() { __asan::get_allocator().ForceLock(); }
1334
1335void UnlockAllocator() { __asan::get_allocator().ForceUnlock(); }
1336
1337void GetAllocatorGlobalRange(uptr* begin, uptr* end) {
1338 *begin = (uptr)&__asan::get_allocator();
1339 *end = *begin + sizeof(__asan::get_allocator());
1340}
1341
1342uptr PointsIntoChunk(void* p) {
1343 uptr addr = reinterpret_cast<uptr>(p);
1344 __asan::AsanChunk* m = __asan::instance.GetAsanChunkByAddrFastLocked(p: addr);
1345 if (!m || atomic_load(a: &m->chunk_state, mo: memory_order_acquire) !=
1346 __asan::CHUNK_ALLOCATED)
1347 return 0;
1348 uptr chunk = m->Beg();
1349 if (m->AddrIsInside(addr))
1350 return chunk;
1351 if (IsSpecialCaseOfOperatorNew0(chunk_beg: chunk, chunk_size: m->UsedSize(), addr))
1352 return chunk;
1353 return 0;
1354}
1355
1356uptr GetUserBegin(uptr chunk) {
1357 // FIXME: All usecases provide chunk address, GetAsanChunkByAddrFastLocked is
1358 // not needed.
1359 __asan::AsanChunk* m = __asan::instance.GetAsanChunkByAddrFastLocked(p: chunk);
1360 return m ? m->Beg() : 0;
1361}
1362
1363uptr GetUserAddr(uptr chunk) { return chunk; }
1364
1365LsanMetadata::LsanMetadata(uptr chunk) {
1366 metadata_ = chunk ? reinterpret_cast<void*>(chunk - __asan::kChunkHeaderSize)
1367 : nullptr;
1368}
1369
1370bool LsanMetadata::allocated() const {
1371 if (!metadata_)
1372 return false;
1373 __asan::AsanChunk* m = reinterpret_cast<__asan::AsanChunk*>(metadata_);
1374 return atomic_load(a: &m->chunk_state, mo: memory_order_relaxed) ==
1375 __asan::CHUNK_ALLOCATED;
1376}
1377
1378ChunkTag LsanMetadata::tag() const {
1379 __asan::AsanChunk* m = reinterpret_cast<__asan::AsanChunk*>(metadata_);
1380 return static_cast<ChunkTag>(m->lsan_tag);
1381}
1382
1383void LsanMetadata::set_tag(ChunkTag value) {
1384 __asan::AsanChunk* m = reinterpret_cast<__asan::AsanChunk*>(metadata_);
1385 m->lsan_tag = value;
1386}
1387
1388uptr LsanMetadata::requested_size() const {
1389 __asan::AsanChunk* m = reinterpret_cast<__asan::AsanChunk*>(metadata_);
1390 return m->UsedSize();
1391}
1392
1393u32 LsanMetadata::stack_trace_id() const {
1394 __asan::AsanChunk* m = reinterpret_cast<__asan::AsanChunk*>(metadata_);
1395 u32 tid = 0;
1396 u32 stack = 0;
1397 m->GetAllocContext(tid, stack);
1398 return stack;
1399}
1400
1401void ForEachChunk(ForEachChunkCallback callback, void* arg) {
1402 __asan::get_allocator().ForEachChunk(callback, arg);
1403}
1404
1405IgnoreObjectResult IgnoreObject(const void* p) {
1406 uptr addr = reinterpret_cast<uptr>(p);
1407 __asan::AsanChunk* m = __asan::instance.GetAsanChunkByAddr(p: addr);
1408 if (!m ||
1409 (atomic_load(a: &m->chunk_state, mo: memory_order_acquire) !=
1410 __asan::CHUNK_ALLOCATED) ||
1411 !m->AddrIsInside(addr)) {
1412 return kIgnoreObjectInvalid;
1413 }
1414 if (m->lsan_tag == kIgnored)
1415 return kIgnoreObjectAlreadyIgnored;
1416 m->lsan_tag = __lsan::kIgnored;
1417 return kIgnoreObjectSuccess;
1418}
1419
1420} // namespace __lsan
1421
1422// ---------------------- Interface ---------------- {{{1
1423using namespace __asan;
1424
1425static const void* AllocationBegin(const void* p) {
1426 AsanChunk* m = __asan::instance.GetAsanChunkByAddr(p: (uptr)p);
1427 if (!m)
1428 return nullptr;
1429 if (atomic_load(a: &m->chunk_state, mo: memory_order_acquire) != CHUNK_ALLOCATED)
1430 return nullptr;
1431 if (m->UsedSize() == 0)
1432 return nullptr;
1433 return (const void*)(m->Beg());
1434}
1435
1436// ASan allocator doesn't reserve extra bytes, so normally we would
1437// just return "size". We don't want to expose our redzone sizes, etc here.
1438uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; }
1439
1440int __sanitizer_get_ownership(const void* p) {
1441 uptr ptr = reinterpret_cast<uptr>(p);
1442 return instance.AllocationSize(p: ptr) > 0;
1443}
1444
1445uptr __sanitizer_get_allocated_size(const void* p) {
1446 if (!p)
1447 return 0;
1448 uptr ptr = reinterpret_cast<uptr>(p);
1449 uptr allocated_size = instance.AllocationSize(p: ptr);
1450 // Die if p is not malloced or if it is already freed.
1451 if (allocated_size == 0) {
1452 GET_STACK_TRACE_FATAL_HERE;
1453 ReportSanitizerGetAllocatedSizeNotOwned(addr: ptr, stack: &stack);
1454 }
1455 return allocated_size;
1456}
1457
1458uptr __sanitizer_get_allocated_size_fast(const void* p) {
1459 DCHECK_EQ(p, __sanitizer_get_allocated_begin(p));
1460 uptr ret = instance.AllocationSizeFast(p: reinterpret_cast<uptr>(p));
1461 DCHECK_EQ(ret, __sanitizer_get_allocated_size(p));
1462 return ret;
1463}
1464
1465const void* __sanitizer_get_allocated_begin(const void* p) {
1466 return AllocationBegin(p);
1467}
1468
1469void __sanitizer_purge_allocator() {
1470 GET_STACK_TRACE_MALLOC;
1471 instance.Purge(stack: &stack);
1472}
1473
1474int __asan_update_allocation_context(void* addr) {
1475 GET_STACK_TRACE_MALLOC;
1476 return instance.UpdateAllocationStack(addr: (uptr)addr, stack: &stack);
1477}
1478