1//===-- memprof_allocator.cpp --------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is a part of MemProfiler, a memory profiler.
10//
11// Implementation of MemProf's memory allocator, which uses the allocator
12// from sanitizer_common.
13//
14//===----------------------------------------------------------------------===//
15
16#include "memprof_allocator.h"
17#include "memprof_mapping.h"
18#include "memprof_mibmap.h"
19#include "memprof_rawprofile.h"
20#include "memprof_stack.h"
21#include "memprof_thread.h"
22#include "profile/MemProfData.inc"
23#include "sanitizer_common/sanitizer_allocator_checks.h"
24#include "sanitizer_common/sanitizer_allocator_interface.h"
25#include "sanitizer_common/sanitizer_allocator_report.h"
26#include "sanitizer_common/sanitizer_array_ref.h"
27#include "sanitizer_common/sanitizer_common.h"
28#include "sanitizer_common/sanitizer_errno.h"
29#include "sanitizer_common/sanitizer_file.h"
30#include "sanitizer_common/sanitizer_flags.h"
31#include "sanitizer_common/sanitizer_internal_defs.h"
32#include "sanitizer_common/sanitizer_stackdepot.h"
33
34#include <sched.h>
35#include <time.h>
36
37#define MAX_HISTOGRAM_PRINT_SIZE 32U
38
39extern bool __memprof_histogram;
40
41namespace __memprof {
42namespace {
43using ::llvm::memprof::MemInfoBlock;
44
45void Print(const MemInfoBlock &M, const u64 id, bool print_terse) {
46 u64 p;
47
48 if (print_terse) {
49 p = M.TotalSize * 100 / M.AllocCount;
50 Printf(format: "MIB:%llu/%u/%llu.%02llu/%u/%u/", id, M.AllocCount, p / 100, p % 100,
51 M.MinSize, M.MaxSize);
52 p = M.TotalAccessCount * 100 / M.AllocCount;
53 Printf(format: "%llu.%02llu/%llu/%llu/", p / 100, p % 100, M.MinAccessCount,
54 M.MaxAccessCount);
55 p = M.TotalLifetime * 100 / M.AllocCount;
56 Printf(format: "%llu.%02llu/%u/%u/", p / 100, p % 100, M.MinLifetime,
57 M.MaxLifetime);
58 Printf(format: "%u/%u/%u/%u\n", M.NumMigratedCpu, M.NumLifetimeOverlaps,
59 M.NumSameAllocCpu, M.NumSameDeallocCpu);
60 } else {
61 p = M.TotalSize * 100 / M.AllocCount;
62 Printf(format: "Memory allocation stack id = %llu\n", id);
63 Printf(format: "\talloc_count %u, size (ave/min/max) %llu.%02llu / %u / %u\n",
64 M.AllocCount, p / 100, p % 100, M.MinSize, M.MaxSize);
65 p = M.TotalAccessCount * 100 / M.AllocCount;
66 Printf(format: "\taccess_count (ave/min/max): %llu.%02llu / %llu / %llu\n", p / 100,
67 p % 100, M.MinAccessCount, M.MaxAccessCount);
68 p = M.TotalLifetime * 100 / M.AllocCount;
69 Printf(format: "\tlifetime (ave/min/max): %llu.%02llu / %u / %u\n", p / 100,
70 p % 100, M.MinLifetime, M.MaxLifetime);
71 Printf(format: "\tnum migrated: %u, num lifetime overlaps: %u, num same alloc "
72 "cpu: %u, num same dealloc_cpu: %u\n",
73 M.NumMigratedCpu, M.NumLifetimeOverlaps, M.NumSameAllocCpu,
74 M.NumSameDeallocCpu);
75 Printf(format: "AccessCountHistogram[%u]: ", M.AccessHistogramSize);
76 uint32_t PrintSize = M.AccessHistogramSize > MAX_HISTOGRAM_PRINT_SIZE
77 ? MAX_HISTOGRAM_PRINT_SIZE
78 : M.AccessHistogramSize;
79 for (size_t i = 0; i < PrintSize; ++i) {
80 Printf(format: "%llu ", ((uint64_t *)M.AccessHistogram)[i]);
81 }
82 Printf(format: "\n");
83 }
84}
85} // namespace
86
87static int GetCpuId(void) {
88 // _memprof_preinit is called via the preinit_array, which subsequently calls
89 // malloc. Since this is before _dl_init calls VDSO_SETUP, sched_getcpu
90 // will seg fault as the address of __vdso_getcpu will be null.
91 if (!memprof_inited)
92 return -1;
93 return sched_getcpu();
94}
95
96// Compute the timestamp in ms.
97static int GetTimestamp(void) {
98 // timespec_get will segfault if called from dl_init
99 if (!memprof_timestamp_inited) {
100 // By returning 0, this will be effectively treated as being
101 // timestamped at memprof init time (when memprof_init_timestamp_s
102 // is initialized).
103 return 0;
104 }
105 timespec ts;
106 clock_gettime(CLOCK_REALTIME, tp: &ts);
107 return (ts.tv_sec - memprof_init_timestamp_s) * 1000 + ts.tv_nsec / 1000000;
108}
109
110static MemprofAllocator &get_allocator();
111
112// The memory chunk allocated from the underlying allocator looks like this:
113// H H U U U U U U
114// H -- ChunkHeader (32 bytes)
115// U -- user memory.
116
117// If there is left padding before the ChunkHeader (due to use of memalign),
118// we store a magic value in the first uptr word of the memory block and
119// store the address of ChunkHeader in the next uptr.
120// M B L L L L L L L L L H H U U U U U U
121// | ^
122// ---------------------|
123// M -- magic value kAllocBegMagic
124// B -- address of ChunkHeader pointing to the first 'H'
125
126constexpr uptr kMaxAllowedMallocBits = 40;
127
128// Should be no more than 32-bytes
129struct ChunkHeader {
130 // 1-st 4 bytes.
131 u32 alloc_context_id;
132 // 2-nd 4 bytes
133 u32 cpu_id;
134 // 3-rd 4 bytes
135 u32 timestamp_ms;
136 // 4-th 4 bytes
137 // Note only 1 bit is needed for this flag if we need space in the future for
138 // more fields.
139 u32 from_memalign;
140 // 5-th and 6-th 4 bytes
141 // The max size of an allocation is 2^40 (kMaxAllowedMallocSize), so this
142 // could be shrunk to kMaxAllowedMallocBits if we need space in the future for
143 // more fields.
144 atomic_uint64_t user_requested_size;
145 // 23 bits available
146 // 7-th and 8-th 4 bytes
147 u64 data_type_id; // TODO: hash of type name
148};
149
150static const uptr kChunkHeaderSize = sizeof(ChunkHeader);
151COMPILER_CHECK(kChunkHeaderSize == 32);
152
153struct MemprofChunk : ChunkHeader {
154 uptr Beg() { return reinterpret_cast<uptr>(this) + kChunkHeaderSize; }
155 uptr UsedSize() {
156 return atomic_load(a: &user_requested_size, mo: memory_order_relaxed);
157 }
158 void *AllocBeg() {
159 if (from_memalign)
160 return get_allocator().GetBlockBegin(p: reinterpret_cast<void *>(this));
161 return reinterpret_cast<void *>(this);
162 }
163};
164
165class LargeChunkHeader {
166 static constexpr uptr kAllocBegMagic =
167 FIRST_32_SECOND_64(0xCC6E96B9, 0xCC6E96B9CC6E96B9ULL);
168 atomic_uintptr_t magic;
169 MemprofChunk *chunk_header;
170
171public:
172 MemprofChunk *Get() const {
173 return atomic_load(a: &magic, mo: memory_order_acquire) == kAllocBegMagic
174 ? chunk_header
175 : nullptr;
176 }
177
178 void Set(MemprofChunk *p) {
179 if (p) {
180 chunk_header = p;
181 atomic_store(a: &magic, v: kAllocBegMagic, mo: memory_order_release);
182 return;
183 }
184
185 uptr old = kAllocBegMagic;
186 if (!atomic_compare_exchange_strong(a: &magic, cmp: &old, xchg: 0,
187 mo: memory_order_release)) {
188 CHECK_EQ(old, kAllocBegMagic);
189 }
190 }
191};
192
193void FlushUnneededMemProfShadowMemory(uptr p, uptr size) {
194 // Since memprof's mapping is compacting, the shadow chunk may be
195 // not page-aligned, so we only flush the page-aligned portion.
196 ReleaseMemoryPagesToOS(beg: MemToShadow(p), end: MemToShadow(p: p + size));
197}
198
199void MemprofMapUnmapCallback::OnMap(uptr p, uptr size) const {
200 // Statistics.
201 MemprofStats &thread_stats = GetCurrentThreadStats();
202 thread_stats.mmaps++;
203 thread_stats.mmaped += size;
204}
205
206void MemprofMapUnmapCallback::OnUnmap(uptr p, uptr size) const {
207 // We are about to unmap a chunk of user memory.
208 // Mark the corresponding shadow memory as not needed.
209 FlushUnneededMemProfShadowMemory(p, size);
210 // Statistics.
211 MemprofStats &thread_stats = GetCurrentThreadStats();
212 thread_stats.munmaps++;
213 thread_stats.munmaped += size;
214}
215
216AllocatorCache *GetAllocatorCache(MemprofThreadLocalMallocStorage *ms) {
217 CHECK(ms);
218 return &ms->allocator_cache;
219}
220
221// Clears the shadow counters (when memory is allocated).
222void ClearShadow(uptr addr, uptr size) {
223 CHECK(AddrIsAlignedByGranularity(addr));
224 CHECK(AddrIsInMem(addr));
225 CHECK(AddrIsAlignedByGranularity(addr + size));
226 CHECK(AddrIsInMem(addr + size - SHADOW_GRANULARITY));
227 CHECK(REAL(memset));
228 uptr shadow_beg;
229 uptr shadow_end;
230 if (__memprof_histogram) {
231 shadow_beg = HISTOGRAM_MEM_TO_SHADOW(addr);
232 shadow_end = HISTOGRAM_MEM_TO_SHADOW(addr + size);
233 } else {
234 shadow_beg = MEM_TO_SHADOW(addr);
235 shadow_end = MEM_TO_SHADOW(addr + size - SHADOW_GRANULARITY) + 1;
236 }
237
238 if (shadow_end - shadow_beg < common_flags()->clear_shadow_mmap_threshold) {
239 REAL(memset)((void *)shadow_beg, 0, shadow_end - shadow_beg);
240 } else {
241 uptr page_size = GetPageSizeCached();
242 uptr page_beg = RoundUpTo(size: shadow_beg, boundary: page_size);
243 uptr page_end = RoundDownTo(x: shadow_end, boundary: page_size);
244
245 if (page_beg >= page_end) {
246 REAL(memset)((void *)shadow_beg, 0, shadow_end - shadow_beg);
247 } else {
248 if (page_beg != shadow_beg) {
249 REAL(memset)((void *)shadow_beg, 0, page_beg - shadow_beg);
250 }
251 if (page_end != shadow_end) {
252 REAL(memset)((void *)page_end, 0, shadow_end - page_end);
253 }
254 ReserveShadowMemoryRange(beg: page_beg, end: page_end - 1, name: nullptr);
255 }
256 }
257}
258
259struct Allocator {
260 static const uptr kMaxAllowedMallocSize = 1ULL << kMaxAllowedMallocBits;
261
262 MemprofAllocator allocator;
263 StaticSpinMutex fallback_mutex;
264 AllocatorCache fallback_allocator_cache;
265
266 uptr max_user_defined_malloc_size;
267
268 // Holds the mapping of stack ids to MemInfoBlocks.
269 MIBMapTy MIBMap;
270
271 atomic_uint8_t destructing;
272 atomic_uint8_t constructed;
273
274 // ------------------- Initialization ------------------------
275 explicit Allocator(LinkerInitialized) {
276 atomic_store_relaxed(a: &destructing, v: 0);
277 atomic_store_relaxed(a: &constructed, v: 1);
278 }
279
280 ~Allocator() {
281 atomic_store_relaxed(a: &destructing, v: 1);
282 if (flags()->dump_at_exit)
283 FinishAndWrite();
284 }
285
286 static void PrintCallback(const uptr Key, LockedMemInfoBlock *const &Value,
287 void *Arg) {
288 SpinMutexLock l(&Value->mutex);
289 Print(M: Value->mib, id: Key, print_terse: bool(Arg));
290 }
291
292 // See memprof_mapping.h for an overview on histogram counters.
293 static MemInfoBlock CreateNewMIB(uptr p, MemprofChunk *m, u64 user_size) {
294 if (__memprof_histogram) {
295 return CreateNewMIBWithHistogram(p, m, user_size);
296 } else {
297 return CreateNewMIBWithoutHistogram(p, m, user_size);
298 }
299 }
300
301 static MemInfoBlock CreateNewMIBWithHistogram(uptr p, MemprofChunk *m,
302 u64 user_size) {
303
304 u64 c = GetShadowCountHistogram(p, size: user_size);
305 long curtime = GetTimestamp();
306 uint32_t HistogramSize =
307 RoundUpTo(size: user_size, HISTOGRAM_GRANULARITY) / HISTOGRAM_GRANULARITY;
308 uintptr_t Histogram =
309 (uintptr_t)InternalAlloc(size: HistogramSize * sizeof(uint64_t));
310 memset(s: (void *)Histogram, c: 0, n: HistogramSize * sizeof(uint64_t));
311 for (size_t i = 0; i < HistogramSize; ++i) {
312 u8 Counter =
313 *((u8 *)HISTOGRAM_MEM_TO_SHADOW(p + HISTOGRAM_GRANULARITY * i));
314 ((uint64_t *)Histogram)[i] = (uint64_t)Counter;
315 }
316 MemInfoBlock newMIB(user_size, c, m->timestamp_ms, curtime, m->cpu_id,
317 GetCpuId(), Histogram, HistogramSize);
318 return newMIB;
319 }
320
321 static MemInfoBlock CreateNewMIBWithoutHistogram(uptr p, MemprofChunk *m,
322 u64 user_size) {
323 u64 c = GetShadowCount(p, size: user_size);
324 long curtime = GetTimestamp();
325 MemInfoBlock newMIB(user_size, c, m->timestamp_ms, curtime, m->cpu_id,
326 GetCpuId(), 0, 0);
327 return newMIB;
328 }
329
330 void FinishAndWrite() {
331 if (flags()->print_text && common_flags()->print_module_map)
332 DumpProcessMap();
333
334 allocator.ForceLock();
335
336 InsertLiveBlocks();
337 if (flags()->print_text) {
338 if (!flags()->print_terse)
339 Printf(format: "Recorded MIBs (incl. live on exit):\n");
340 MIBMap.ForEach(cb: PrintCallback,
341 arg: reinterpret_cast<void *>(flags()->print_terse));
342 StackDepotPrintAll();
343 } else {
344 // Serialize the contents to a raw profile. Format documented in
345 // memprof_rawprofile.h.
346 char *Buffer = nullptr;
347
348 __sanitizer::ListOfModules List;
349 List.init();
350 ArrayRef<LoadedModule> Modules(List.begin(), List.end());
351 u64 BytesSerialized = SerializeToRawProfile(BlockCache&: MIBMap, Modules, Buffer);
352 CHECK(Buffer && BytesSerialized && "could not serialize to buffer");
353 report_file.Write(buffer: Buffer, length: BytesSerialized);
354 }
355
356 allocator.ForceUnlock();
357 }
358
359 // Inserts any blocks which have been allocated but not yet deallocated.
360 void InsertLiveBlocks() {
361 allocator.ForEachChunk(
362 callback: [](uptr chunk, void *alloc) {
363 u64 user_requested_size;
364 Allocator *A = (Allocator *)alloc;
365 MemprofChunk *m =
366 A->GetMemprofChunk(alloc_beg: (void *)chunk, user_requested_size);
367 if (!m)
368 return;
369 uptr user_beg = ((uptr)m) + kChunkHeaderSize;
370 MemInfoBlock newMIB = CreateNewMIB(p: user_beg, m, user_size: user_requested_size);
371 InsertOrMerge(Id: m->alloc_context_id, Block: newMIB, Map&: A->MIBMap);
372 },
373 arg: this);
374 }
375
376 void InitLinkerInitialized() {
377 SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
378 allocator.InitLinkerInitialized(
379 release_to_os_interval_ms: common_flags()->allocator_release_to_os_interval_ms);
380 max_user_defined_malloc_size = common_flags()->max_allocation_size_mb
381 ? common_flags()->max_allocation_size_mb
382 << 20
383 : kMaxAllowedMallocSize;
384 }
385
386 // -------------------- Allocation/Deallocation routines ---------------
387 void *Allocate(uptr size, uptr alignment, BufferedStackTrace *stack,
388 AllocType alloc_type) {
389 if (UNLIKELY(!memprof_inited))
390 MemprofInitFromRtl();
391 if (UNLIKELY(IsRssLimitExceeded())) {
392 if (AllocatorMayReturnNull())
393 return nullptr;
394 ReportRssLimitExceeded(stack);
395 }
396 CHECK(stack);
397 const uptr min_alignment = MEMPROF_ALIGNMENT;
398 if (alignment < min_alignment)
399 alignment = min_alignment;
400 if (size == 0) {
401 // We'd be happy to avoid allocating memory for zero-size requests, but
402 // some programs/tests depend on this behavior and assume that malloc
403 // would not return NULL even for zero-size allocations. Moreover, it
404 // looks like operator new should never return NULL, and results of
405 // consecutive "new" calls must be different even if the allocated size
406 // is zero.
407 size = 1;
408 }
409 CHECK(IsPowerOfTwo(alignment));
410 uptr rounded_size = RoundUpTo(size, boundary: alignment);
411 uptr needed_size = rounded_size + kChunkHeaderSize;
412 if (alignment > min_alignment)
413 needed_size += alignment;
414 CHECK(IsAligned(needed_size, min_alignment));
415 if (size > kMaxAllowedMallocSize || needed_size > kMaxAllowedMallocSize ||
416 size > max_user_defined_malloc_size) {
417 if (AllocatorMayReturnNull()) {
418 Report(format: "WARNING: MemProfiler failed to allocate 0x%zx bytes\n", size);
419 return nullptr;
420 }
421 uptr malloc_limit =
422 Min(a: kMaxAllowedMallocSize, b: max_user_defined_malloc_size);
423 ReportAllocationSizeTooBig(user_size: size, max_size: malloc_limit, stack);
424 }
425
426 MemprofThread *t = GetCurrentThread();
427 void *allocated;
428 if (t) {
429 AllocatorCache *cache = GetAllocatorCache(ms: &t->malloc_storage());
430 allocated = allocator.Allocate(cache, size: needed_size, alignment: 8);
431 } else {
432 SpinMutexLock l(&fallback_mutex);
433 AllocatorCache *cache = &fallback_allocator_cache;
434 allocated = allocator.Allocate(cache, size: needed_size, alignment: 8);
435 }
436 if (UNLIKELY(!allocated)) {
437 SetAllocatorOutOfMemory();
438 if (AllocatorMayReturnNull())
439 return nullptr;
440 ReportOutOfMemory(requested_size: size, stack);
441 }
442
443 uptr alloc_beg = reinterpret_cast<uptr>(allocated);
444 uptr alloc_end = alloc_beg + needed_size;
445 uptr beg_plus_header = alloc_beg + kChunkHeaderSize;
446 uptr user_beg = beg_plus_header;
447 if (!IsAligned(a: user_beg, alignment))
448 user_beg = RoundUpTo(size: user_beg, boundary: alignment);
449 uptr user_end = user_beg + size;
450 CHECK_LE(user_end, alloc_end);
451 uptr chunk_beg = user_beg - kChunkHeaderSize;
452 MemprofChunk *m = reinterpret_cast<MemprofChunk *>(chunk_beg);
453 m->from_memalign = alloc_beg != chunk_beg;
454 CHECK(size);
455
456 m->cpu_id = GetCpuId();
457 m->timestamp_ms = GetTimestamp();
458 m->alloc_context_id = StackDepotPut(stack: *stack);
459
460 ClearShadow(addr: user_beg, size: RoundUpTo(size, SHADOW_GRANULARITY));
461
462 MemprofStats &thread_stats = GetCurrentThreadStats();
463 thread_stats.mallocs++;
464 thread_stats.malloced += size;
465 thread_stats.malloced_overhead += needed_size - size;
466 if (needed_size > SizeClassMap::kMaxSize)
467 thread_stats.malloc_large++;
468 else
469 thread_stats.malloced_by_size[SizeClassMap::ClassID(size: needed_size)]++;
470
471 void *res = reinterpret_cast<void *>(user_beg);
472 atomic_store(a: &m->user_requested_size, v: size, mo: memory_order_release);
473 if (alloc_beg != chunk_beg) {
474 CHECK_LE(alloc_beg + sizeof(LargeChunkHeader), chunk_beg);
475 reinterpret_cast<LargeChunkHeader *>(alloc_beg)->Set(m);
476 }
477 RunMallocHooks(ptr: res, size);
478 return res;
479 }
480
481 void Deallocate(void *ptr, uptr delete_size, uptr delete_alignment,
482 BufferedStackTrace *stack, AllocType alloc_type) {
483 uptr p = reinterpret_cast<uptr>(ptr);
484 if (p == 0)
485 return;
486
487 RunFreeHooks(ptr);
488
489 uptr chunk_beg = p - kChunkHeaderSize;
490 MemprofChunk *m = reinterpret_cast<MemprofChunk *>(chunk_beg);
491
492 u64 user_requested_size =
493 atomic_exchange(a: &m->user_requested_size, v: 0, mo: memory_order_acquire);
494 if (memprof_inited && atomic_load_relaxed(a: &constructed) &&
495 !atomic_load_relaxed(a: &destructing)) {
496 MemInfoBlock newMIB = this->CreateNewMIB(p, m, user_size: user_requested_size);
497 InsertOrMerge(Id: m->alloc_context_id, Block: newMIB, Map&: MIBMap);
498 }
499
500 MemprofStats &thread_stats = GetCurrentThreadStats();
501 thread_stats.frees++;
502 thread_stats.freed += user_requested_size;
503
504 void *alloc_beg = m->AllocBeg();
505 if (alloc_beg != m) {
506 // Clear the magic value, as allocator internals may overwrite the
507 // contents of deallocated chunk, confusing GetMemprofChunk lookup.
508 reinterpret_cast<LargeChunkHeader *>(alloc_beg)->Set(nullptr);
509 }
510
511 MemprofThread *t = GetCurrentThread();
512 if (t) {
513 AllocatorCache *cache = GetAllocatorCache(ms: &t->malloc_storage());
514 allocator.Deallocate(cache, p: alloc_beg);
515 } else {
516 SpinMutexLock l(&fallback_mutex);
517 AllocatorCache *cache = &fallback_allocator_cache;
518 allocator.Deallocate(cache, p: alloc_beg);
519 }
520 }
521
522 void *Reallocate(void *old_ptr, uptr new_size, BufferedStackTrace *stack) {
523 CHECK(old_ptr && new_size);
524 uptr p = reinterpret_cast<uptr>(old_ptr);
525 uptr chunk_beg = p - kChunkHeaderSize;
526 MemprofChunk *m = reinterpret_cast<MemprofChunk *>(chunk_beg);
527
528 MemprofStats &thread_stats = GetCurrentThreadStats();
529 thread_stats.reallocs++;
530 thread_stats.realloced += new_size;
531
532 void *new_ptr = Allocate(size: new_size, alignment: 8, stack, alloc_type: FROM_MALLOC);
533 if (new_ptr) {
534 CHECK_NE(REAL(memcpy), nullptr);
535 uptr memcpy_size = Min(a: new_size, b: m->UsedSize());
536 REAL(memcpy)(new_ptr, old_ptr, memcpy_size);
537 Deallocate(ptr: old_ptr, delete_size: 0, delete_alignment: 0, stack, alloc_type: FROM_MALLOC);
538 }
539 return new_ptr;
540 }
541
542 void *Calloc(uptr nmemb, uptr size, BufferedStackTrace *stack) {
543 if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
544 if (AllocatorMayReturnNull())
545 return nullptr;
546 ReportCallocOverflow(count: nmemb, size, stack);
547 }
548 void *ptr = Allocate(size: nmemb * size, alignment: 8, stack, alloc_type: FROM_MALLOC);
549 // If the memory comes from the secondary allocator no need to clear it
550 // as it comes directly from mmap.
551 if (ptr && allocator.FromPrimary(p: ptr))
552 REAL(memset)(ptr, 0, nmemb * size);
553 return ptr;
554 }
555
556 void CommitBack(MemprofThreadLocalMallocStorage *ms) {
557 AllocatorCache *ac = GetAllocatorCache(ms);
558 allocator.SwallowCache(cache: ac);
559 }
560
561 // -------------------------- Chunk lookup ----------------------
562
563 // Assumes alloc_beg == allocator.GetBlockBegin(alloc_beg).
564 MemprofChunk *GetMemprofChunk(void *alloc_beg, u64 &user_requested_size) {
565 if (!alloc_beg)
566 return nullptr;
567 MemprofChunk *p = reinterpret_cast<LargeChunkHeader *>(alloc_beg)->Get();
568 if (!p) {
569 if (!allocator.FromPrimary(p: alloc_beg))
570 return nullptr;
571 p = reinterpret_cast<MemprofChunk *>(alloc_beg);
572 }
573 // The size is reset to 0 on deallocation (and a min of 1 on
574 // allocation).
575 user_requested_size =
576 atomic_load(a: &p->user_requested_size, mo: memory_order_acquire);
577 if (user_requested_size)
578 return p;
579 return nullptr;
580 }
581
582 MemprofChunk *GetMemprofChunkByAddr(uptr p, u64 &user_requested_size) {
583 void *alloc_beg = allocator.GetBlockBegin(p: reinterpret_cast<void *>(p));
584 return GetMemprofChunk(alloc_beg, user_requested_size);
585 }
586
587 uptr AllocationSize(uptr p) {
588 u64 user_requested_size;
589 MemprofChunk *m = GetMemprofChunkByAddr(p, user_requested_size);
590 if (!m)
591 return 0;
592 if (m->Beg() != p)
593 return 0;
594 return user_requested_size;
595 }
596
597 uptr AllocationSizeFast(uptr p) {
598 return reinterpret_cast<MemprofChunk *>(p - kChunkHeaderSize)->UsedSize();
599 }
600
601 void Purge() { allocator.ForceReleaseToOS(); }
602
603 void PrintStats() { allocator.PrintStats(); }
604
605 void ForceLock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
606 allocator.ForceLock();
607 fallback_mutex.Lock();
608 }
609
610 void ForceUnlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
611 fallback_mutex.Unlock();
612 allocator.ForceUnlock();
613 }
614};
615
616static Allocator instance(LINKER_INITIALIZED);
617
618static MemprofAllocator &get_allocator() { return instance.allocator; }
619
620void InitializeAllocator() { instance.InitLinkerInitialized(); }
621
622void MemprofThreadLocalMallocStorage::CommitBack() {
623 instance.CommitBack(ms: this);
624}
625
626void PrintInternalAllocatorStats() { instance.PrintStats(); }
627
628void memprof_free(void *ptr, BufferedStackTrace *stack, AllocType alloc_type) {
629 instance.Deallocate(ptr, delete_size: 0, delete_alignment: 0, stack, alloc_type);
630}
631
632void memprof_delete(void *ptr, uptr size, uptr alignment,
633 BufferedStackTrace *stack, AllocType alloc_type) {
634 instance.Deallocate(ptr, delete_size: size, delete_alignment: alignment, stack, alloc_type);
635}
636
637void *memprof_malloc(uptr size, BufferedStackTrace *stack) {
638 return SetErrnoOnNull(instance.Allocate(size, alignment: 8, stack, alloc_type: FROM_MALLOC));
639}
640
641void *memprof_calloc(uptr nmemb, uptr size, BufferedStackTrace *stack) {
642 return SetErrnoOnNull(instance.Calloc(nmemb, size, stack));
643}
644
645void *memprof_reallocarray(void *p, uptr nmemb, uptr size,
646 BufferedStackTrace *stack) {
647 if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
648 errno = errno_ENOMEM;
649 if (AllocatorMayReturnNull())
650 return nullptr;
651 ReportReallocArrayOverflow(count: nmemb, size, stack);
652 }
653 return memprof_realloc(p, size: nmemb * size, stack);
654}
655
656void *memprof_realloc(void *p, uptr size, BufferedStackTrace *stack) {
657 if (!p)
658 return SetErrnoOnNull(instance.Allocate(size, alignment: 8, stack, alloc_type: FROM_MALLOC));
659 if (size == 0) {
660 if (flags()->allocator_frees_and_returns_null_on_realloc_zero) {
661 instance.Deallocate(ptr: p, delete_size: 0, delete_alignment: 0, stack, alloc_type: FROM_MALLOC);
662 return nullptr;
663 }
664 // Allocate a size of 1 if we shouldn't free() on Realloc to 0
665 size = 1;
666 }
667 return SetErrnoOnNull(instance.Reallocate(old_ptr: p, new_size: size, stack));
668}
669
670void *memprof_valloc(uptr size, BufferedStackTrace *stack) {
671 return SetErrnoOnNull(
672 instance.Allocate(size, alignment: GetPageSizeCached(), stack, alloc_type: FROM_MALLOC));
673}
674
675void *memprof_pvalloc(uptr size, BufferedStackTrace *stack) {
676 uptr PageSize = GetPageSizeCached();
677 if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) {
678 errno = errno_ENOMEM;
679 if (AllocatorMayReturnNull())
680 return nullptr;
681 ReportPvallocOverflow(size, stack);
682 }
683 // pvalloc(0) should allocate one page.
684 size = size ? RoundUpTo(size, boundary: PageSize) : PageSize;
685 return SetErrnoOnNull(instance.Allocate(size, alignment: PageSize, stack, alloc_type: FROM_MALLOC));
686}
687
688void *memprof_memalign(uptr alignment, uptr size, BufferedStackTrace *stack,
689 AllocType alloc_type) {
690 if (UNLIKELY(!IsPowerOfTwo(alignment))) {
691 errno = errno_EINVAL;
692 if (AllocatorMayReturnNull())
693 return nullptr;
694 ReportInvalidAllocationAlignment(alignment, stack);
695 }
696 return SetErrnoOnNull(instance.Allocate(size, alignment, stack, alloc_type));
697}
698
699void *memprof_aligned_alloc(uptr alignment, uptr size,
700 BufferedStackTrace *stack) {
701 if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) {
702 errno = errno_EINVAL;
703 if (AllocatorMayReturnNull())
704 return nullptr;
705 ReportInvalidAlignedAllocAlignment(size, alignment, stack);
706 }
707 return SetErrnoOnNull(instance.Allocate(size, alignment, stack, alloc_type: FROM_MALLOC));
708}
709
710int memprof_posix_memalign(void **memptr, uptr alignment, uptr size,
711 BufferedStackTrace *stack) {
712 if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) {
713 if (AllocatorMayReturnNull())
714 return errno_EINVAL;
715 ReportInvalidPosixMemalignAlignment(alignment, stack);
716 }
717 void *ptr = instance.Allocate(size, alignment, stack, alloc_type: FROM_MALLOC);
718 if (UNLIKELY(!ptr))
719 // OOM error is already taken care of by Allocate.
720 return errno_ENOMEM;
721 CHECK(IsAligned((uptr)ptr, alignment));
722 *memptr = ptr;
723 return 0;
724}
725
726static const void *memprof_malloc_begin(const void *p) {
727 u64 user_requested_size;
728 MemprofChunk *m =
729 instance.GetMemprofChunkByAddr(p: (uptr)p, user_requested_size);
730 if (!m)
731 return nullptr;
732 if (user_requested_size == 0)
733 return nullptr;
734
735 return (const void *)m->Beg();
736}
737
738uptr memprof_malloc_usable_size(const void *ptr) {
739 if (!ptr)
740 return 0;
741 uptr usable_size = instance.AllocationSize(p: reinterpret_cast<uptr>(ptr));
742 return usable_size;
743}
744
745} // namespace __memprof
746
747// ---------------------- Interface ---------------- {{{1
748using namespace __memprof;
749
750uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; }
751
752int __sanitizer_get_ownership(const void *p) {
753 return memprof_malloc_usable_size(ptr: p) != 0;
754}
755
756const void *__sanitizer_get_allocated_begin(const void *p) {
757 return memprof_malloc_begin(p);
758}
759
760uptr __sanitizer_get_allocated_size(const void *p) {
761 return memprof_malloc_usable_size(ptr: p);
762}
763
764uptr __sanitizer_get_allocated_size_fast(const void *p) {
765 DCHECK_EQ(p, __sanitizer_get_allocated_begin(p));
766 uptr ret = instance.AllocationSizeFast(p: reinterpret_cast<uptr>(p));
767 DCHECK_EQ(ret, __sanitizer_get_allocated_size(p));
768 return ret;
769}
770
771void __sanitizer_purge_allocator() { instance.Purge(); }
772
773int __memprof_profile_dump() {
774 instance.FinishAndWrite();
775 // In the future we may want to return non-zero if there are any errors
776 // detected during the dumping process.
777 return 0;
778}
779
780void __memprof_profile_reset() {
781 if (report_file.fd != kInvalidFd && report_file.fd != kStdoutFd &&
782 report_file.fd != kStderrFd) {
783 CloseFile(report_file.fd);
784 // Setting the file descriptor to kInvalidFd ensures that we will reopen the
785 // file when invoking Write again.
786 report_file.fd = kInvalidFd;
787 }
788}
789