1//===-- msan_allocator.cpp -------------------------- ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is a part of MemorySanitizer.
10//
11// MemorySanitizer allocator.
12//===----------------------------------------------------------------------===//
13
14#include "msan_allocator.h"
15
16#include "msan.h"
17#include "msan_interface_internal.h"
18#include "msan_origin.h"
19#include "msan_poisoning.h"
20#include "msan_thread.h"
21#include "sanitizer_common/sanitizer_allocator.h"
22#include "sanitizer_common/sanitizer_allocator_checks.h"
23#include "sanitizer_common/sanitizer_allocator_interface.h"
24#include "sanitizer_common/sanitizer_allocator_report.h"
25#include "sanitizer_common/sanitizer_errno.h"
26
27using namespace __msan;
28
29namespace {
30struct Metadata {
31 uptr requested_size;
32};
33
34struct MsanMapUnmapCallback {
35 void OnMap(uptr p, uptr size) const {}
36 void OnMapSecondary(uptr p, uptr size, uptr user_begin,
37 uptr user_size) const {}
38 void OnUnmap(uptr p, uptr size) const {
39 __msan_unpoison(a: (void *)p, size);
40
41 // We are about to unmap a chunk of user memory.
42 // Mark the corresponding shadow memory as not needed.
43 uptr shadow_p = MEM_TO_SHADOW(p);
44 ReleaseMemoryPagesToOS(beg: shadow_p, end: shadow_p + size);
45 if (__msan_get_track_origins()) {
46 uptr origin_p = MEM_TO_ORIGIN(p);
47 ReleaseMemoryPagesToOS(beg: origin_p, end: origin_p + size);
48 }
49 }
50};
51
52// Note: to ensure that the allocator is compatible with the application memory
53// layout (especially with high-entropy ASLR), kSpaceBeg and kSpaceSize must be
54// duplicated as MappingDesc::ALLOCATOR in msan.h.
55#if defined(__mips64)
56const uptr kMaxAllowedMallocSize = 2UL << 30;
57
58struct AP32 {
59 static const uptr kSpaceBeg = SANITIZER_MMAP_BEGIN;
60 static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE;
61 static const uptr kMetadataSize = sizeof(Metadata);
62 using SizeClassMap = __sanitizer::CompactSizeClassMap;
63 static const uptr kRegionSizeLog = 20;
64 using AddressSpaceView = LocalAddressSpaceView;
65 using MapUnmapCallback = MsanMapUnmapCallback;
66 static const uptr kFlags = 0;
67};
68using PrimaryAllocator = SizeClassAllocator32<AP32>;
69#elif defined(__x86_64__)
70#if SANITIZER_NETBSD || SANITIZER_LINUX
71const uptr kAllocatorSpace = 0x700000000000ULL;
72#else
73const uptr kAllocatorSpace = 0x600000000000ULL;
74#endif
75const uptr kMaxAllowedMallocSize = 1ULL << 40;
76
77struct AP64 { // Allocator64 parameters. Deliberately using a short name.
78 static const uptr kSpaceBeg = kAllocatorSpace;
79 static const uptr kSpaceSize = 0x40000000000; // 4T.
80 static const uptr kMetadataSize = sizeof(Metadata);
81 using SizeClassMap = DefaultSizeClassMap;
82 using MapUnmapCallback = MsanMapUnmapCallback;
83 static const uptr kFlags = 0;
84 using AddressSpaceView = LocalAddressSpaceView;
85};
86
87using PrimaryAllocator = SizeClassAllocator64<AP64>;
88
89#elif defined(__loongarch_lp64)
90const uptr kAllocatorSpace = 0x700000000000ULL;
91const uptr kMaxAllowedMallocSize = 8UL << 30;
92
93struct AP64 { // Allocator64 parameters. Deliberately using a short name.
94 static const uptr kSpaceBeg = kAllocatorSpace;
95 static const uptr kSpaceSize = 0x40000000000; // 4T.
96 static const uptr kMetadataSize = sizeof(Metadata);
97 using SizeClassMap = DefaultSizeClassMap;
98 using MapUnmapCallback = MsanMapUnmapCallback;
99 static const uptr kFlags = 0;
100 using AddressSpaceView = LocalAddressSpaceView;
101};
102
103using PrimaryAllocator = SizeClassAllocator64<AP64>;
104
105#elif defined(__powerpc64__)
106const uptr kMaxAllowedMallocSize = 2UL << 30; // 2G
107
108struct AP64 { // Allocator64 parameters. Deliberately using a short name.
109 static const uptr kSpaceBeg = 0x300000000000;
110 static const uptr kSpaceSize = 0x020000000000; // 2T.
111 static const uptr kMetadataSize = sizeof(Metadata);
112 using SizeClassMap = DefaultSizeClassMap;
113 using MapUnmapCallback = MsanMapUnmapCallback;
114 static const uptr kFlags = 0;
115 using AddressSpaceView = LocalAddressSpaceView;
116};
117
118using PrimaryAllocator = SizeClassAllocator64<AP64>;
119#elif defined(__s390x__)
120const uptr kMaxAllowedMallocSize = 2UL << 30; // 2G
121
122struct AP64 { // Allocator64 parameters. Deliberately using a short name.
123 static const uptr kSpaceBeg = 0x440000000000;
124 static const uptr kSpaceSize = 0x020000000000; // 2T.
125 static const uptr kMetadataSize = sizeof(Metadata);
126 using SizeClassMap = DefaultSizeClassMap;
127 using MapUnmapCallback = MsanMapUnmapCallback;
128 static const uptr kFlags = 0;
129 using AddressSpaceView = LocalAddressSpaceView;
130};
131
132using PrimaryAllocator = SizeClassAllocator64<AP64>;
133#elif defined(__aarch64__)
134const uptr kMaxAllowedMallocSize = 8UL << 30;
135
136struct AP64 {
137 static const uptr kSpaceBeg = 0xE00000000000ULL;
138 static const uptr kSpaceSize = 0x40000000000; // 4T.
139 static const uptr kMetadataSize = sizeof(Metadata);
140 using SizeClassMap = DefaultSizeClassMap;
141 using MapUnmapCallback = MsanMapUnmapCallback;
142 static const uptr kFlags = 0;
143 using AddressSpaceView = LocalAddressSpaceView;
144};
145using PrimaryAllocator = SizeClassAllocator64<AP64>;
146#endif
147using Allocator = CombinedAllocator<PrimaryAllocator>;
148using AllocatorCache = Allocator::AllocatorCache;
149} // namespace __msan
150
151static Allocator allocator;
152static AllocatorCache fallback_allocator_cache;
153static StaticSpinMutex fallback_mutex;
154
155static uptr max_malloc_size;
156
157void __msan::MsanAllocatorInit() {
158 SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
159 allocator.Init(release_to_os_interval_ms: common_flags()->allocator_release_to_os_interval_ms);
160 if (common_flags()->max_allocation_size_mb)
161 max_malloc_size = Min(a: common_flags()->max_allocation_size_mb << 20,
162 b: kMaxAllowedMallocSize);
163 else
164 max_malloc_size = kMaxAllowedMallocSize;
165}
166
167void __msan::LockAllocator() { allocator.ForceLock(); }
168
169void __msan::UnlockAllocator() { allocator.ForceUnlock(); }
170
171AllocatorCache *GetAllocatorCache(MsanThreadLocalMallocStorage *ms) {
172 CHECK_LE(sizeof(AllocatorCache), sizeof(ms->allocator_cache));
173 return reinterpret_cast<AllocatorCache *>(ms->allocator_cache);
174}
175
176void MsanThreadLocalMallocStorage::Init() {
177 allocator.InitCache(cache: GetAllocatorCache(ms: this));
178}
179
180void MsanThreadLocalMallocStorage::CommitBack() {
181 allocator.SwallowCache(cache: GetAllocatorCache(ms: this));
182 allocator.DestroyCache(cache: GetAllocatorCache(ms: this));
183}
184
185static void *MsanAllocate(BufferedStackTrace *stack, uptr size, uptr alignment,
186 bool zero) {
187 if (UNLIKELY(size > max_malloc_size)) {
188 if (AllocatorMayReturnNull()) {
189 Report(format: "WARNING: MemorySanitizer failed to allocate 0x%zx bytes\n", size);
190 return nullptr;
191 }
192 GET_FATAL_STACK_TRACE_IF_EMPTY(stack);
193 ReportAllocationSizeTooBig(user_size: size, max_size: max_malloc_size, stack);
194 }
195 if (UNLIKELY(IsRssLimitExceeded())) {
196 if (AllocatorMayReturnNull())
197 return nullptr;
198 GET_FATAL_STACK_TRACE_IF_EMPTY(stack);
199 ReportRssLimitExceeded(stack);
200 }
201 MsanThread *t = GetCurrentThread();
202 void *allocated;
203 if (t) {
204 AllocatorCache *cache = GetAllocatorCache(ms: &t->malloc_storage());
205 allocated = allocator.Allocate(cache, size, alignment);
206 } else {
207 SpinMutexLock l(&fallback_mutex);
208 AllocatorCache *cache = &fallback_allocator_cache;
209 allocated = allocator.Allocate(cache, size, alignment);
210 }
211 if (UNLIKELY(!allocated)) {
212 SetAllocatorOutOfMemory();
213 if (AllocatorMayReturnNull())
214 return nullptr;
215 GET_FATAL_STACK_TRACE_IF_EMPTY(stack);
216 ReportOutOfMemory(requested_size: size, stack);
217 }
218 auto *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(p: allocated));
219 meta->requested_size = size;
220 uptr actually_allocated_size = allocator.GetActuallyAllocatedSize(p: allocated);
221 void* padding_start = reinterpret_cast<char*>(allocated) + size;
222 uptr padding_size = actually_allocated_size - size;
223
224 // - With calloc(7,1), we can set the ideal tagging:
225 // bytes 0-6: initialized, origin not set (and irrelevant)
226 // byte 7: uninitialized, origin TAG_ALLOC_PADDING
227 // bytes 8-15: uninitialized, origin TAG_ALLOC_PADDING
228 // - If we have malloc(7) and __msan_get_track_origins() > 1, the 4-byte
229 // origin granularity only allows the slightly suboptimal tagging:
230 // bytes 0-6: uninitialized, origin TAG_ALLOC
231 // byte 7: uninitialized, origin TAG_ALLOC (suboptimal)
232 // bytes 8-15: uninitialized, origin TAG_ALLOC_PADDING
233 // - If we have malloc(7) and __msan_get_track_origins() == 1, we use a
234 // single origin bean to reduce overhead:
235 // bytes 0-6: uninitialized, origin TAG_ALLOC
236 // byte 7: uninitialized, origin TAG_ALLOC (suboptimal)
237 // bytes 8-15: uninitialized, origin TAG_ALLOC (suboptimal)
238 if (__msan_get_track_origins() && flags()->poison_in_malloc &&
239 (zero || (__msan_get_track_origins() > 1))) {
240 stack->tag = STACK_TRACE_TAG_ALLOC_PADDING;
241 Origin o2 = Origin::CreateHeapOrigin(stack);
242 __msan_set_origin(a: padding_start, size: padding_size, origin: o2.raw_id());
243 }
244
245 if (zero) {
246 if (allocator.FromPrimary(p: allocated))
247 __msan_clear_and_unpoison(a: allocated, size);
248 else
249 __msan_unpoison(a: allocated, size); // Mem is already zeroed.
250
251 if (flags()->poison_in_malloc)
252 __msan_poison(a: padding_start, size: padding_size);
253 } else if (flags()->poison_in_malloc) {
254 __msan_poison(a: allocated, size: actually_allocated_size);
255
256 if (__msan_get_track_origins()) {
257 stack->tag = StackTrace::TAG_ALLOC;
258 Origin o = Origin::CreateHeapOrigin(stack);
259 __msan_set_origin(
260 a: allocated,
261 size: __msan_get_track_origins() == 1 ? actually_allocated_size : size,
262 origin: o.raw_id());
263 }
264 }
265
266 UnpoisonParam(n: 2);
267 RunMallocHooks(ptr: allocated, size);
268 return allocated;
269}
270
271void __msan::MsanDeallocate(BufferedStackTrace *stack, void *p) {
272 DCHECK(p);
273 UnpoisonParam(n: 1);
274 RunFreeHooks(ptr: p);
275
276 Metadata *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(p));
277 uptr size = meta->requested_size;
278 meta->requested_size = 0;
279 // This memory will not be reused by anyone else, so we are free to keep it
280 // poisoned. The secondary allocator will unmap and unpoison by
281 // MsanMapUnmapCallback, no need to poison it here.
282 if (flags()->poison_in_free && allocator.FromPrimary(p)) {
283 __msan_poison(a: p, size);
284 if (__msan_get_track_origins()) {
285 uptr actually_allocated_size = allocator.GetActuallyAllocatedSize(p);
286 stack->tag = StackTrace::TAG_DEALLOC;
287 Origin o = Origin::CreateHeapOrigin(stack);
288 __msan_set_origin(a: p, size: actually_allocated_size, origin: o.raw_id());
289 }
290 }
291 if (MsanThread *t = GetCurrentThread()) {
292 AllocatorCache *cache = GetAllocatorCache(ms: &t->malloc_storage());
293 allocator.Deallocate(cache, p);
294 } else {
295 SpinMutexLock l(&fallback_mutex);
296 AllocatorCache *cache = &fallback_allocator_cache;
297 allocator.Deallocate(cache, p);
298 }
299}
300
301static void *MsanReallocate(BufferedStackTrace *stack, void *old_p,
302 uptr new_size, uptr alignment) {
303 Metadata *meta = reinterpret_cast<Metadata*>(allocator.GetMetaData(p: old_p));
304 uptr old_size = meta->requested_size;
305 uptr actually_allocated_size = allocator.GetActuallyAllocatedSize(p: old_p);
306 if (new_size <= actually_allocated_size) {
307 // We are not reallocating here.
308 meta->requested_size = new_size;
309 if (new_size > old_size) {
310 if (flags()->poison_in_malloc) {
311 stack->tag = StackTrace::TAG_ALLOC;
312 PoisonMemory(dst: (char *)old_p + old_size, size: new_size - old_size, stack);
313 }
314 }
315 return old_p;
316 }
317 uptr memcpy_size = Min(a: new_size, b: old_size);
318 void *new_p = MsanAllocate(stack, size: new_size, alignment, zero: false);
319 if (new_p) {
320 CopyMemory(dst: new_p, src: old_p, size: memcpy_size, stack);
321 MsanDeallocate(stack, p: old_p);
322 }
323 return new_p;
324}
325
326static void *MsanCalloc(BufferedStackTrace *stack, uptr nmemb, uptr size) {
327 if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
328 if (AllocatorMayReturnNull())
329 return nullptr;
330 GET_FATAL_STACK_TRACE_IF_EMPTY(stack);
331 ReportCallocOverflow(count: nmemb, size, stack);
332 }
333 return MsanAllocate(stack, size: nmemb * size, alignment: sizeof(u64), zero: true);
334}
335
336static const void *AllocationBegin(const void *p) {
337 if (!p)
338 return nullptr;
339 void *beg = allocator.GetBlockBegin(p);
340 if (!beg)
341 return nullptr;
342 auto *b = reinterpret_cast<Metadata *>(allocator.GetMetaData(p: beg));
343 if (!b)
344 return nullptr;
345 if (b->requested_size == 0)
346 return nullptr;
347
348 return beg;
349}
350
351static uptr AllocationSizeFast(const void *p) {
352 return reinterpret_cast<Metadata *>(allocator.GetMetaData(p))->requested_size;
353}
354
355static uptr AllocationSize(const void *p) {
356 if (!p)
357 return 0;
358 if (allocator.GetBlockBegin(p) != p)
359 return 0;
360 return AllocationSizeFast(p);
361}
362
363void *__msan::msan_malloc(uptr size, BufferedStackTrace *stack) {
364 return SetErrnoOnNull(MsanAllocate(stack, size, alignment: sizeof(u64), zero: false));
365}
366
367void *__msan::msan_calloc(uptr nmemb, uptr size, BufferedStackTrace *stack) {
368 return SetErrnoOnNull(MsanCalloc(stack, nmemb, size));
369}
370
371void *__msan::msan_realloc(void *ptr, uptr size, BufferedStackTrace *stack) {
372 if (!ptr)
373 return SetErrnoOnNull(MsanAllocate(stack, size, alignment: sizeof(u64), zero: false));
374 if (size == 0) {
375 MsanDeallocate(stack, p: ptr);
376 return nullptr;
377 }
378 return SetErrnoOnNull(MsanReallocate(stack, old_p: ptr, new_size: size, alignment: sizeof(u64)));
379}
380
381void *__msan::msan_reallocarray(void *ptr, uptr nmemb, uptr size,
382 BufferedStackTrace *stack) {
383 if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
384 errno = errno_ENOMEM;
385 if (AllocatorMayReturnNull())
386 return nullptr;
387 GET_FATAL_STACK_TRACE_IF_EMPTY(stack);
388 ReportReallocArrayOverflow(count: nmemb, size, stack);
389 }
390 return msan_realloc(ptr, size: nmemb * size, stack);
391}
392
393void *__msan::msan_valloc(uptr size, BufferedStackTrace *stack) {
394 return SetErrnoOnNull(MsanAllocate(stack, size, alignment: GetPageSizeCached(), zero: false));
395}
396
397void *__msan::msan_pvalloc(uptr size, BufferedStackTrace *stack) {
398 uptr PageSize = GetPageSizeCached();
399 if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) {
400 errno = errno_ENOMEM;
401 if (AllocatorMayReturnNull())
402 return nullptr;
403 GET_FATAL_STACK_TRACE_IF_EMPTY(stack);
404 ReportPvallocOverflow(size, stack);
405 }
406 // pvalloc(0) should allocate one page.
407 size = size ? RoundUpTo(size, boundary: PageSize) : PageSize;
408 return SetErrnoOnNull(MsanAllocate(stack, size, alignment: PageSize, zero: false));
409}
410
411void *__msan::msan_aligned_alloc(uptr alignment, uptr size,
412 BufferedStackTrace *stack) {
413 if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) {
414 errno = errno_EINVAL;
415 if (AllocatorMayReturnNull())
416 return nullptr;
417 GET_FATAL_STACK_TRACE_IF_EMPTY(stack);
418 ReportInvalidAlignedAllocAlignment(size, alignment, stack);
419 }
420 return SetErrnoOnNull(MsanAllocate(stack, size, alignment, zero: false));
421}
422
423void *__msan::msan_memalign(uptr alignment, uptr size,
424 BufferedStackTrace *stack) {
425 if (UNLIKELY(!IsPowerOfTwo(alignment))) {
426 errno = errno_EINVAL;
427 if (AllocatorMayReturnNull())
428 return nullptr;
429 GET_FATAL_STACK_TRACE_IF_EMPTY(stack);
430 ReportInvalidAllocationAlignment(alignment, stack);
431 }
432 return SetErrnoOnNull(MsanAllocate(stack, size, alignment, zero: false));
433}
434
435int __msan::msan_posix_memalign(void **memptr, uptr alignment, uptr size,
436 BufferedStackTrace *stack) {
437 if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) {
438 if (AllocatorMayReturnNull())
439 return errno_EINVAL;
440 GET_FATAL_STACK_TRACE_IF_EMPTY(stack);
441 ReportInvalidPosixMemalignAlignment(alignment, stack);
442 }
443 void *ptr = MsanAllocate(stack, size, alignment, zero: false);
444 if (UNLIKELY(!ptr))
445 // OOM error is already taken care of by MsanAllocate.
446 return errno_ENOMEM;
447 CHECK(IsAligned((uptr)ptr, alignment));
448 *memptr = ptr;
449 return 0;
450}
451
452extern "C" {
453uptr __sanitizer_get_current_allocated_bytes() {
454 uptr stats[AllocatorStatCount];
455 allocator.GetStats(s: stats);
456 return stats[AllocatorStatAllocated];
457}
458
459uptr __sanitizer_get_heap_size() {
460 uptr stats[AllocatorStatCount];
461 allocator.GetStats(s: stats);
462 return stats[AllocatorStatMapped];
463}
464
465uptr __sanitizer_get_free_bytes() { return 1; }
466
467uptr __sanitizer_get_unmapped_bytes() { return 1; }
468
469uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; }
470
471int __sanitizer_get_ownership(const void *p) { return AllocationSize(p) != 0; }
472
473const void *__sanitizer_get_allocated_begin(const void *p) {
474 return AllocationBegin(p);
475}
476
477uptr __sanitizer_get_allocated_size(const void *p) { return AllocationSize(p); }
478
479uptr __sanitizer_get_allocated_size_fast(const void *p) {
480 DCHECK_EQ(p, __sanitizer_get_allocated_begin(p));
481 uptr ret = AllocationSizeFast(p);
482 DCHECK_EQ(ret, __sanitizer_get_allocated_size(p));
483 return ret;
484}
485
486void __sanitizer_purge_allocator() { allocator.ForceReleaseToOS(); }
487}
488