1//===-- msan_allocator.cpp -------------------------- ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is a part of MemorySanitizer.
10//
11// MemorySanitizer allocator.
12//===----------------------------------------------------------------------===//
13
14#include "msan_allocator.h"
15
16#include "msan.h"
17#include "msan_interface_internal.h"
18#include "msan_origin.h"
19#include "msan_poisoning.h"
20#include "msan_thread.h"
21#include "sanitizer_common/sanitizer_allocator.h"
22#include "sanitizer_common/sanitizer_allocator_checks.h"
23#include "sanitizer_common/sanitizer_allocator_interface.h"
24#include "sanitizer_common/sanitizer_allocator_report.h"
25#include "sanitizer_common/sanitizer_errno.h"
26
27using namespace __msan;
28
29namespace {
30struct Metadata {
31 uptr requested_size;
32};
33
34struct MsanMapUnmapCallback {
35 void OnMap(uptr p, uptr size) const {}
36 void OnMapSecondary(uptr p, uptr size, uptr user_begin,
37 uptr user_size) const {}
38 void OnUnmap(uptr p, uptr size) const {
39 __msan_unpoison(a: (void *)p, size);
40
41 // We are about to unmap a chunk of user memory.
42 // Mark the corresponding shadow memory as not needed.
43 uptr shadow_p = MEM_TO_SHADOW(p);
44 ReleaseMemoryPagesToOS(beg: shadow_p, end: shadow_p + size);
45 if (__msan_get_track_origins()) {
46 uptr origin_p = MEM_TO_ORIGIN(p);
47 ReleaseMemoryPagesToOS(beg: origin_p, end: origin_p + size);
48 }
49 }
50};
51
52// Note: to ensure that the allocator is compatible with the application memory
53// layout (especially with high-entropy ASLR), kSpaceBeg and kSpaceSize must be
54// duplicated as MappingDesc::ALLOCATOR in msan.h.
55#if defined(__mips64)
56const uptr kMaxAllowedMallocSize = 2UL << 30;
57
58struct AP32 {
59 static const uptr kSpaceBeg = 0;
60 static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE;
61 static const uptr kMetadataSize = sizeof(Metadata);
62 using SizeClassMap = __sanitizer::CompactSizeClassMap;
63 static const uptr kRegionSizeLog = 20;
64 using AddressSpaceView = LocalAddressSpaceView;
65 using MapUnmapCallback = MsanMapUnmapCallback;
66 static const uptr kFlags = 0;
67};
68using PrimaryAllocator = SizeClassAllocator32<AP32>;
69#elif defined(__x86_64__)
70#if SANITIZER_NETBSD || SANITIZER_LINUX
71const uptr kAllocatorSpace = 0x700000000000ULL;
72#else
73const uptr kAllocatorSpace = 0x600000000000ULL;
74#endif
75const uptr kMaxAllowedMallocSize = 1ULL << 40;
76
77struct AP64 { // Allocator64 parameters. Deliberately using a short name.
78 static const uptr kSpaceBeg = kAllocatorSpace;
79 static const uptr kSpaceSize = 0x40000000000; // 4T.
80 static const uptr kMetadataSize = sizeof(Metadata);
81 using SizeClassMap = DefaultSizeClassMap;
82 using MapUnmapCallback = MsanMapUnmapCallback;
83 static const uptr kFlags = 0;
84 using AddressSpaceView = LocalAddressSpaceView;
85};
86
87using PrimaryAllocator = SizeClassAllocator64<AP64>;
88
89#elif defined(__loongarch_lp64)
90const uptr kAllocatorSpace = 0x700000000000ULL;
91const uptr kMaxAllowedMallocSize = 8UL << 30;
92
93struct AP64 { // Allocator64 parameters. Deliberately using a short name.
94 static const uptr kSpaceBeg = kAllocatorSpace;
95 static const uptr kSpaceSize = 0x40000000000; // 4T.
96 static const uptr kMetadataSize = sizeof(Metadata);
97 using SizeClassMap = DefaultSizeClassMap;
98 using MapUnmapCallback = MsanMapUnmapCallback;
99 static const uptr kFlags = 0;
100 using AddressSpaceView = LocalAddressSpaceView;
101};
102
103using PrimaryAllocator = SizeClassAllocator64<AP64>;
104
105#elif defined(__powerpc64__)
106const uptr kMaxAllowedMallocSize = 2UL << 30; // 2G
107
108struct AP64 { // Allocator64 parameters. Deliberately using a short name.
109 static const uptr kSpaceBeg = 0x300000000000;
110 static const uptr kSpaceSize = 0x020000000000; // 2T.
111 static const uptr kMetadataSize = sizeof(Metadata);
112 using SizeClassMap = DefaultSizeClassMap;
113 using MapUnmapCallback = MsanMapUnmapCallback;
114 static const uptr kFlags = 0;
115 using AddressSpaceView = LocalAddressSpaceView;
116};
117
118using PrimaryAllocator = SizeClassAllocator64<AP64>;
119#elif defined(__s390x__)
120const uptr kMaxAllowedMallocSize = 2UL << 30; // 2G
121
122struct AP64 { // Allocator64 parameters. Deliberately using a short name.
123 static const uptr kSpaceBeg = 0x440000000000;
124 static const uptr kSpaceSize = 0x020000000000; // 2T.
125 static const uptr kMetadataSize = sizeof(Metadata);
126 using SizeClassMap = DefaultSizeClassMap;
127 using MapUnmapCallback = MsanMapUnmapCallback;
128 static const uptr kFlags = 0;
129 using AddressSpaceView = LocalAddressSpaceView;
130};
131
132using PrimaryAllocator = SizeClassAllocator64<AP64>;
133#elif defined(__aarch64__)
134const uptr kMaxAllowedMallocSize = 8UL << 30;
135
136struct AP64 {
137 static const uptr kSpaceBeg = 0xE00000000000ULL;
138 static const uptr kSpaceSize = 0x40000000000; // 4T.
139 static const uptr kMetadataSize = sizeof(Metadata);
140 using SizeClassMap = DefaultSizeClassMap;
141 using MapUnmapCallback = MsanMapUnmapCallback;
142 static const uptr kFlags = 0;
143 using AddressSpaceView = LocalAddressSpaceView;
144};
145using PrimaryAllocator = SizeClassAllocator64<AP64>;
146#endif
147using Allocator = CombinedAllocator<PrimaryAllocator>;
148using AllocatorCache = Allocator::AllocatorCache;
149} // namespace __msan
150
151static Allocator allocator;
152static AllocatorCache fallback_allocator_cache;
153static StaticSpinMutex fallback_mutex;
154
155static uptr max_malloc_size;
156
157void __msan::MsanAllocatorInit() {
158 SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
159 allocator.Init(release_to_os_interval_ms: common_flags()->allocator_release_to_os_interval_ms);
160 if (common_flags()->max_allocation_size_mb)
161 max_malloc_size = Min(a: common_flags()->max_allocation_size_mb << 20,
162 b: kMaxAllowedMallocSize);
163 else
164 max_malloc_size = kMaxAllowedMallocSize;
165}
166
167void __msan::LockAllocator() { allocator.ForceLock(); }
168
169void __msan::UnlockAllocator() { allocator.ForceUnlock(); }
170
171AllocatorCache *GetAllocatorCache(MsanThreadLocalMallocStorage *ms) {
172 CHECK_LE(sizeof(AllocatorCache), sizeof(ms->allocator_cache));
173 return reinterpret_cast<AllocatorCache *>(ms->allocator_cache);
174}
175
176void MsanThreadLocalMallocStorage::Init() {
177 allocator.InitCache(cache: GetAllocatorCache(ms: this));
178}
179
180void MsanThreadLocalMallocStorage::CommitBack() {
181 allocator.SwallowCache(cache: GetAllocatorCache(ms: this));
182 allocator.DestroyCache(cache: GetAllocatorCache(ms: this));
183}
184
185static void *MsanAllocate(BufferedStackTrace *stack, uptr size, uptr alignment,
186 bool zero) {
187 if (UNLIKELY(size > max_malloc_size)) {
188 if (AllocatorMayReturnNull()) {
189 Report(format: "WARNING: MemorySanitizer failed to allocate 0x%zx bytes\n", size);
190 return nullptr;
191 }
192 GET_FATAL_STACK_TRACE_IF_EMPTY(stack);
193 ReportAllocationSizeTooBig(user_size: size, max_size: max_malloc_size, stack);
194 }
195 if (UNLIKELY(IsRssLimitExceeded())) {
196 if (AllocatorMayReturnNull())
197 return nullptr;
198 GET_FATAL_STACK_TRACE_IF_EMPTY(stack);
199 ReportRssLimitExceeded(stack);
200 }
201 MsanThread *t = GetCurrentThread();
202 void *allocated;
203 if (t) {
204 AllocatorCache *cache = GetAllocatorCache(ms: &t->malloc_storage());
205 allocated = allocator.Allocate(cache, size, alignment);
206 } else {
207 SpinMutexLock l(&fallback_mutex);
208 AllocatorCache *cache = &fallback_allocator_cache;
209 allocated = allocator.Allocate(cache, size, alignment);
210 }
211 if (UNLIKELY(!allocated)) {
212 SetAllocatorOutOfMemory();
213 if (AllocatorMayReturnNull())
214 return nullptr;
215 GET_FATAL_STACK_TRACE_IF_EMPTY(stack);
216 ReportOutOfMemory(requested_size: size, stack);
217 }
218 auto *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(p: allocated));
219 meta->requested_size = size;
220 if (zero) {
221 if (allocator.FromPrimary(p: allocated))
222 __msan_clear_and_unpoison(a: allocated, size);
223 else
224 __msan_unpoison(a: allocated, size); // Mem is already zeroed.
225 } else if (flags()->poison_in_malloc) {
226 __msan_poison(a: allocated, size);
227 if (__msan_get_track_origins()) {
228 stack->tag = StackTrace::TAG_ALLOC;
229 Origin o = Origin::CreateHeapOrigin(stack);
230 __msan_set_origin(a: allocated, size, origin: o.raw_id());
231 }
232 }
233 UnpoisonParam(n: 2);
234 RunMallocHooks(ptr: allocated, size);
235 return allocated;
236}
237
238void __msan::MsanDeallocate(BufferedStackTrace *stack, void *p) {
239 DCHECK(p);
240 UnpoisonParam(n: 1);
241 RunFreeHooks(ptr: p);
242
243 Metadata *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(p));
244 uptr size = meta->requested_size;
245 meta->requested_size = 0;
246 // This memory will not be reused by anyone else, so we are free to keep it
247 // poisoned. The secondary allocator will unmap and unpoison by
248 // MsanMapUnmapCallback, no need to poison it here.
249 if (flags()->poison_in_free && allocator.FromPrimary(p)) {
250 __msan_poison(a: p, size);
251 if (__msan_get_track_origins()) {
252 stack->tag = StackTrace::TAG_DEALLOC;
253 Origin o = Origin::CreateHeapOrigin(stack);
254 __msan_set_origin(a: p, size, origin: o.raw_id());
255 }
256 }
257 if (MsanThread *t = GetCurrentThread()) {
258 AllocatorCache *cache = GetAllocatorCache(ms: &t->malloc_storage());
259 allocator.Deallocate(cache, p);
260 } else {
261 SpinMutexLock l(&fallback_mutex);
262 AllocatorCache *cache = &fallback_allocator_cache;
263 allocator.Deallocate(cache, p);
264 }
265}
266
267static void *MsanReallocate(BufferedStackTrace *stack, void *old_p,
268 uptr new_size, uptr alignment) {
269 Metadata *meta = reinterpret_cast<Metadata*>(allocator.GetMetaData(p: old_p));
270 uptr old_size = meta->requested_size;
271 uptr actually_allocated_size = allocator.GetActuallyAllocatedSize(p: old_p);
272 if (new_size <= actually_allocated_size) {
273 // We are not reallocating here.
274 meta->requested_size = new_size;
275 if (new_size > old_size) {
276 if (flags()->poison_in_malloc) {
277 stack->tag = StackTrace::TAG_ALLOC;
278 PoisonMemory(dst: (char *)old_p + old_size, size: new_size - old_size, stack);
279 }
280 }
281 return old_p;
282 }
283 uptr memcpy_size = Min(a: new_size, b: old_size);
284 void *new_p = MsanAllocate(stack, size: new_size, alignment, zero: false);
285 if (new_p) {
286 CopyMemory(dst: new_p, src: old_p, size: memcpy_size, stack);
287 MsanDeallocate(stack, p: old_p);
288 }
289 return new_p;
290}
291
292static void *MsanCalloc(BufferedStackTrace *stack, uptr nmemb, uptr size) {
293 if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
294 if (AllocatorMayReturnNull())
295 return nullptr;
296 GET_FATAL_STACK_TRACE_IF_EMPTY(stack);
297 ReportCallocOverflow(count: nmemb, size, stack);
298 }
299 return MsanAllocate(stack, size: nmemb * size, alignment: sizeof(u64), zero: true);
300}
301
302static const void *AllocationBegin(const void *p) {
303 if (!p)
304 return nullptr;
305 void *beg = allocator.GetBlockBegin(p);
306 if (!beg)
307 return nullptr;
308 auto *b = reinterpret_cast<Metadata *>(allocator.GetMetaData(p: beg));
309 if (!b)
310 return nullptr;
311 if (b->requested_size == 0)
312 return nullptr;
313
314 return beg;
315}
316
317static uptr AllocationSizeFast(const void *p) {
318 return reinterpret_cast<Metadata *>(allocator.GetMetaData(p))->requested_size;
319}
320
321static uptr AllocationSize(const void *p) {
322 if (!p)
323 return 0;
324 if (allocator.GetBlockBegin(p) != p)
325 return 0;
326 return AllocationSizeFast(p);
327}
328
329void *__msan::msan_malloc(uptr size, BufferedStackTrace *stack) {
330 return SetErrnoOnNull(MsanAllocate(stack, size, alignment: sizeof(u64), zero: false));
331}
332
333void *__msan::msan_calloc(uptr nmemb, uptr size, BufferedStackTrace *stack) {
334 return SetErrnoOnNull(MsanCalloc(stack, nmemb, size));
335}
336
337void *__msan::msan_realloc(void *ptr, uptr size, BufferedStackTrace *stack) {
338 if (!ptr)
339 return SetErrnoOnNull(MsanAllocate(stack, size, alignment: sizeof(u64), zero: false));
340 if (size == 0) {
341 MsanDeallocate(stack, p: ptr);
342 return nullptr;
343 }
344 return SetErrnoOnNull(MsanReallocate(stack, old_p: ptr, new_size: size, alignment: sizeof(u64)));
345}
346
347void *__msan::msan_reallocarray(void *ptr, uptr nmemb, uptr size,
348 BufferedStackTrace *stack) {
349 if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
350 errno = errno_ENOMEM;
351 if (AllocatorMayReturnNull())
352 return nullptr;
353 GET_FATAL_STACK_TRACE_IF_EMPTY(stack);
354 ReportReallocArrayOverflow(count: nmemb, size, stack);
355 }
356 return msan_realloc(ptr, size: nmemb * size, stack);
357}
358
359void *__msan::msan_valloc(uptr size, BufferedStackTrace *stack) {
360 return SetErrnoOnNull(MsanAllocate(stack, size, alignment: GetPageSizeCached(), zero: false));
361}
362
363void *__msan::msan_pvalloc(uptr size, BufferedStackTrace *stack) {
364 uptr PageSize = GetPageSizeCached();
365 if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) {
366 errno = errno_ENOMEM;
367 if (AllocatorMayReturnNull())
368 return nullptr;
369 GET_FATAL_STACK_TRACE_IF_EMPTY(stack);
370 ReportPvallocOverflow(size, stack);
371 }
372 // pvalloc(0) should allocate one page.
373 size = size ? RoundUpTo(size, boundary: PageSize) : PageSize;
374 return SetErrnoOnNull(MsanAllocate(stack, size, alignment: PageSize, zero: false));
375}
376
377void *__msan::msan_aligned_alloc(uptr alignment, uptr size,
378 BufferedStackTrace *stack) {
379 if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) {
380 errno = errno_EINVAL;
381 if (AllocatorMayReturnNull())
382 return nullptr;
383 GET_FATAL_STACK_TRACE_IF_EMPTY(stack);
384 ReportInvalidAlignedAllocAlignment(size, alignment, stack);
385 }
386 return SetErrnoOnNull(MsanAllocate(stack, size, alignment, zero: false));
387}
388
389void *__msan::msan_memalign(uptr alignment, uptr size,
390 BufferedStackTrace *stack) {
391 if (UNLIKELY(!IsPowerOfTwo(alignment))) {
392 errno = errno_EINVAL;
393 if (AllocatorMayReturnNull())
394 return nullptr;
395 GET_FATAL_STACK_TRACE_IF_EMPTY(stack);
396 ReportInvalidAllocationAlignment(alignment, stack);
397 }
398 return SetErrnoOnNull(MsanAllocate(stack, size, alignment, zero: false));
399}
400
401int __msan::msan_posix_memalign(void **memptr, uptr alignment, uptr size,
402 BufferedStackTrace *stack) {
403 if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) {
404 if (AllocatorMayReturnNull())
405 return errno_EINVAL;
406 GET_FATAL_STACK_TRACE_IF_EMPTY(stack);
407 ReportInvalidPosixMemalignAlignment(alignment, stack);
408 }
409 void *ptr = MsanAllocate(stack, size, alignment, zero: false);
410 if (UNLIKELY(!ptr))
411 // OOM error is already taken care of by MsanAllocate.
412 return errno_ENOMEM;
413 CHECK(IsAligned((uptr)ptr, alignment));
414 *memptr = ptr;
415 return 0;
416}
417
418extern "C" {
419uptr __sanitizer_get_current_allocated_bytes() {
420 uptr stats[AllocatorStatCount];
421 allocator.GetStats(s: stats);
422 return stats[AllocatorStatAllocated];
423}
424
425uptr __sanitizer_get_heap_size() {
426 uptr stats[AllocatorStatCount];
427 allocator.GetStats(s: stats);
428 return stats[AllocatorStatMapped];
429}
430
431uptr __sanitizer_get_free_bytes() { return 1; }
432
433uptr __sanitizer_get_unmapped_bytes() { return 1; }
434
435uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; }
436
437int __sanitizer_get_ownership(const void *p) { return AllocationSize(p) != 0; }
438
439const void *__sanitizer_get_allocated_begin(const void *p) {
440 return AllocationBegin(p);
441}
442
443uptr __sanitizer_get_allocated_size(const void *p) { return AllocationSize(p); }
444
445uptr __sanitizer_get_allocated_size_fast(const void *p) {
446 DCHECK_EQ(p, __sanitizer_get_allocated_begin(p));
447 uptr ret = AllocationSizeFast(p);
448 DCHECK_EQ(ret, __sanitizer_get_allocated_size(p));
449 return ret;
450}
451
452void __sanitizer_purge_allocator() { allocator.ForceReleaseToOS(); }
453}
454