1//===- nsan_allocator.cpp -------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// NumericalStabilitySanitizer allocator.
10//
11//===----------------------------------------------------------------------===//
12
13#include "nsan_allocator.h"
14#include "interception/interception.h"
15#include "nsan.h"
16#include "nsan_flags.h"
17#include "nsan_platform.h"
18#include "nsan_thread.h"
19#include "sanitizer_common/sanitizer_allocator.h"
20#include "sanitizer_common/sanitizer_allocator_checks.h"
21#include "sanitizer_common/sanitizer_allocator_interface.h"
22#include "sanitizer_common/sanitizer_allocator_report.h"
23#include "sanitizer_common/sanitizer_common.h"
24#include "sanitizer_common/sanitizer_errno.h"
25
26using namespace __nsan;
27
28DECLARE_REAL(void *, memcpy, void *dest, const void *src, SIZE_T n)
29DECLARE_REAL(void *, memset, void *dest, int c, SIZE_T n)
30
31namespace {
32struct Metadata {
33 uptr requested_size;
34};
35
36struct NsanMapUnmapCallback {
37 void OnMap(uptr p, uptr size) const {}
38 void OnMapSecondary(uptr p, uptr size, uptr user_begin,
39 uptr user_size) const {}
40 void OnUnmap(uptr p, uptr size) const {}
41};
42
43const uptr kMaxAllowedMallocSize = 1ULL << 40;
44
45// Allocator64 parameters. Deliberately using a short name.
46struct AP64 {
47 static const uptr kSpaceBeg = Mapping::kHeapMemBeg;
48 static const uptr kSpaceSize = 0x40000000000; // 4T.
49 static const uptr kMetadataSize = sizeof(Metadata);
50 using SizeClassMap = DefaultSizeClassMap;
51 using MapUnmapCallback = NsanMapUnmapCallback;
52 static const uptr kFlags = 0;
53 using AddressSpaceView = LocalAddressSpaceView;
54};
55} // namespace
56
57using PrimaryAllocator = SizeClassAllocator64<AP64>;
58using Allocator = CombinedAllocator<PrimaryAllocator>;
59using AllocatorCache = Allocator::AllocatorCache;
60
61static Allocator allocator;
62static AllocatorCache fallback_allocator_cache;
63static StaticSpinMutex fallback_mutex;
64
65static uptr max_malloc_size;
66
67void __nsan::NsanAllocatorInit() {
68 SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
69 allocator.Init(release_to_os_interval_ms: common_flags()->allocator_release_to_os_interval_ms);
70 if (common_flags()->max_allocation_size_mb)
71 max_malloc_size = Min(a: common_flags()->max_allocation_size_mb << 20,
72 b: kMaxAllowedMallocSize);
73 else
74 max_malloc_size = kMaxAllowedMallocSize;
75}
76
77static AllocatorCache *GetAllocatorCache(NsanThreadLocalMallocStorage *ms) {
78 CHECK_LE(sizeof(AllocatorCache), sizeof(ms->allocator_cache));
79 return reinterpret_cast<AllocatorCache *>(ms->allocator_cache);
80}
81
82void NsanThreadLocalMallocStorage::Init() {
83 allocator.InitCache(cache: GetAllocatorCache(ms: this));
84}
85
86void NsanThreadLocalMallocStorage::CommitBack() {
87 allocator.SwallowCache(cache: GetAllocatorCache(ms: this));
88 allocator.DestroyCache(cache: GetAllocatorCache(ms: this));
89}
90
91static void *NsanAllocate(uptr size, uptr alignment, bool zero) {
92 if (UNLIKELY(size > max_malloc_size)) {
93 if (AllocatorMayReturnNull()) {
94 Report(format: "WARNING: NumericalStabilitySanitizer failed to allocate 0x%zx "
95 "bytes\n",
96 size);
97 return nullptr;
98 }
99 BufferedStackTrace stack;
100 GET_FATAL_STACK_TRACE_IF_EMPTY(&stack);
101 ReportAllocationSizeTooBig(user_size: size, max_size: max_malloc_size, stack: &stack);
102 }
103 if (UNLIKELY(IsRssLimitExceeded())) {
104 if (AllocatorMayReturnNull())
105 return nullptr;
106 BufferedStackTrace stack;
107 GET_FATAL_STACK_TRACE_IF_EMPTY(&stack);
108 ReportRssLimitExceeded(stack: &stack);
109 }
110
111 void *allocated;
112 if (NsanThread *t = GetCurrentThread()) {
113 AllocatorCache *cache = GetAllocatorCache(ms: &t->malloc_storage());
114 allocated = allocator.Allocate(cache, size, alignment);
115 } else {
116 SpinMutexLock l(&fallback_mutex);
117 AllocatorCache *cache = &fallback_allocator_cache;
118 allocated = allocator.Allocate(cache, size, alignment);
119 }
120 if (UNLIKELY(!allocated)) {
121 SetAllocatorOutOfMemory();
122 if (AllocatorMayReturnNull())
123 return nullptr;
124 BufferedStackTrace stack;
125 GET_FATAL_STACK_TRACE_IF_EMPTY(&stack);
126 ReportOutOfMemory(requested_size: size, stack: &stack);
127 }
128 auto *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(p: allocated));
129 meta->requested_size = size;
130 if (zero && allocator.FromPrimary(p: allocated))
131 REAL(memset)(allocated, 0, size);
132 __nsan_set_value_unknown(addr: allocated, size);
133 RunMallocHooks(ptr: allocated, size);
134 return allocated;
135}
136
137void __nsan::NsanDeallocate(void *p) {
138 DCHECK(p);
139 RunFreeHooks(ptr: p);
140 auto *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(p));
141 uptr size = meta->requested_size;
142 meta->requested_size = 0;
143 if (flags().poison_in_free)
144 __nsan_set_value_unknown(addr: p, size);
145 if (NsanThread *t = GetCurrentThread()) {
146 AllocatorCache *cache = GetAllocatorCache(ms: &t->malloc_storage());
147 allocator.Deallocate(cache, p);
148 } else {
149 // In a just created thread, glibc's _dl_deallocate_tls might reach here
150 // before nsan_current_thread is set.
151 SpinMutexLock l(&fallback_mutex);
152 AllocatorCache *cache = &fallback_allocator_cache;
153 allocator.Deallocate(cache, p);
154 }
155}
156
157static void *NsanReallocate(void *ptr, uptr new_size, uptr alignment) {
158 Metadata *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(p: ptr));
159 uptr old_size = meta->requested_size;
160 uptr actually_allocated_size = allocator.GetActuallyAllocatedSize(p: ptr);
161 if (new_size <= actually_allocated_size) {
162 // We are not reallocating here.
163 meta->requested_size = new_size;
164 if (new_size > old_size)
165 __nsan_set_value_unknown(addr: (u8 *)ptr + old_size, size: new_size - old_size);
166 return ptr;
167 }
168 void *new_p = NsanAllocate(size: new_size, alignment, zero: false);
169 if (new_p) {
170 uptr memcpy_size = Min(a: new_size, b: old_size);
171 REAL(memcpy)(new_p, ptr, memcpy_size);
172 __nsan_copy_values(daddr: new_p, saddr: ptr, size: memcpy_size);
173 NsanDeallocate(p: ptr);
174 }
175 return new_p;
176}
177
178static void *NsanCalloc(uptr nmemb, uptr size) {
179 if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
180 if (AllocatorMayReturnNull())
181 return nullptr;
182 BufferedStackTrace stack;
183 GET_FATAL_STACK_TRACE_IF_EMPTY(&stack);
184 ReportCallocOverflow(count: nmemb, size, stack: &stack);
185 }
186 return NsanAllocate(size: nmemb * size, alignment: sizeof(u64), zero: true);
187}
188
189static const void *AllocationBegin(const void *p) {
190 if (!p)
191 return nullptr;
192 void *beg = allocator.GetBlockBegin(p);
193 if (!beg)
194 return nullptr;
195 auto *b = reinterpret_cast<Metadata *>(allocator.GetMetaData(p: beg));
196 if (!b)
197 return nullptr;
198 if (b->requested_size == 0)
199 return nullptr;
200
201 return beg;
202}
203
204static uptr AllocationSizeFast(const void *p) {
205 return reinterpret_cast<Metadata *>(allocator.GetMetaData(p))->requested_size;
206}
207
208static uptr AllocationSize(const void *p) {
209 if (!p)
210 return 0;
211 if (allocator.GetBlockBegin(p) != p)
212 return 0;
213 return AllocationSizeFast(p);
214}
215
216void *__nsan::nsan_malloc(uptr size) {
217 return SetErrnoOnNull(NsanAllocate(size, alignment: sizeof(u64), zero: false));
218}
219
220void *__nsan::nsan_calloc(uptr nmemb, uptr size) {
221 return SetErrnoOnNull(NsanCalloc(nmemb, size));
222}
223
224void *__nsan::nsan_realloc(void *ptr, uptr size) {
225 if (!ptr)
226 return SetErrnoOnNull(NsanAllocate(size, alignment: sizeof(u64), zero: false));
227 if (size == 0) {
228 NsanDeallocate(p: ptr);
229 return nullptr;
230 }
231 return SetErrnoOnNull(NsanReallocate(ptr, new_size: size, alignment: sizeof(u64)));
232}
233
234void *__nsan::nsan_reallocarray(void *ptr, uptr nmemb, uptr size) {
235 if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
236 errno = errno_ENOMEM;
237 if (AllocatorMayReturnNull())
238 return nullptr;
239 BufferedStackTrace stack;
240 GET_FATAL_STACK_TRACE_IF_EMPTY(&stack);
241 ReportReallocArrayOverflow(count: nmemb, size, stack: &stack);
242 }
243 return nsan_realloc(ptr, size: nmemb * size);
244}
245
246void *__nsan::nsan_valloc(uptr size) {
247 return SetErrnoOnNull(NsanAllocate(size, alignment: GetPageSizeCached(), zero: false));
248}
249
250void *__nsan::nsan_pvalloc(uptr size) {
251 uptr PageSize = GetPageSizeCached();
252 if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) {
253 errno = errno_ENOMEM;
254 if (AllocatorMayReturnNull())
255 return nullptr;
256 BufferedStackTrace stack;
257 GET_FATAL_STACK_TRACE_IF_EMPTY(&stack);
258 ReportPvallocOverflow(size, stack: &stack);
259 }
260 // pvalloc(0) should allocate one page.
261 size = size ? RoundUpTo(size, boundary: PageSize) : PageSize;
262 return SetErrnoOnNull(NsanAllocate(size, alignment: PageSize, zero: false));
263}
264
265void *__nsan::nsan_aligned_alloc(uptr alignment, uptr size) {
266 if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) {
267 errno = errno_EINVAL;
268 if (AllocatorMayReturnNull())
269 return nullptr;
270 BufferedStackTrace stack;
271 GET_FATAL_STACK_TRACE_IF_EMPTY(&stack);
272 ReportInvalidAlignedAllocAlignment(size, alignment, stack: &stack);
273 }
274 return SetErrnoOnNull(NsanAllocate(size, alignment, zero: false));
275}
276
277void *__nsan::nsan_memalign(uptr alignment, uptr size) {
278 if (UNLIKELY(!IsPowerOfTwo(alignment))) {
279 errno = errno_EINVAL;
280 if (AllocatorMayReturnNull())
281 return nullptr;
282 BufferedStackTrace stack;
283 GET_FATAL_STACK_TRACE_IF_EMPTY(&stack);
284 ReportInvalidAllocationAlignment(alignment, stack: &stack);
285 }
286 return SetErrnoOnNull(NsanAllocate(size, alignment, zero: false));
287}
288
289int __nsan::nsan_posix_memalign(void **memptr, uptr alignment, uptr size) {
290 if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) {
291 if (AllocatorMayReturnNull())
292 return errno_EINVAL;
293 BufferedStackTrace stack;
294 ReportInvalidPosixMemalignAlignment(alignment, stack: &stack);
295 }
296 void *ptr = NsanAllocate(size, alignment, zero: false);
297 if (UNLIKELY(!ptr))
298 // OOM error is already taken care of by NsanAllocate.
299 return errno_ENOMEM;
300 DCHECK(IsAligned((uptr)ptr, alignment));
301 *memptr = ptr;
302 return 0;
303}
304
305extern "C" {
306uptr __sanitizer_get_current_allocated_bytes() {
307 uptr stats[AllocatorStatCount];
308 allocator.GetStats(s: stats);
309 return stats[AllocatorStatAllocated];
310}
311
312uptr __sanitizer_get_heap_size() {
313 uptr stats[AllocatorStatCount];
314 allocator.GetStats(s: stats);
315 return stats[AllocatorStatMapped];
316}
317
318uptr __sanitizer_get_free_bytes() { return 1; }
319
320uptr __sanitizer_get_unmapped_bytes() { return 1; }
321
322uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; }
323
324int __sanitizer_get_ownership(const void *p) { return AllocationSize(p) != 0; }
325
326const void *__sanitizer_get_allocated_begin(const void *p) {
327 return AllocationBegin(p);
328}
329
330uptr __sanitizer_get_allocated_size(const void *p) { return AllocationSize(p); }
331
332uptr __sanitizer_get_allocated_size_fast(const void *p) {
333 DCHECK_EQ(p, __sanitizer_get_allocated_begin(p));
334 uptr ret = AllocationSizeFast(p);
335 DCHECK_EQ(ret, __sanitizer_get_allocated_size(p));
336 return ret;
337}
338
339void __sanitizer_purge_allocator() { allocator.ForceReleaseToOS(); }
340}
341