1//===-- dfsan_allocator.cpp -------------------------- --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is a part of DataflowSanitizer.
10//
11// DataflowSanitizer allocator.
12//===----------------------------------------------------------------------===//
13
14#include "dfsan_allocator.h"
15
16#include "dfsan.h"
17#include "dfsan_flags.h"
18#include "dfsan_thread.h"
19#include "sanitizer_common/sanitizer_allocator.h"
20#include "sanitizer_common/sanitizer_allocator_checks.h"
21#include "sanitizer_common/sanitizer_allocator_interface.h"
22#include "sanitizer_common/sanitizer_allocator_report.h"
23#include "sanitizer_common/sanitizer_errno.h"
24
25using namespace __dfsan;
26
27namespace {
28
29struct Metadata {
30 uptr requested_size;
31};
32
33struct DFsanMapUnmapCallback {
34 void OnMap(uptr p, uptr size) const { dfsan_set_label(label: 0, addr: (void *)p, size); }
35 void OnMapSecondary(uptr p, uptr size, uptr user_begin,
36 uptr user_size) const {
37 OnMap(p, size);
38 }
39 void OnUnmap(uptr p, uptr size) const { dfsan_set_label(label: 0, addr: (void *)p, size); }
40};
41
42// Note: to ensure that the allocator is compatible with the application memory
43// layout (especially with high-entropy ASLR), kSpaceBeg and kSpaceSize must be
44// duplicated as MappingDesc::ALLOCATOR in dfsan_platform.h.
45#if defined(__aarch64__)
46const uptr kAllocatorSpace = 0xE00000000000ULL;
47#else
48const uptr kAllocatorSpace = 0x700000000000ULL;
49#endif
50const uptr kMaxAllowedMallocSize = 1ULL << 40;
51
52struct AP64 { // Allocator64 parameters. Deliberately using a short name.
53 static const uptr kSpaceBeg = kAllocatorSpace;
54 static const uptr kSpaceSize = 0x40000000000; // 4T.
55 static const uptr kMetadataSize = sizeof(Metadata);
56 typedef DefaultSizeClassMap SizeClassMap;
57 typedef DFsanMapUnmapCallback MapUnmapCallback;
58 static const uptr kFlags = 0;
59 using AddressSpaceView = LocalAddressSpaceView;
60};
61
62typedef SizeClassAllocator64<AP64> PrimaryAllocator;
63
64typedef CombinedAllocator<PrimaryAllocator> Allocator;
65typedef Allocator::AllocatorCache AllocatorCache;
66
67static Allocator allocator;
68static AllocatorCache fallback_allocator_cache;
69static StaticSpinMutex fallback_mutex;
70
71static uptr max_malloc_size;
72} // namespace
73
74void __dfsan::dfsan_allocator_init() {
75 SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
76 allocator.Init(release_to_os_interval_ms: common_flags()->allocator_release_to_os_interval_ms);
77 if (common_flags()->max_allocation_size_mb)
78 max_malloc_size = Min(a: common_flags()->max_allocation_size_mb << 20,
79 b: kMaxAllowedMallocSize);
80 else
81 max_malloc_size = kMaxAllowedMallocSize;
82}
83
84static AllocatorCache *GetAllocatorCache(DFsanThreadLocalMallocStorage *ms) {
85 CHECK(ms);
86 CHECK_LE(sizeof(AllocatorCache), sizeof(ms->allocator_cache));
87 return reinterpret_cast<AllocatorCache *>(ms->allocator_cache);
88}
89
90void DFsanThreadLocalMallocStorage::CommitBack() {
91 allocator.SwallowCache(cache: GetAllocatorCache(ms: this));
92}
93
94static void *DFsanAllocate(uptr size, uptr alignment, bool zeroise) {
95 if (size > max_malloc_size) {
96 if (AllocatorMayReturnNull()) {
97 Report(format: "WARNING: DataflowSanitizer failed to allocate 0x%zx bytes\n",
98 size);
99 return nullptr;
100 }
101 UNINITIALIZED BufferedStackTrace stack;
102 ReportAllocationSizeTooBig(user_size: size, max_size: max_malloc_size, stack: &stack);
103 }
104 if (UNLIKELY(IsRssLimitExceeded())) {
105 if (AllocatorMayReturnNull())
106 return nullptr;
107 UNINITIALIZED BufferedStackTrace stack;
108 ReportRssLimitExceeded(stack: &stack);
109 }
110 DFsanThread *t = GetCurrentThread();
111 void *allocated;
112 if (t) {
113 AllocatorCache *cache = GetAllocatorCache(ms: &t->malloc_storage());
114 allocated = allocator.Allocate(cache, size, alignment);
115 } else {
116 SpinMutexLock l(&fallback_mutex);
117 AllocatorCache *cache = &fallback_allocator_cache;
118 allocated = allocator.Allocate(cache, size, alignment);
119 }
120 if (UNLIKELY(!allocated)) {
121 SetAllocatorOutOfMemory();
122 if (AllocatorMayReturnNull())
123 return nullptr;
124 UNINITIALIZED BufferedStackTrace stack;
125 ReportOutOfMemory(requested_size: size, stack: &stack);
126 }
127 Metadata *meta =
128 reinterpret_cast<Metadata *>(allocator.GetMetaData(p: allocated));
129 meta->requested_size = size;
130 if (zeroise) {
131 internal_memset(s: allocated, c: 0, n: size);
132 dfsan_set_label(label: 0, addr: allocated, size);
133 } else if (flags().zero_in_malloc) {
134 dfsan_set_label(label: 0, addr: allocated, size);
135 }
136 return allocated;
137}
138
139void __dfsan::dfsan_deallocate(void *p) {
140 CHECK(p);
141 Metadata *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(p));
142 uptr size = meta->requested_size;
143 meta->requested_size = 0;
144 if (flags().zero_in_free)
145 dfsan_set_label(label: 0, addr: p, size);
146 DFsanThread *t = GetCurrentThread();
147 if (t) {
148 AllocatorCache *cache = GetAllocatorCache(ms: &t->malloc_storage());
149 allocator.Deallocate(cache, p);
150 } else {
151 SpinMutexLock l(&fallback_mutex);
152 AllocatorCache *cache = &fallback_allocator_cache;
153 allocator.Deallocate(cache, p);
154 }
155}
156
157static void *DFsanReallocate(void *old_p, uptr new_size, uptr alignment) {
158 Metadata *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(p: old_p));
159 uptr old_size = meta->requested_size;
160 uptr actually_allocated_size = allocator.GetActuallyAllocatedSize(p: old_p);
161 if (new_size <= actually_allocated_size) {
162 // We are not reallocating here.
163 meta->requested_size = new_size;
164 if (new_size > old_size && flags().zero_in_malloc)
165 dfsan_set_label(label: 0, addr: (char *)old_p + old_size, size: new_size - old_size);
166 return old_p;
167 }
168 uptr memcpy_size = Min(a: new_size, b: old_size);
169 void *new_p = DFsanAllocate(size: new_size, alignment, zeroise: false /*zeroise*/);
170 if (new_p) {
171 dfsan_copy_memory(dst: new_p, src: old_p, size: memcpy_size);
172 dfsan_deallocate(p: old_p);
173 }
174 return new_p;
175}
176
177static void *DFsanCalloc(uptr nmemb, uptr size) {
178 if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
179 if (AllocatorMayReturnNull())
180 return nullptr;
181 UNINITIALIZED BufferedStackTrace stack;
182 ReportCallocOverflow(count: nmemb, size, stack: &stack);
183 }
184 return DFsanAllocate(size: nmemb * size, alignment: sizeof(u64), zeroise: true /*zeroise*/);
185}
186
187static const void *AllocationBegin(const void *p) {
188 if (!p)
189 return nullptr;
190 void *beg = allocator.GetBlockBegin(p);
191 if (!beg)
192 return nullptr;
193 Metadata *b = (Metadata *)allocator.GetMetaData(p: beg);
194 if (!b)
195 return nullptr;
196 if (b->requested_size == 0)
197 return nullptr;
198 return (const void *)beg;
199}
200
201static uptr AllocationSize(const void *p) {
202 if (!p)
203 return 0;
204 const void *beg = allocator.GetBlockBegin(p);
205 if (beg != p)
206 return 0;
207 Metadata *b = (Metadata *)allocator.GetMetaData(p);
208 return b->requested_size;
209}
210
211static uptr AllocationSizeFast(const void *p) {
212 return reinterpret_cast<Metadata *>(allocator.GetMetaData(p))->requested_size;
213}
214
215void *__dfsan::dfsan_malloc(uptr size) {
216 return SetErrnoOnNull(DFsanAllocate(size, alignment: sizeof(u64), zeroise: false /*zeroise*/));
217}
218
219void *__dfsan::dfsan_calloc(uptr nmemb, uptr size) {
220 return SetErrnoOnNull(DFsanCalloc(nmemb, size));
221}
222
223void *__dfsan::dfsan_realloc(void *ptr, uptr size) {
224 if (!ptr)
225 return SetErrnoOnNull(DFsanAllocate(size, alignment: sizeof(u64), zeroise: false /*zeroise*/));
226 if (size == 0) {
227 dfsan_deallocate(p: ptr);
228 return nullptr;
229 }
230 return SetErrnoOnNull(DFsanReallocate(old_p: ptr, new_size: size, alignment: sizeof(u64)));
231}
232
233void *__dfsan::dfsan_reallocarray(void *ptr, uptr nmemb, uptr size) {
234 if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
235 errno = errno_ENOMEM;
236 if (AllocatorMayReturnNull())
237 return nullptr;
238 UNINITIALIZED BufferedStackTrace stack;
239 ReportReallocArrayOverflow(count: nmemb, size, stack: &stack);
240 }
241 return dfsan_realloc(ptr, size: nmemb * size);
242}
243
244void *__dfsan::dfsan_valloc(uptr size) {
245 return SetErrnoOnNull(
246 DFsanAllocate(size, alignment: GetPageSizeCached(), zeroise: false /*zeroise*/));
247}
248
249void *__dfsan::dfsan_pvalloc(uptr size) {
250 uptr PageSize = GetPageSizeCached();
251 if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) {
252 errno = errno_ENOMEM;
253 if (AllocatorMayReturnNull())
254 return nullptr;
255 UNINITIALIZED BufferedStackTrace stack;
256 ReportPvallocOverflow(size, stack: &stack);
257 }
258 // pvalloc(0) should allocate one page.
259 size = size ? RoundUpTo(size, boundary: PageSize) : PageSize;
260 return SetErrnoOnNull(DFsanAllocate(size, alignment: PageSize, zeroise: false /*zeroise*/));
261}
262
263void *__dfsan::dfsan_aligned_alloc(uptr alignment, uptr size) {
264 if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) {
265 errno = errno_EINVAL;
266 if (AllocatorMayReturnNull())
267 return nullptr;
268 UNINITIALIZED BufferedStackTrace stack;
269 ReportInvalidAlignedAllocAlignment(size, alignment, stack: &stack);
270 }
271 return SetErrnoOnNull(DFsanAllocate(size, alignment, zeroise: false /*zeroise*/));
272}
273
274void *__dfsan::dfsan_memalign(uptr alignment, uptr size) {
275 if (UNLIKELY(!IsPowerOfTwo(alignment))) {
276 errno = errno_EINVAL;
277 if (AllocatorMayReturnNull())
278 return nullptr;
279 UNINITIALIZED BufferedStackTrace stack;
280 ReportInvalidAllocationAlignment(alignment, stack: &stack);
281 }
282 return SetErrnoOnNull(DFsanAllocate(size, alignment, zeroise: false /*zeroise*/));
283}
284
285int __dfsan::dfsan_posix_memalign(void **memptr, uptr alignment, uptr size) {
286 if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) {
287 if (AllocatorMayReturnNull())
288 return errno_EINVAL;
289 UNINITIALIZED BufferedStackTrace stack;
290 ReportInvalidPosixMemalignAlignment(alignment, stack: &stack);
291 }
292 void *ptr = DFsanAllocate(size, alignment, zeroise: false /*zeroise*/);
293 if (UNLIKELY(!ptr))
294 // OOM error is already taken care of by DFsanAllocate.
295 return errno_ENOMEM;
296 CHECK(IsAligned((uptr)ptr, alignment));
297 *memptr = ptr;
298 return 0;
299}
300
301extern "C" {
302uptr __sanitizer_get_current_allocated_bytes() {
303 uptr stats[AllocatorStatCount];
304 allocator.GetStats(s: stats);
305 return stats[AllocatorStatAllocated];
306}
307
308uptr __sanitizer_get_heap_size() {
309 uptr stats[AllocatorStatCount];
310 allocator.GetStats(s: stats);
311 return stats[AllocatorStatMapped];
312}
313
314uptr __sanitizer_get_free_bytes() { return 1; }
315
316uptr __sanitizer_get_unmapped_bytes() { return 1; }
317
318uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; }
319
320int __sanitizer_get_ownership(const void *p) { return AllocationSize(p) != 0; }
321
322const void *__sanitizer_get_allocated_begin(const void *p) {
323 return AllocationBegin(p);
324}
325
326uptr __sanitizer_get_allocated_size(const void *p) { return AllocationSize(p); }
327
328uptr __sanitizer_get_allocated_size_fast(const void *p) {
329 DCHECK_EQ(p, __sanitizer_get_allocated_begin(p));
330 uptr ret = AllocationSizeFast(p);
331 DCHECK_EQ(ret, __sanitizer_get_allocated_size(p));
332 return ret;
333}
334}
335