1//===-- dfsan_allocator.cpp -------------------------- --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is a part of DataflowSanitizer.
10//
11// DataflowSanitizer allocator.
12//===----------------------------------------------------------------------===//
13
14#include "dfsan_allocator.h"
15
16#include "dfsan.h"
17#include "dfsan_flags.h"
18#include "dfsan_thread.h"
19#include "sanitizer_common/sanitizer_allocator.h"
20#include "sanitizer_common/sanitizer_allocator_checks.h"
21#include "sanitizer_common/sanitizer_allocator_interface.h"
22#include "sanitizer_common/sanitizer_allocator_report.h"
23#include "sanitizer_common/sanitizer_errno.h"
24
25using namespace __dfsan;
26
27namespace {
28
29struct Metadata {
30 uptr requested_size;
31};
32
33struct DFsanMapUnmapCallback {
34 void OnMap(uptr p, uptr size) const { dfsan_set_label(label: 0, addr: (void *)p, size); }
35 void OnMapSecondary(uptr p, uptr size, uptr user_begin,
36 uptr user_size) const {
37 OnMap(p, size);
38 }
39 void OnUnmap(uptr p, uptr size) const { dfsan_set_label(label: 0, addr: (void *)p, size); }
40};
41
42// Note: to ensure that the allocator is compatible with the application memory
43// layout (especially with high-entropy ASLR), kSpaceBeg and kSpaceSize must be
44// duplicated as MappingDesc::ALLOCATOR in dfsan_platform.h.
45#if defined(__aarch64__)
46const uptr kAllocatorSpace = 0xE00000000000ULL;
47const uptr kAllocatorSpaceSize = 0x40000000000; // 4T.
48#elif defined(__s390x__)
49const uptr kAllocatorSpace = 0x440000000000ULL;
50const uptr kAllocatorSpaceSize = 0x020000000000; // 2T.
51#else
52const uptr kAllocatorSpace = 0x700000000000ULL;
53const uptr kAllocatorSpaceSize = 0x40000000000; // 4T.
54#endif
55#if defined(__s390x__)
56const uptr kMaxAllowedMallocSize = 2UL << 30; // 2G.
57#else
58const uptr kMaxAllowedMallocSize = 1ULL << 40;
59#endif
60
61struct AP64 { // Allocator64 parameters. Deliberately using a short name.
62 static const uptr kSpaceBeg = kAllocatorSpace;
63 static const uptr kSpaceSize = kAllocatorSpaceSize;
64 static const uptr kMetadataSize = sizeof(Metadata);
65 using SizeClassMap = DefaultSizeClassMap;
66 using MapUnmapCallback = DFsanMapUnmapCallback;
67 static const uptr kFlags = 0;
68 using AddressSpaceView = LocalAddressSpaceView;
69};
70
71typedef SizeClassAllocator64<AP64> PrimaryAllocator;
72
73typedef CombinedAllocator<PrimaryAllocator> Allocator;
74typedef Allocator::AllocatorCache AllocatorCache;
75
76static Allocator allocator;
77static AllocatorCache fallback_allocator_cache;
78static StaticSpinMutex fallback_mutex;
79
80static uptr max_malloc_size;
81} // namespace
82
83void __dfsan::dfsan_allocator_init() {
84 SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
85 allocator.Init(release_to_os_interval_ms: common_flags()->allocator_release_to_os_interval_ms);
86 if (common_flags()->max_allocation_size_mb)
87 max_malloc_size = Min(a: common_flags()->max_allocation_size_mb << 20,
88 b: kMaxAllowedMallocSize);
89 else
90 max_malloc_size = kMaxAllowedMallocSize;
91}
92
93static AllocatorCache *GetAllocatorCache(DFsanThreadLocalMallocStorage *ms) {
94 CHECK(ms);
95 CHECK_LE(sizeof(AllocatorCache), sizeof(ms->allocator_cache));
96 return reinterpret_cast<AllocatorCache *>(ms->allocator_cache);
97}
98
99void DFsanThreadLocalMallocStorage::CommitBack() {
100 allocator.SwallowCache(cache: GetAllocatorCache(ms: this));
101}
102
103static void *DFsanAllocate(uptr size, uptr alignment, bool zeroise) {
104 if (size > max_malloc_size) {
105 if (AllocatorMayReturnNull()) {
106 Report(format: "WARNING: DataflowSanitizer failed to allocate 0x%zx bytes\n",
107 size);
108 return nullptr;
109 }
110 UNINITIALIZED BufferedStackTrace stack;
111 ReportAllocationSizeTooBig(user_size: size, max_size: max_malloc_size, stack: &stack);
112 }
113 if (UNLIKELY(IsRssLimitExceeded())) {
114 if (AllocatorMayReturnNull())
115 return nullptr;
116 UNINITIALIZED BufferedStackTrace stack;
117 ReportRssLimitExceeded(stack: &stack);
118 }
119 DFsanThread *t = GetCurrentThread();
120 void *allocated;
121 if (t) {
122 AllocatorCache *cache = GetAllocatorCache(ms: &t->malloc_storage());
123 allocated = allocator.Allocate(cache, size, alignment);
124 } else {
125 SpinMutexLock l(&fallback_mutex);
126 AllocatorCache *cache = &fallback_allocator_cache;
127 allocated = allocator.Allocate(cache, size, alignment);
128 }
129 if (UNLIKELY(!allocated)) {
130 SetAllocatorOutOfMemory();
131 if (AllocatorMayReturnNull())
132 return nullptr;
133 UNINITIALIZED BufferedStackTrace stack;
134 ReportOutOfMemory(requested_size: size, stack: &stack);
135 }
136 Metadata *meta =
137 reinterpret_cast<Metadata *>(allocator.GetMetaData(p: allocated));
138 meta->requested_size = size;
139 if (zeroise) {
140 internal_memset(s: allocated, c: 0, n: size);
141 dfsan_set_label(label: 0, addr: allocated, size);
142 } else if (flags().zero_in_malloc) {
143 dfsan_set_label(label: 0, addr: allocated, size);
144 }
145 return allocated;
146}
147
148void __dfsan::dfsan_deallocate(void *p) {
149 CHECK(p);
150 Metadata *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(p));
151 uptr size = meta->requested_size;
152 meta->requested_size = 0;
153 if (flags().zero_in_free)
154 dfsan_set_label(label: 0, addr: p, size);
155 DFsanThread *t = GetCurrentThread();
156 if (t) {
157 AllocatorCache *cache = GetAllocatorCache(ms: &t->malloc_storage());
158 allocator.Deallocate(cache, p);
159 } else {
160 SpinMutexLock l(&fallback_mutex);
161 AllocatorCache *cache = &fallback_allocator_cache;
162 allocator.Deallocate(cache, p);
163 }
164}
165
166static void *DFsanReallocate(void *old_p, uptr new_size, uptr alignment) {
167 Metadata *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(p: old_p));
168 uptr old_size = meta->requested_size;
169 uptr actually_allocated_size = allocator.GetActuallyAllocatedSize(p: old_p);
170 if (new_size <= actually_allocated_size) {
171 // We are not reallocating here.
172 meta->requested_size = new_size;
173 if (new_size > old_size && flags().zero_in_malloc)
174 dfsan_set_label(label: 0, addr: (char *)old_p + old_size, size: new_size - old_size);
175 return old_p;
176 }
177 uptr memcpy_size = Min(a: new_size, b: old_size);
178 void *new_p = DFsanAllocate(size: new_size, alignment, zeroise: false /*zeroise*/);
179 if (new_p) {
180 dfsan_copy_memory(dst: new_p, src: old_p, size: memcpy_size);
181 dfsan_deallocate(p: old_p);
182 }
183 return new_p;
184}
185
186static void *DFsanCalloc(uptr nmemb, uptr size) {
187 if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
188 if (AllocatorMayReturnNull())
189 return nullptr;
190 UNINITIALIZED BufferedStackTrace stack;
191 ReportCallocOverflow(count: nmemb, size, stack: &stack);
192 }
193 return DFsanAllocate(size: nmemb * size, alignment: sizeof(u64), zeroise: true /*zeroise*/);
194}
195
196static const void *AllocationBegin(const void *p) {
197 if (!p)
198 return nullptr;
199 void *beg = allocator.GetBlockBegin(p);
200 if (!beg)
201 return nullptr;
202 Metadata *b = (Metadata *)allocator.GetMetaData(p: beg);
203 if (!b)
204 return nullptr;
205 if (b->requested_size == 0)
206 return nullptr;
207 return (const void *)beg;
208}
209
210static uptr AllocationSize(const void *p) {
211 if (!p)
212 return 0;
213 const void *beg = allocator.GetBlockBegin(p);
214 if (beg != p)
215 return 0;
216 Metadata *b = (Metadata *)allocator.GetMetaData(p);
217 return b->requested_size;
218}
219
220static uptr AllocationSizeFast(const void *p) {
221 return reinterpret_cast<Metadata *>(allocator.GetMetaData(p))->requested_size;
222}
223
224void *__dfsan::dfsan_malloc(uptr size) {
225 return SetErrnoOnNull(DFsanAllocate(size, alignment: sizeof(u64), zeroise: false /*zeroise*/));
226}
227
228void *__dfsan::dfsan_calloc(uptr nmemb, uptr size) {
229 return SetErrnoOnNull(DFsanCalloc(nmemb, size));
230}
231
232void *__dfsan::dfsan_realloc(void *ptr, uptr size) {
233 if (!ptr)
234 return SetErrnoOnNull(DFsanAllocate(size, alignment: sizeof(u64), zeroise: false /*zeroise*/));
235 if (size == 0) {
236 dfsan_deallocate(p: ptr);
237 return nullptr;
238 }
239 return SetErrnoOnNull(DFsanReallocate(old_p: ptr, new_size: size, alignment: sizeof(u64)));
240}
241
242void *__dfsan::dfsan_reallocarray(void *ptr, uptr nmemb, uptr size) {
243 if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
244 errno = errno_ENOMEM;
245 if (AllocatorMayReturnNull())
246 return nullptr;
247 UNINITIALIZED BufferedStackTrace stack;
248 ReportReallocArrayOverflow(count: nmemb, size, stack: &stack);
249 }
250 return dfsan_realloc(ptr, size: nmemb * size);
251}
252
253void *__dfsan::dfsan_valloc(uptr size) {
254 return SetErrnoOnNull(
255 DFsanAllocate(size, alignment: GetPageSizeCached(), zeroise: false /*zeroise*/));
256}
257
258void *__dfsan::dfsan_pvalloc(uptr size) {
259 uptr PageSize = GetPageSizeCached();
260 if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) {
261 errno = errno_ENOMEM;
262 if (AllocatorMayReturnNull())
263 return nullptr;
264 UNINITIALIZED BufferedStackTrace stack;
265 ReportPvallocOverflow(size, stack: &stack);
266 }
267 // pvalloc(0) should allocate one page.
268 size = size ? RoundUpTo(size, boundary: PageSize) : PageSize;
269 return SetErrnoOnNull(DFsanAllocate(size, alignment: PageSize, zeroise: false /*zeroise*/));
270}
271
272void *__dfsan::dfsan_aligned_alloc(uptr alignment, uptr size) {
273 if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) {
274 errno = errno_EINVAL;
275 if (AllocatorMayReturnNull())
276 return nullptr;
277 UNINITIALIZED BufferedStackTrace stack;
278 ReportInvalidAlignedAllocAlignment(size, alignment, stack: &stack);
279 }
280 return SetErrnoOnNull(DFsanAllocate(size, alignment, zeroise: false /*zeroise*/));
281}
282
283void *__dfsan::dfsan_memalign(uptr alignment, uptr size) {
284 if (UNLIKELY(!IsPowerOfTwo(alignment))) {
285 errno = errno_EINVAL;
286 if (AllocatorMayReturnNull())
287 return nullptr;
288 UNINITIALIZED BufferedStackTrace stack;
289 ReportInvalidAllocationAlignment(alignment, stack: &stack);
290 }
291 return SetErrnoOnNull(DFsanAllocate(size, alignment, zeroise: false /*zeroise*/));
292}
293
294int __dfsan::dfsan_posix_memalign(void **memptr, uptr alignment, uptr size) {
295 if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) {
296 if (AllocatorMayReturnNull())
297 return errno_EINVAL;
298 UNINITIALIZED BufferedStackTrace stack;
299 ReportInvalidPosixMemalignAlignment(alignment, stack: &stack);
300 }
301 void *ptr = DFsanAllocate(size, alignment, zeroise: false /*zeroise*/);
302 if (UNLIKELY(!ptr))
303 // OOM error is already taken care of by DFsanAllocate.
304 return errno_ENOMEM;
305 CHECK(IsAligned((uptr)ptr, alignment));
306 *memptr = ptr;
307 return 0;
308}
309
310extern "C" {
311uptr __sanitizer_get_current_allocated_bytes() {
312 uptr stats[AllocatorStatCount];
313 allocator.GetStats(s: stats);
314 return stats[AllocatorStatAllocated];
315}
316
317uptr __sanitizer_get_heap_size() {
318 uptr stats[AllocatorStatCount];
319 allocator.GetStats(s: stats);
320 return stats[AllocatorStatMapped];
321}
322
323uptr __sanitizer_get_free_bytes() { return 1; }
324
325uptr __sanitizer_get_unmapped_bytes() { return 1; }
326
327uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; }
328
329int __sanitizer_get_ownership(const void *p) { return AllocationSize(p) != 0; }
330
331const void *__sanitizer_get_allocated_begin(const void *p) {
332 return AllocationBegin(p);
333}
334
335uptr __sanitizer_get_allocated_size(const void *p) { return AllocationSize(p); }
336
337uptr __sanitizer_get_allocated_size_fast(const void *p) {
338 DCHECK_EQ(p, __sanitizer_get_allocated_begin(p));
339 uptr ret = AllocationSizeFast(p);
340 DCHECK_EQ(ret, __sanitizer_get_allocated_size(p));
341 return ret;
342}
343}
344