1//===-- asan_allocator.h ----------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is a part of AddressSanitizer, an address sanity checker.
10//
11// ASan-private header for asan_allocator.cpp.
12//===----------------------------------------------------------------------===//
13
14#ifndef ASAN_ALLOCATOR_H
15#define ASAN_ALLOCATOR_H
16
17#include "asan_flags.h"
18#include "asan_interceptors.h"
19#include "asan_internal.h"
20#include "sanitizer_common/sanitizer_allocator.h"
21#include "sanitizer_common/sanitizer_list.h"
22#include "sanitizer_common/sanitizer_platform.h"
23
24namespace __asan {
25
26enum AllocType {
27 FROM_MALLOC = 1, // Memory block came from malloc, calloc, realloc, etc.
28 FROM_NEW = 2, // Memory block came from operator new.
29 FROM_NEW_BR = 3 // Memory block came from operator new [ ]
30};
31
32class AsanChunk;
33
34struct AllocatorOptions {
35 u32 quarantine_size_mb;
36 u32 thread_local_quarantine_size_kb;
37 u16 min_redzone;
38 u16 max_redzone;
39 u8 may_return_null;
40 u8 alloc_dealloc_mismatch;
41 s32 release_to_os_interval_ms;
42
43 void SetFrom(const Flags* f, const CommonFlags* cf);
44 void CopyTo(Flags* f, CommonFlags* cf);
45};
46
47void InitializeAllocator(const AllocatorOptions& options);
48void ReInitializeAllocator(const AllocatorOptions& options);
49void GetAllocatorOptions(AllocatorOptions* options);
50void ApplyAllocatorOptions(const AllocatorOptions& options);
51
52class AsanChunkView {
53 public:
54 explicit AsanChunkView(AsanChunk* chunk) : chunk_(chunk) {}
55 bool IsValid() const; // Checks if AsanChunkView points to a valid
56 // allocated or quarantined chunk.
57 bool IsAllocated() const; // Checks if the memory is currently allocated.
58 bool IsQuarantined() const; // Checks if the memory is currently quarantined.
59 uptr Beg() const; // First byte of user memory.
60 uptr End() const; // Last byte of user memory.
61 uptr UsedSize() const; // Size requested by the user.
62 u32 UserRequestedAlignment() const; // Originally requested alignment.
63 uptr AllocTid() const;
64 uptr FreeTid() const;
65 bool Eq(const AsanChunkView& c) const { return chunk_ == c.chunk_; }
66 u32 GetAllocStackId() const;
67 u32 GetFreeStackId() const;
68 AllocType GetAllocType() const;
69 bool AddrIsInside(uptr addr, uptr access_size, sptr* offset) const {
70 if (addr >= Beg() && (addr + access_size) <= End()) {
71 *offset = addr - Beg();
72 return true;
73 }
74 return false;
75 }
76 bool AddrIsAtLeft(uptr addr, uptr access_size, sptr* offset) const {
77 (void)access_size;
78 if (addr < Beg()) {
79 *offset = Beg() - addr;
80 return true;
81 }
82 return false;
83 }
84 bool AddrIsAtRight(uptr addr, uptr access_size, sptr* offset) const {
85 if (addr + access_size > End()) {
86 *offset = addr - End();
87 return true;
88 }
89 return false;
90 }
91
92 private:
93 AsanChunk* const chunk_;
94};
95
96AsanChunkView FindHeapChunkByAddress(uptr address);
97AsanChunkView FindHeapChunkByAllocBeg(uptr address);
98
99// List of AsanChunks with total size.
100class AsanChunkFifoList : public IntrusiveList<AsanChunk> {
101 public:
102 explicit AsanChunkFifoList(LinkerInitialized) {}
103 AsanChunkFifoList() { clear(); }
104 void Push(AsanChunk* n);
105 void PushList(AsanChunkFifoList* q);
106 AsanChunk* Pop();
107 uptr size() { return size_; }
108 void clear() {
109 IntrusiveList<AsanChunk>::clear();
110 size_ = 0;
111 }
112
113 private:
114 uptr size_;
115};
116
117struct AsanMapUnmapCallback {
118 void OnMap(uptr p, uptr size) const;
119 void OnMapSecondary(uptr p, uptr size, uptr user_begin, uptr user_size) const;
120 void OnUnmap(uptr p, uptr size) const;
121};
122
123#if SANITIZER_CAN_USE_ALLOCATOR64
124# if SANITIZER_FUCHSIA
125// This is a sentinel indicating we do not want the primary allocator arena to
126// be placed at a fixed address. It will be anonymously mmap'd.
127const uptr kAllocatorSpace = ~(uptr)0;
128# if SANITIZER_RISCV64
129
130// These are sanitizer tunings that allow all bringup tests for RISCV-64 Sv39 +
131// Fuchsia to run with asan-instrumented. That is, we can run bringup, e2e,
132// libc, and scudo tests with this configuration.
133//
134// TODO: This is specifically tuned for Sv39. 48/57 will likely require other
135// tunings, or possibly use the same tunings Fuchsia uses for other archs. The
136// VMA size isn't technically tied to the Fuchsia System ABI, so once 48/57 is
137// supported, we'd need a way of dynamically checking what the VMA size is and
138// determining optimal configuration.
139
140// This indicates the total amount of space dedicated for the primary allocator
141// during initialization. This is roughly proportional to the size set by the
142// FuchsiaConfig for scudo (~11.25GB == ~2^33.49). Requesting any more could
143// lead to some failures in sanitized bringup tests where we can't allocate new
144// vmars because there wouldn't be enough contiguous space. We could try 2^34 if
145// we re-evaluate the SizeClassMap settings.
146const uptr kAllocatorSize = UINT64_C(1) << 33; // 8GB
147
148// This is roughly equivalent to the configuration for the VeryDenseSizeClassMap
149// but has fewer size classes (ideally at most 32). Fewer class sizes means the
150// region size for each class is larger, thus less chances of running out of
151// space for each region. The main differences are the MidSizeLog (which is
152// smaller) and the MaxSizeLog (which is larger).
153//
154// - The MaxSizeLog is higher to allow some of the largest allocations I've
155// observed to be placed in the primary allocator's arena as opposed to being
156// mmap'd by the secondary allocator. This helps reduce fragmentation from
157// large classes. A huge example of this the scudo allocator tests (and its
158// testing infrastructure) which malloc's/new's objects on the order of
159// hundreds of kilobytes which normally would not be in the primary allocator
160// arena with the default VeryDenseSizeClassMap.
161// - The MidSizeLog is reduced to help shrink the number of size classes and
162// increase region size. Without this, we'd see ASan complain many times about
163// a region running out of available space.
164//
165// This differs a bit from the fuchsia config in scudo, mainly from the NumBits,
166// MaxSizeLog, and NumCachedHintT. This should place the number of size classes
167// for scudo at 45 and some large objects allocated by this config would be
168// placed in the arena whereas scudo would mmap them. The asan allocator needs
169// to have a number of classes that are a power of 2 for various internal things
170// to work, so we can't match the scudo settings to a tee. The sanitizer
171// allocator is slightly slower than scudo's but this is enough to get
172// memory-intensive scudo tests to run with asan instrumentation.
173typedef SizeClassMap</*kNumBits=*/2,
174 /*kMinSizeLog=*/5,
175 /*kMidSizeLog=*/8,
176 /*kMaxSizeLog=*/18,
177 /*kNumCachedHintT=*/8,
178 /*kMaxBytesCachedLog=*/10>
179 SizeClassMap;
180static_assert(SizeClassMap::kNumClassesRounded <= 32,
181 "The above tunings were specifically selected to ensure there "
182 "would be at most 32 size classes. This restriction could be "
183 "loosened to 64 size classes if we can find a configuration of "
184 "allocator size and SizeClassMap tunings that allows us to "
185 "reliably run all bringup tests in a sanitized environment.");
186
187# else // SANITIZER_RISCV64
188// These are the default allocator tunings for non-RISCV environments where the
189// VMA is usually 48 bits and we have lots of space.
190const uptr kAllocatorSize = 0x40000000000ULL; // 4T.
191typedef DefaultSizeClassMap SizeClassMap;
192# endif // SANITIZER_RISCV64
193# else // SANITIZER_FUCHSIA
194
195# if SANITIZER_APPLE
196const uptr kAllocatorSpace = 0x600000000000ULL;
197# else // SANITIZER_APPLE
198const uptr kAllocatorSpace = ~(uptr)0;
199# endif // SANITIZER_APPLE
200
201# if defined(__powerpc64__)
202# if SANITIZER_AIX
203const uptr kAllocatorSize = 1ULL << 38; // 256G.
204# else
205const uptr kAllocatorSize = 0x20000000000ULL; // 2T.
206# endif
207typedef DefaultSizeClassMap SizeClassMap;
208# elif defined(__aarch64__) && \
209 (SANITIZER_ANDROID || defined(SANITIZER_AARCH64_39BIT_VA))
210// Android needs to support 39, 42 and 48 bit VMA.
211const uptr kAllocatorSize = 0x2000000000ULL; // 128G.
212typedef VeryCompactSizeClassMap SizeClassMap;
213# elif SANITIZER_RISCV64
214const uptr kAllocatorSize = 0x2000000000ULL; // 128G.
215typedef VeryDenseSizeClassMap SizeClassMap;
216# elif defined(__sparc__)
217const uptr kAllocatorSize = 0x20000000000ULL; // 2T.
218typedef DefaultSizeClassMap SizeClassMap;
219# elif SANITIZER_ALPHA
220// Alpha has a 42-bit user VAS (TASK_SIZE = 0x40000000000). With fixed shadow
221// offset 0x70000000000, all app memory is in LowMem [0, 7T). Use 512G so the
222// allocator fits comfortably within LowMem alongside other mappings.
223const uptr kAllocatorSize = 0x8000000000ULL; // 512G.
224typedef DefaultSizeClassMap SizeClassMap;
225# elif SANITIZER_WINDOWS
226const uptr kAllocatorSize = 0x8000000000ULL; // 500G
227typedef DefaultSizeClassMap SizeClassMap;
228# elif SANITIZER_APPLE
229const uptr kAllocatorSize = 0x40000000000ULL; // 4T.
230typedef DefaultSizeClassMap SizeClassMap;
231# else
232const uptr kAllocatorSize = 0x40000000000ULL; // 4T.
233typedef DefaultSizeClassMap SizeClassMap;
234# endif // defined(__powerpc64__) etc.
235# endif // SANITIZER_FUCHSIA
236template <typename AddressSpaceViewTy>
237struct AP64 { // Allocator64 parameters. Deliberately using a short name.
238 static const uptr kSpaceBeg = kAllocatorSpace;
239 static const uptr kSpaceSize = kAllocatorSize;
240 static const uptr kMetadataSize = 0;
241 typedef __asan::SizeClassMap SizeClassMap;
242 typedef AsanMapUnmapCallback MapUnmapCallback;
243 static const uptr kFlags = 0;
244 using AddressSpaceView = AddressSpaceViewTy;
245};
246
247template <typename AddressSpaceView>
248using PrimaryAllocatorASVT = SizeClassAllocator64<AP64<AddressSpaceView>>;
249using PrimaryAllocator = PrimaryAllocatorASVT<LocalAddressSpaceView>;
250#else // SANITIZER_CAN_USE_ALLOCATOR64. Fallback to SizeClassAllocator32.
251typedef CompactSizeClassMap SizeClassMap;
252template <typename AddressSpaceViewTy>
253struct AP32 {
254 static const uptr kSpaceBeg = SANITIZER_MMAP_BEGIN;
255 static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE;
256 static const uptr kMetadataSize = 0;
257 typedef __asan::SizeClassMap SizeClassMap;
258 static const uptr kRegionSizeLog = 20;
259 using AddressSpaceView = AddressSpaceViewTy;
260 typedef AsanMapUnmapCallback MapUnmapCallback;
261 static const uptr kFlags = 0;
262};
263template <typename AddressSpaceView>
264using PrimaryAllocatorASVT = SizeClassAllocator32<AP32<AddressSpaceView>>;
265using PrimaryAllocator = PrimaryAllocatorASVT<LocalAddressSpaceView>;
266#endif // SANITIZER_CAN_USE_ALLOCATOR64
267
268static const uptr kNumberOfSizeClasses = SizeClassMap::kNumClasses;
269
270template <typename AddressSpaceView>
271using AsanAllocatorASVT =
272 CombinedAllocator<PrimaryAllocatorASVT<AddressSpaceView>>;
273using AsanAllocator = AsanAllocatorASVT<LocalAddressSpaceView>;
274using AllocatorCache = AsanAllocator::AllocatorCache;
275
276struct AsanThreadLocalMallocStorage {
277 uptr quarantine_cache[16];
278 AllocatorCache allocator_cache;
279 void CommitBack();
280
281 private:
282 // These objects are allocated via mmap() and are zero-initialized.
283 AsanThreadLocalMallocStorage() {}
284};
285
286void* asan_memalign(uptr alignment, uptr size, BufferedStackTrace* stack);
287void asan_free(void* ptr, BufferedStackTrace* stack);
288void asan_free_sized(void* ptr, uptr size, BufferedStackTrace* stack);
289void asan_free_aligned_sized(void* ptr, uptr alignment, uptr size,
290 BufferedStackTrace* stack);
291
292void* asan_malloc(uptr size, BufferedStackTrace* stack);
293void* asan_calloc(uptr nmemb, uptr size, BufferedStackTrace* stack);
294#if SANITIZER_AIX
295void* asan_vec_malloc(uptr size, BufferedStackTrace* stack);
296void* asan_vec_calloc(uptr nmemb, uptr size, BufferedStackTrace* stack);
297#endif
298void* asan_realloc(void* p, uptr size, BufferedStackTrace* stack);
299void* asan_reallocarray(void* p, uptr nmemb, uptr size,
300 BufferedStackTrace* stack);
301void* asan_valloc(uptr size, BufferedStackTrace* stack);
302void* asan_pvalloc(uptr size, BufferedStackTrace* stack);
303
304void* asan_aligned_alloc(uptr alignment, uptr size, BufferedStackTrace* stack);
305int asan_posix_memalign(void** memptr, uptr alignment, uptr size,
306 BufferedStackTrace* stack);
307uptr asan_malloc_usable_size(const void* ptr, uptr pc, uptr bp);
308
309void* asan_new(uptr size, BufferedStackTrace* stack);
310void* asan_new_aligned(uptr size, uptr alignment, BufferedStackTrace* stack);
311void* asan_new_array(uptr size, BufferedStackTrace* stack);
312void* asan_new_array_aligned(uptr size, uptr alignment,
313 BufferedStackTrace* stack);
314void asan_delete(void* ptr, BufferedStackTrace* stack);
315void asan_delete_aligned(void* ptr, uptr alignment, BufferedStackTrace* stack);
316void asan_delete_sized(void* ptr, uptr size, BufferedStackTrace* stack);
317void asan_delete_sized_aligned(void* ptr, uptr size, uptr alignment,
318 BufferedStackTrace* stack);
319void asan_delete_array(void* ptr, BufferedStackTrace* stack);
320void asan_delete_array_aligned(void* ptr, uptr alignment,
321 BufferedStackTrace* stack);
322void asan_delete_array_sized(void* ptr, uptr size, BufferedStackTrace* stack);
323void asan_delete_array_sized_aligned(void* ptr, uptr size, uptr alignment,
324 BufferedStackTrace* stack);
325
326uptr asan_mz_size(const void* ptr);
327void asan_mz_force_lock();
328void asan_mz_force_unlock();
329
330void PrintInternalAllocatorStats();
331void AsanSoftRssLimitExceededCallback(bool exceeded);
332
333} // namespace __asan
334#endif // ASAN_ALLOCATOR_H
335