1//===-- copyprof_allocator.cpp -------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// This file implements the standalone memory allocator for CopyProf, managing
10/// internal dynamic allocations required during profiling.
11///
12//===----------------------------------------------------------------------===//
13
14#include "copyprof_allocator.h"
15
16#include <pthread.h>
17
18#include "copyprof_interceptors.h"
19#include "copyprof_interface_internal.h"
20#include "copyprof_per_object_state.h"
21#include "copyprof_shadow.h"
22#include "copyprof_state.h"
23#include "sanitizer_common/sanitizer_allocator.h"
24#include "sanitizer_common/sanitizer_allocator_checks.h"
25#include "sanitizer_common/sanitizer_allocator_interface.h"
26#include "sanitizer_common/sanitizer_allocator_report.h"
27#include "sanitizer_common/sanitizer_common.h"
28#include "sanitizer_common/sanitizer_errno.h"
29#include "sanitizer_common/sanitizer_flags.h"
30#include "sanitizer_common/sanitizer_internal_defs.h"
31#include "sanitizer_common/sanitizer_platform_limits_posix.h"
32#include "sanitizer_common/sanitizer_stacktrace.h"
33
34namespace __copyprof {
35namespace {
36
37using namespace __sanitizer;
38
39// Callbacks invoked by the sanitizer allocator whenever it maps or unmaps
40// memory. They are no-ops, as CopyProf does not need to track the allocator's
41// mappings: shadow memory is updated per allocation instead (see
42// `CopyProfAllocator::Allocate`).
43struct CopyProfMapUnmapCallback {
44 // Called when the primary allocator maps memory for a region from which it
45 // serves small and medium sized allocations.
46 void OnMap(uptr p, uptr size) const {}
47 // Called when the secondary allocator maps memory for an allocation that is
48 // too large for the primary allocator. Each such allocation is mmap()ed
49 // separately.
50 void OnMapSecondary(uptr p, uptr size, uptr user_begin,
51 uptr user_size) const {}
52 // Called when either allocator unmaps memory.
53 void OnUnmap(uptr p, uptr size) const {}
54};
55
56struct CopyProfMetadata {
57 // The requested allocation size in bytes.
58 uptr requested_size;
59};
60
61struct AllocatorParameters {
62 static constexpr uptr kSpaceBeg = ~(uptr)0;
63 static constexpr uptr kSpaceSize = 0x40000000000ULL;
64 static constexpr uptr kMetadataSize = sizeof(CopyProfMetadata);
65 static constexpr uptr kFlags = 0;
66 using SizeClassMap = DefaultSizeClassMap;
67 using MapUnmapCallback = CopyProfMapUnmapCallback;
68 using AddressSpaceView = LocalAddressSpaceView;
69};
70
71using PrimaryAllocator = SizeClassAllocator64<AllocatorParameters>;
72using Allocator = CombinedAllocator<PrimaryAllocator>;
73using AllocatorCache = Allocator::AllocatorCache;
74
75// Per-thread allocator cache. It is mmap()ed, and thus zero-initialized as
76// `AllocatorCache` requires, and committed back to `allocator` when its thread
77// exits. As CopyProf does not intercept thread creation, the cache is created
78// by the thread's first (de)allocation, which also arms the thread-specific
79// data (TSD) destructor that commits it back.
80pthread_key_t thread_cache_key;
81__attribute__((
82 tls_model("initial-exec"))) THREADLOCAL AllocatorCache* thread_cache;
83// Set once the exiting thread's cache has been committed back. Later TSD
84// destructors may still (de)allocate on this thread so they use the fallback
85// cache, as a re-created cache could leak.
86__attribute__((
87 tls_model("initial-exec"))) THREADLOCAL bool thread_cache_committed_back;
88
89uptr ThreadCacheSize() {
90 return RoundUpTo(size: sizeof(AllocatorCache), boundary: GetPageSizeCached());
91}
92
93// Returns the calling thread's allocator cache, or `nullptr` if it has already
94// been committed back.
95AllocatorCache* GetThreadCache() {
96 if (LIKELY(thread_cache))
97 return thread_cache;
98 if (UNLIKELY(thread_cache_committed_back))
99 return nullptr;
100 thread_cache =
101 static_cast<AllocatorCache*>(MmapOrDie(size: ThreadCacheSize(), mem_type: __func__));
102 CHECK_EQ(0, pthread_setspecific(thread_cache_key, thread_cache));
103 return thread_cache;
104}
105
106// Custom allocator that updates shadow memory accordingly.
107struct CopyProfAllocator {
108 // Heap allocations must be at least 8-byte aligned so independent
109 // allocations never share a shadow byte (avoiding data races).
110 static_assert(kMinAlignment == 1 << kShadowScale);
111
112 Allocator allocator;
113 // Fallback allocator cache and mutex used in the slow path when no per-thread
114 // allocator cache is available.
115 StaticSpinMutex fallback_mutex;
116 AllocatorCache fallback_allocator_cache;
117
118 void Init() {
119 SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
120 allocator.Init(release_to_os_interval_ms: common_flags()->allocator_release_to_os_interval_ms);
121 }
122
123 CopyProfMetadata* GetMetadata(const void* ptr) {
124 return reinterpret_cast<CopyProfMetadata*>(allocator.GetMetaData(p: ptr));
125 }
126
127 // `align` must be a non-zero power of two; callers are responsible for
128 // validating it.
129 void* Allocate(uptr size, uptr align = kMinAlignment) {
130 if (UNLIKELY(!copyprof_is_initialized))
131 __copyprof_init_once();
132 align = (align < kMinAlignment) ? kMinAlignment : align;
133 CHECK(IsPowerOfTwo(align));
134 // C++ standard requires zero-sized allocations to return a non-null
135 // pointer.
136 if (size == 0)
137 size = 1;
138 void* allocated;
139 if (AllocatorCache* cache = GetThreadCache()) {
140 allocated = allocator.Allocate(cache, size, alignment: align);
141 } else {
142 SpinMutexLock lock(&fallback_mutex);
143 allocated = allocator.Allocate(cache: &fallback_allocator_cache, size, alignment: align);
144 }
145
146 if (UNLIKELY(!allocated)) {
147 SetAllocatorOutOfMemory();
148 if (AllocatorMayReturnNull())
149 return nullptr;
150 UNINITIALIZED BufferedStackTrace stack;
151 stack.Unwind(pc: StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(),
152 /*context=*/nullptr, request_fast: common_flags()->fast_unwind_on_fatal);
153 ReportOutOfMemory(requested_size: size, stack: &stack);
154 }
155 GetMetadata(ptr: allocated)->requested_size = size;
156 SmfContext ctx = __copyprof_state.smf_context;
157 MarkApplicationMemory(app_addr: allocated, num_bytes: size, is_copy: ShouldMarkNewMemoryAsCopy(ctx));
158 if (ctx == SmfContext::COPY)
159 RecordPerObjectAllocation(this_ptr: __copyprof_state.current_this_ptr, num_bytes: size);
160 return allocated;
161 }
162
163 void Deallocate(void* ptr) {
164 if (UNLIKELY(!ptr))
165 return;
166 if (__copyprof_state.smf_context == SmfContext::DTOR) {
167 __copyprof_state.is_transitive_copy &=
168 IsMarkedAsCopy(app_addr: ptr, num_bytes: GetMetadata(ptr)->requested_size);
169 }
170 if (AllocatorCache* cache = GetThreadCache()) {
171 allocator.Deallocate(cache, p: ptr);
172 } else {
173 SpinMutexLock lock(&fallback_mutex);
174 allocator.Deallocate(cache: &fallback_allocator_cache, p: ptr);
175 }
176 }
177
178 void* Calloc(uptr num, uptr size) {
179 if (UNLIKELY(CheckForCallocOverflow(size, num))) {
180 if (AllocatorMayReturnNull())
181 return nullptr;
182 UNINITIALIZED BufferedStackTrace stack;
183 stack.Unwind(pc: StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(),
184 /*context=*/nullptr, request_fast: common_flags()->fast_unwind_on_fatal);
185 ReportCallocOverflow(count: num, size, stack: &stack);
186 }
187 void* ptr = Allocate(size: num * size);
188 // Memory from the secondary allocator is freshly mmap()ed, and thus already
189 // zeroed.
190 if (ptr && allocator.FromPrimary(p: ptr))
191 REAL(memset)(ptr, 0, num * size);
192 return ptr;
193 }
194
195 // Implemented via `Allocate` and `Deallocate` rather than
196 // `CombinedAllocator::Reallocate`, s.t. shadow memory, metadata and
197 // per-object state are updated as for any other (de)allocation.
198 // Note that the shadow of the old memory is not carried over.
199 // TODO: figure out whether this assumption is a problem in practice.
200 void* Reallocate(void* old_ptr, uptr new_size) {
201 CHECK(old_ptr && new_size);
202 void* new_ptr = Allocate(size: new_size);
203 if (new_ptr) {
204 CHECK_NE(REAL(memcpy), nullptr);
205 uptr memcpy_size = Min(a: new_size, b: GetMetadata(ptr: old_ptr)->requested_size);
206 REAL(memcpy)(new_ptr, old_ptr, memcpy_size);
207 Deallocate(ptr: old_ptr);
208 }
209 return new_ptr;
210 }
211};
212
213CopyProfAllocator instance;
214
215// TSD destructor that commits the exiting thread's cache back to `allocator`.
216void CommitBackThreadCache(void* cache) {
217 thread_cache = nullptr;
218 thread_cache_committed_back = true;
219 instance.allocator.SwallowCache(cache: static_cast<AllocatorCache*>(cache));
220 UnmapOrDie(addr: cache, size: ThreadCacheSize());
221}
222
223} // namespace
224
225void InitializeCopyProfAllocator() {
226 instance.Init();
227 CHECK_EQ(0, pthread_key_create(&thread_cache_key, CommitBackThreadCache));
228}
229
230void* CopyProfAllocate(uptr size, uptr align) {
231 if (UNLIKELY(align == 0 || !IsPowerOfTwo(align))) {
232 errno = errno_EINVAL;
233 if (AllocatorMayReturnNull())
234 return nullptr;
235 UNINITIALIZED BufferedStackTrace stack;
236 stack.Unwind(pc: StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(),
237 /*context=*/nullptr, request_fast: common_flags()->fast_unwind_on_fatal);
238 ReportInvalidAllocationAlignment(alignment: align, stack: &stack);
239 }
240 return SetErrnoOnNull(instance.Allocate(size, align));
241}
242
243void CopyProfDeallocate(void* ptr, uptr /*size*/, uptr /*align*/) {
244 instance.Deallocate(ptr);
245}
246
247void* CopyProfMalloc(uptr size) {
248 return SetErrnoOnNull(instance.Allocate(size));
249}
250
251void CopyProfFree(void* ptr) { instance.Deallocate(ptr); }
252
253void* CopyProfCalloc(uptr num, uptr size) {
254 return SetErrnoOnNull(instance.Calloc(num, size));
255}
256
257void* CopyProfRealloc(void* ptr, uptr size) {
258 if (!ptr)
259 return SetErrnoOnNull(instance.Allocate(size));
260 if (size == 0) {
261 instance.Deallocate(ptr);
262 return nullptr;
263 }
264 return SetErrnoOnNull(instance.Reallocate(old_ptr: ptr, new_size: size));
265}
266
267void* CopyProfReallocarray(void* ptr, uptr num, uptr size) {
268 if (UNLIKELY(CheckForCallocOverflow(size, num))) {
269 errno = errno_ENOMEM;
270 if (AllocatorMayReturnNull())
271 return nullptr;
272 UNINITIALIZED BufferedStackTrace stack;
273 stack.Unwind(pc: StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(),
274 /*context=*/nullptr, request_fast: common_flags()->fast_unwind_on_fatal);
275 ReportReallocArrayOverflow(count: num, size, stack: &stack);
276 }
277 return CopyProfRealloc(ptr, size: num * size);
278}
279
280void* CopyProfValloc(uptr size) {
281 return SetErrnoOnNull(instance.Allocate(size, align: GetPageSizeCached()));
282}
283
284void* CopyProfPvalloc(uptr size) {
285 uptr page_size = GetPageSizeCached();
286 if (UNLIKELY(CheckForPvallocOverflow(size, page_size))) {
287 errno = errno_ENOMEM;
288 if (AllocatorMayReturnNull())
289 return nullptr;
290 UNINITIALIZED BufferedStackTrace stack;
291 stack.Unwind(pc: StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(),
292 /*context=*/nullptr, request_fast: common_flags()->fast_unwind_on_fatal);
293 ReportPvallocOverflow(size, stack: &stack);
294 }
295 // `pvalloc(0)` allocates one page.
296 size = size ? RoundUpTo(size, boundary: page_size) : page_size;
297 return SetErrnoOnNull(instance.Allocate(size, align: page_size));
298}
299
300int CopyProfPosixMemalign(void** memptr, uptr alignment, uptr size) {
301 if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) {
302 if (AllocatorMayReturnNull())
303 return errno_EINVAL;
304 UNINITIALIZED BufferedStackTrace stack;
305 stack.Unwind(pc: StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(),
306 /*context=*/nullptr, request_fast: common_flags()->fast_unwind_on_fatal);
307 ReportInvalidPosixMemalignAlignment(alignment, stack: &stack);
308 }
309 void* ptr = instance.Allocate(size, align: alignment);
310 // Running out of memory has already been handled by `Allocate`.
311 if (UNLIKELY(!ptr))
312 return errno_ENOMEM;
313 *memptr = ptr;
314 return 0;
315}
316
317uptr CopyProfMallocUsableSize(const void* ptr) {
318 // Return the requested size, and 0 for memory that was not allocated by
319 // CopyProf.
320 if (!ptr || instance.allocator.GetBlockBegin(p: ptr) != ptr)
321 return 0;
322 return instance.GetMetadata(ptr)->requested_size;
323}
324
325} // namespace __copyprof
326