| 1 | //===-- copyprof_allocator.cpp -------------------------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | /// |
| 9 | /// This file implements the standalone memory allocator for CopyProf, managing |
| 10 | /// internal dynamic allocations required during profiling. |
| 11 | /// |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
| 14 | #include "copyprof_allocator.h" |
| 15 | |
| 16 | #include <pthread.h> |
| 17 | |
| 18 | #include "copyprof_interceptors.h" |
| 19 | #include "copyprof_interface_internal.h" |
| 20 | #include "copyprof_per_object_state.h" |
| 21 | #include "copyprof_shadow.h" |
| 22 | #include "copyprof_state.h" |
| 23 | #include "sanitizer_common/sanitizer_allocator.h" |
| 24 | #include "sanitizer_common/sanitizer_allocator_checks.h" |
| 25 | #include "sanitizer_common/sanitizer_allocator_interface.h" |
| 26 | #include "sanitizer_common/sanitizer_allocator_report.h" |
| 27 | #include "sanitizer_common/sanitizer_common.h" |
| 28 | #include "sanitizer_common/sanitizer_errno.h" |
| 29 | #include "sanitizer_common/sanitizer_flags.h" |
| 30 | #include "sanitizer_common/sanitizer_internal_defs.h" |
| 31 | #include "sanitizer_common/sanitizer_platform_limits_posix.h" |
| 32 | #include "sanitizer_common/sanitizer_stacktrace.h" |
| 33 | |
| 34 | namespace __copyprof { |
| 35 | namespace { |
| 36 | |
| 37 | using namespace __sanitizer; |
| 38 | |
| 39 | // Callbacks invoked by the sanitizer allocator whenever it maps or unmaps |
| 40 | // memory. They are no-ops, as CopyProf does not need to track the allocator's |
| 41 | // mappings: shadow memory is updated per allocation instead (see |
| 42 | // `CopyProfAllocator::Allocate`). |
| 43 | struct CopyProfMapUnmapCallback { |
| 44 | // Called when the primary allocator maps memory for a region from which it |
| 45 | // serves small and medium sized allocations. |
| 46 | void OnMap(uptr p, uptr size) const {} |
| 47 | // Called when the secondary allocator maps memory for an allocation that is |
| 48 | // too large for the primary allocator. Each such allocation is mmap()ed |
| 49 | // separately. |
| 50 | void OnMapSecondary(uptr p, uptr size, uptr user_begin, |
| 51 | uptr user_size) const {} |
| 52 | // Called when either allocator unmaps memory. |
| 53 | void OnUnmap(uptr p, uptr size) const {} |
| 54 | }; |
| 55 | |
| 56 | struct CopyProfMetadata { |
| 57 | // The requested allocation size in bytes. |
| 58 | uptr requested_size; |
| 59 | }; |
| 60 | |
| 61 | struct AllocatorParameters { |
| 62 | static constexpr uptr kSpaceBeg = ~(uptr)0; |
| 63 | static constexpr uptr kSpaceSize = 0x40000000000ULL; |
| 64 | static constexpr uptr kMetadataSize = sizeof(CopyProfMetadata); |
| 65 | static constexpr uptr kFlags = 0; |
| 66 | using SizeClassMap = DefaultSizeClassMap; |
| 67 | using MapUnmapCallback = CopyProfMapUnmapCallback; |
| 68 | using AddressSpaceView = LocalAddressSpaceView; |
| 69 | }; |
| 70 | |
| 71 | using PrimaryAllocator = SizeClassAllocator64<AllocatorParameters>; |
| 72 | using Allocator = CombinedAllocator<PrimaryAllocator>; |
| 73 | using AllocatorCache = Allocator::AllocatorCache; |
| 74 | |
| 75 | // Per-thread allocator cache. It is mmap()ed, and thus zero-initialized as |
| 76 | // `AllocatorCache` requires, and committed back to `allocator` when its thread |
| 77 | // exits. As CopyProf does not intercept thread creation, the cache is created |
| 78 | // by the thread's first (de)allocation, which also arms the thread-specific |
| 79 | // data (TSD) destructor that commits it back. |
| 80 | pthread_key_t thread_cache_key; |
| 81 | __attribute__(( |
| 82 | tls_model("initial-exec" ))) THREADLOCAL AllocatorCache* thread_cache; |
| 83 | // Set once the exiting thread's cache has been committed back. Later TSD |
| 84 | // destructors may still (de)allocate on this thread so they use the fallback |
| 85 | // cache, as a re-created cache could leak. |
| 86 | __attribute__(( |
| 87 | tls_model("initial-exec" ))) THREADLOCAL bool thread_cache_committed_back; |
| 88 | |
| 89 | uptr ThreadCacheSize() { |
| 90 | return RoundUpTo(size: sizeof(AllocatorCache), boundary: GetPageSizeCached()); |
| 91 | } |
| 92 | |
| 93 | // Returns the calling thread's allocator cache, or `nullptr` if it has already |
| 94 | // been committed back. |
| 95 | AllocatorCache* GetThreadCache() { |
| 96 | if (LIKELY(thread_cache)) |
| 97 | return thread_cache; |
| 98 | if (UNLIKELY(thread_cache_committed_back)) |
| 99 | return nullptr; |
| 100 | thread_cache = |
| 101 | static_cast<AllocatorCache*>(MmapOrDie(size: ThreadCacheSize(), mem_type: __func__)); |
| 102 | CHECK_EQ(0, pthread_setspecific(thread_cache_key, thread_cache)); |
| 103 | return thread_cache; |
| 104 | } |
| 105 | |
| 106 | // Custom allocator that updates shadow memory accordingly. |
| 107 | struct CopyProfAllocator { |
| 108 | // Heap allocations must be at least 8-byte aligned so independent |
| 109 | // allocations never share a shadow byte (avoiding data races). |
| 110 | static_assert(kMinAlignment == 1 << kShadowScale); |
| 111 | |
| 112 | Allocator allocator; |
| 113 | // Fallback allocator cache and mutex used in the slow path when no per-thread |
| 114 | // allocator cache is available. |
| 115 | StaticSpinMutex fallback_mutex; |
| 116 | AllocatorCache fallback_allocator_cache; |
| 117 | |
| 118 | void Init() { |
| 119 | SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null); |
| 120 | allocator.Init(release_to_os_interval_ms: common_flags()->allocator_release_to_os_interval_ms); |
| 121 | } |
| 122 | |
| 123 | CopyProfMetadata* GetMetadata(const void* ptr) { |
| 124 | return reinterpret_cast<CopyProfMetadata*>(allocator.GetMetaData(p: ptr)); |
| 125 | } |
| 126 | |
| 127 | // `align` must be a non-zero power of two; callers are responsible for |
| 128 | // validating it. |
| 129 | void* Allocate(uptr size, uptr align = kMinAlignment) { |
| 130 | if (UNLIKELY(!copyprof_is_initialized)) |
| 131 | __copyprof_init_once(); |
| 132 | align = (align < kMinAlignment) ? kMinAlignment : align; |
| 133 | CHECK(IsPowerOfTwo(align)); |
| 134 | // C++ standard requires zero-sized allocations to return a non-null |
| 135 | // pointer. |
| 136 | if (size == 0) |
| 137 | size = 1; |
| 138 | void* allocated; |
| 139 | if (AllocatorCache* cache = GetThreadCache()) { |
| 140 | allocated = allocator.Allocate(cache, size, alignment: align); |
| 141 | } else { |
| 142 | SpinMutexLock lock(&fallback_mutex); |
| 143 | allocated = allocator.Allocate(cache: &fallback_allocator_cache, size, alignment: align); |
| 144 | } |
| 145 | |
| 146 | if (UNLIKELY(!allocated)) { |
| 147 | SetAllocatorOutOfMemory(); |
| 148 | if (AllocatorMayReturnNull()) |
| 149 | return nullptr; |
| 150 | UNINITIALIZED BufferedStackTrace stack; |
| 151 | stack.Unwind(pc: StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), |
| 152 | /*context=*/nullptr, request_fast: common_flags()->fast_unwind_on_fatal); |
| 153 | ReportOutOfMemory(requested_size: size, stack: &stack); |
| 154 | } |
| 155 | GetMetadata(ptr: allocated)->requested_size = size; |
| 156 | SmfContext ctx = __copyprof_state.smf_context; |
| 157 | MarkApplicationMemory(app_addr: allocated, num_bytes: size, is_copy: ShouldMarkNewMemoryAsCopy(ctx)); |
| 158 | if (ctx == SmfContext::COPY) |
| 159 | RecordPerObjectAllocation(this_ptr: __copyprof_state.current_this_ptr, num_bytes: size); |
| 160 | return allocated; |
| 161 | } |
| 162 | |
| 163 | void Deallocate(void* ptr) { |
| 164 | if (UNLIKELY(!ptr)) |
| 165 | return; |
| 166 | if (__copyprof_state.smf_context == SmfContext::DTOR) { |
| 167 | __copyprof_state.is_transitive_copy &= |
| 168 | IsMarkedAsCopy(app_addr: ptr, num_bytes: GetMetadata(ptr)->requested_size); |
| 169 | } |
| 170 | if (AllocatorCache* cache = GetThreadCache()) { |
| 171 | allocator.Deallocate(cache, p: ptr); |
| 172 | } else { |
| 173 | SpinMutexLock lock(&fallback_mutex); |
| 174 | allocator.Deallocate(cache: &fallback_allocator_cache, p: ptr); |
| 175 | } |
| 176 | } |
| 177 | |
| 178 | void* Calloc(uptr num, uptr size) { |
| 179 | if (UNLIKELY(CheckForCallocOverflow(size, num))) { |
| 180 | if (AllocatorMayReturnNull()) |
| 181 | return nullptr; |
| 182 | UNINITIALIZED BufferedStackTrace stack; |
| 183 | stack.Unwind(pc: StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), |
| 184 | /*context=*/nullptr, request_fast: common_flags()->fast_unwind_on_fatal); |
| 185 | ReportCallocOverflow(count: num, size, stack: &stack); |
| 186 | } |
| 187 | void* ptr = Allocate(size: num * size); |
| 188 | // Memory from the secondary allocator is freshly mmap()ed, and thus already |
| 189 | // zeroed. |
| 190 | if (ptr && allocator.FromPrimary(p: ptr)) |
| 191 | REAL(memset)(ptr, 0, num * size); |
| 192 | return ptr; |
| 193 | } |
| 194 | |
| 195 | // Implemented via `Allocate` and `Deallocate` rather than |
| 196 | // `CombinedAllocator::Reallocate`, s.t. shadow memory, metadata and |
| 197 | // per-object state are updated as for any other (de)allocation. |
| 198 | // Note that the shadow of the old memory is not carried over. |
| 199 | // TODO: figure out whether this assumption is a problem in practice. |
| 200 | void* Reallocate(void* old_ptr, uptr new_size) { |
| 201 | CHECK(old_ptr && new_size); |
| 202 | void* new_ptr = Allocate(size: new_size); |
| 203 | if (new_ptr) { |
| 204 | CHECK_NE(REAL(memcpy), nullptr); |
| 205 | uptr memcpy_size = Min(a: new_size, b: GetMetadata(ptr: old_ptr)->requested_size); |
| 206 | REAL(memcpy)(new_ptr, old_ptr, memcpy_size); |
| 207 | Deallocate(ptr: old_ptr); |
| 208 | } |
| 209 | return new_ptr; |
| 210 | } |
| 211 | }; |
| 212 | |
| 213 | CopyProfAllocator instance; |
| 214 | |
| 215 | // TSD destructor that commits the exiting thread's cache back to `allocator`. |
| 216 | void CommitBackThreadCache(void* cache) { |
| 217 | thread_cache = nullptr; |
| 218 | thread_cache_committed_back = true; |
| 219 | instance.allocator.SwallowCache(cache: static_cast<AllocatorCache*>(cache)); |
| 220 | UnmapOrDie(addr: cache, size: ThreadCacheSize()); |
| 221 | } |
| 222 | |
| 223 | } // namespace |
| 224 | |
| 225 | void InitializeCopyProfAllocator() { |
| 226 | instance.Init(); |
| 227 | CHECK_EQ(0, pthread_key_create(&thread_cache_key, CommitBackThreadCache)); |
| 228 | } |
| 229 | |
| 230 | void* CopyProfAllocate(uptr size, uptr align) { |
| 231 | if (UNLIKELY(align == 0 || !IsPowerOfTwo(align))) { |
| 232 | errno = errno_EINVAL; |
| 233 | if (AllocatorMayReturnNull()) |
| 234 | return nullptr; |
| 235 | UNINITIALIZED BufferedStackTrace stack; |
| 236 | stack.Unwind(pc: StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), |
| 237 | /*context=*/nullptr, request_fast: common_flags()->fast_unwind_on_fatal); |
| 238 | ReportInvalidAllocationAlignment(alignment: align, stack: &stack); |
| 239 | } |
| 240 | return SetErrnoOnNull(instance.Allocate(size, align)); |
| 241 | } |
| 242 | |
| 243 | void CopyProfDeallocate(void* ptr, uptr /*size*/, uptr /*align*/) { |
| 244 | instance.Deallocate(ptr); |
| 245 | } |
| 246 | |
| 247 | void* CopyProfMalloc(uptr size) { |
| 248 | return SetErrnoOnNull(instance.Allocate(size)); |
| 249 | } |
| 250 | |
| 251 | void CopyProfFree(void* ptr) { instance.Deallocate(ptr); } |
| 252 | |
| 253 | void* CopyProfCalloc(uptr num, uptr size) { |
| 254 | return SetErrnoOnNull(instance.Calloc(num, size)); |
| 255 | } |
| 256 | |
| 257 | void* CopyProfRealloc(void* ptr, uptr size) { |
| 258 | if (!ptr) |
| 259 | return SetErrnoOnNull(instance.Allocate(size)); |
| 260 | if (size == 0) { |
| 261 | instance.Deallocate(ptr); |
| 262 | return nullptr; |
| 263 | } |
| 264 | return SetErrnoOnNull(instance.Reallocate(old_ptr: ptr, new_size: size)); |
| 265 | } |
| 266 | |
| 267 | void* CopyProfReallocarray(void* ptr, uptr num, uptr size) { |
| 268 | if (UNLIKELY(CheckForCallocOverflow(size, num))) { |
| 269 | errno = errno_ENOMEM; |
| 270 | if (AllocatorMayReturnNull()) |
| 271 | return nullptr; |
| 272 | UNINITIALIZED BufferedStackTrace stack; |
| 273 | stack.Unwind(pc: StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), |
| 274 | /*context=*/nullptr, request_fast: common_flags()->fast_unwind_on_fatal); |
| 275 | ReportReallocArrayOverflow(count: num, size, stack: &stack); |
| 276 | } |
| 277 | return CopyProfRealloc(ptr, size: num * size); |
| 278 | } |
| 279 | |
| 280 | void* CopyProfValloc(uptr size) { |
| 281 | return SetErrnoOnNull(instance.Allocate(size, align: GetPageSizeCached())); |
| 282 | } |
| 283 | |
| 284 | void* CopyProfPvalloc(uptr size) { |
| 285 | uptr page_size = GetPageSizeCached(); |
| 286 | if (UNLIKELY(CheckForPvallocOverflow(size, page_size))) { |
| 287 | errno = errno_ENOMEM; |
| 288 | if (AllocatorMayReturnNull()) |
| 289 | return nullptr; |
| 290 | UNINITIALIZED BufferedStackTrace stack; |
| 291 | stack.Unwind(pc: StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), |
| 292 | /*context=*/nullptr, request_fast: common_flags()->fast_unwind_on_fatal); |
| 293 | ReportPvallocOverflow(size, stack: &stack); |
| 294 | } |
| 295 | // `pvalloc(0)` allocates one page. |
| 296 | size = size ? RoundUpTo(size, boundary: page_size) : page_size; |
| 297 | return SetErrnoOnNull(instance.Allocate(size, align: page_size)); |
| 298 | } |
| 299 | |
| 300 | int CopyProfPosixMemalign(void** memptr, uptr alignment, uptr size) { |
| 301 | if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) { |
| 302 | if (AllocatorMayReturnNull()) |
| 303 | return errno_EINVAL; |
| 304 | UNINITIALIZED BufferedStackTrace stack; |
| 305 | stack.Unwind(pc: StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), |
| 306 | /*context=*/nullptr, request_fast: common_flags()->fast_unwind_on_fatal); |
| 307 | ReportInvalidPosixMemalignAlignment(alignment, stack: &stack); |
| 308 | } |
| 309 | void* ptr = instance.Allocate(size, align: alignment); |
| 310 | // Running out of memory has already been handled by `Allocate`. |
| 311 | if (UNLIKELY(!ptr)) |
| 312 | return errno_ENOMEM; |
| 313 | *memptr = ptr; |
| 314 | return 0; |
| 315 | } |
| 316 | |
| 317 | uptr CopyProfMallocUsableSize(const void* ptr) { |
| 318 | // Return the requested size, and 0 for memory that was not allocated by |
| 319 | // CopyProf. |
| 320 | if (!ptr || instance.allocator.GetBlockBegin(p: ptr) != ptr) |
| 321 | return 0; |
| 322 | return instance.GetMetadata(ptr)->requested_size; |
| 323 | } |
| 324 | |
| 325 | } // namespace __copyprof |
| 326 | |