1//===-- wrappers_c.cpp ------------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "platform.h"
10
11#include "allocator_config.h"
12#include "internal_defs.h"
13#include "scudo/interface.h"
14#include "wrappers_c.h"
15#include "wrappers_c_checks.h"
16
17#include "string_utils.h"
18
19#include <stdint.h>
20#include <stdio.h>
21
22#include <sys/uio.h>
23
24#if defined(SCUDO_PREFIX_NAME)
25#define SCUDO_PREFIX(name) CONCATENATE(SCUDO_PREFIX_NAME, name)
26#define SCUDO_ALLOCATOR_STATIC static
27#else
28#define SCUDO_PREFIX(name) name
29// Export the static allocator so that the C++ wrappers can access it.
30// Technically we could have a completely separated heap for C & C++ but in
31// reality the amount of cross pollination between the two is staggering.
32#define SCUDO_ALLOCATOR_STATIC
33#endif
34
35// malloc-type functions have to be aligned to std::max_align_t. This is
36// distinct from (1U << SCUDO_MIN_ALIGNMENT_LOG), since C++ new-type functions
37// do not have to abide by the same requirement.
38#ifndef SCUDO_MALLOC_ALIGNMENT
39#define SCUDO_MALLOC_ALIGNMENT FIRST_32_SECOND_64(8U, 16U)
40#endif
41
42extern "C" void SCUDO_PREFIX(malloc_postinit)();
43SCUDO_REQUIRE_CONSTANT_INITIALIZATION
44SCUDO_ALLOCATOR_STATIC
45scudo::Allocator<scudo::Config, SCUDO_PREFIX(malloc_postinit)> Allocator;
46
47static void reportAllocation(void *ptr, size_t size) {
48 if (SCUDO_ENABLE_HOOKS)
49 if (__scudo_allocate_hook && ptr)
50 __scudo_allocate_hook(ptr, size);
51}
52static void reportDeallocation(void *ptr) {
53 if (SCUDO_ENABLE_HOOKS)
54 if (__scudo_deallocate_hook)
55 __scudo_deallocate_hook(ptr);
56}
57static void reportReallocAllocation(void *old_ptr, void *new_ptr, size_t size) {
58 DCHECK_NE(new_ptr, nullptr);
59
60 if (SCUDO_ENABLE_HOOKS) {
61 if (__scudo_realloc_allocate_hook)
62 __scudo_realloc_allocate_hook(old_ptr, new_ptr, size);
63 else if (__scudo_allocate_hook)
64 __scudo_allocate_hook(ptr: new_ptr, size);
65 }
66}
67static void reportReallocDeallocation(void *old_ptr) {
68 if (SCUDO_ENABLE_HOOKS) {
69 if (__scudo_realloc_deallocate_hook)
70 __scudo_realloc_deallocate_hook(old_ptr);
71 else if (__scudo_deallocate_hook)
72 __scudo_deallocate_hook(ptr: old_ptr);
73 }
74}
75
76static void enableInChildAfterFork() { Allocator.enable(/*IsChild*/ true); }
77
78extern "C" {
79
80INTERFACE WEAK void *SCUDO_PREFIX(calloc)(size_t nmemb, size_t size) {
81 scudo::uptr Product;
82 if (UNLIKELY(scudo::checkForCallocOverflow(size, nmemb, &Product))) {
83 if (Allocator.canReturnNull()) {
84 errno = ENOMEM;
85 return nullptr;
86 }
87 scudo::reportCallocOverflow(Count: nmemb, Size: size);
88 }
89 void *Ptr = Allocator.allocate(Size: Product, Origin: scudo::Chunk::Origin::Malloc,
90 SCUDO_MALLOC_ALIGNMENT, ZeroContents: true);
91 reportAllocation(ptr: Ptr, size: Product);
92 return scudo::setErrnoOnNull(Ptr);
93}
94
95INTERFACE WEAK void SCUDO_PREFIX(free)(void *ptr) {
96 reportDeallocation(ptr);
97 Allocator.deallocate(Ptr: ptr, Origin: scudo::Chunk::Origin::Malloc);
98}
99
100INTERFACE WEAK void SCUDO_PREFIX(free_sized)(void *ptr, size_t size) {
101 reportDeallocation(ptr);
102 Allocator.deallocateSized(Ptr: ptr, Origin: scudo::Chunk::Origin::Malloc, DeleteSize: size);
103}
104
105INTERFACE WEAK void
106SCUDO_PREFIX(free_aligned_sized)(void *ptr, size_t alignment, size_t size) {
107 reportDeallocation(ptr);
108 Allocator.deallocateSizedAligned(Ptr: ptr, Origin: scudo::Chunk::Origin::Malloc, DeleteSize: size,
109 DeleteAlignment: alignment);
110}
111
112INTERFACE WEAK struct SCUDO_MALLINFO SCUDO_PREFIX(mallinfo)(void) {
113 struct SCUDO_MALLINFO Info = {};
114 scudo::StatCounters Stats;
115 Allocator.getStats(S: Stats);
116 // Space allocated in mmapped regions (bytes)
117 Info.hblkhd = static_cast<__scudo_mallinfo_data_t>(Stats[scudo::StatMapped]);
118 // Maximum total allocated space (bytes)
119 Info.usmblks = Info.hblkhd;
120 // Space in freed fastbin blocks (bytes)
121 Info.fsmblks = static_cast<__scudo_mallinfo_data_t>(Stats[scudo::StatFree]);
122 // Total allocated space (bytes)
123 Info.uordblks =
124 static_cast<__scudo_mallinfo_data_t>(Stats[scudo::StatAllocated]);
125 // Total free space (bytes)
126 Info.fordblks = Info.fsmblks;
127 return Info;
128}
129
130// On Android, mallinfo2 is an alias of mallinfo, so don't define both.
131#if !SCUDO_ANDROID
132INTERFACE WEAK struct __scudo_mallinfo2 SCUDO_PREFIX(mallinfo2)(void) {
133 struct __scudo_mallinfo2 Info = {};
134 scudo::StatCounters Stats;
135 Allocator.getStats(S: Stats);
136 // Space allocated in mmapped regions (bytes)
137 Info.hblkhd = Stats[scudo::StatMapped];
138 // Maximum total allocated space (bytes)
139 Info.usmblks = Info.hblkhd;
140 // Space in freed fastbin blocks (bytes)
141 Info.fsmblks = Stats[scudo::StatFree];
142 // Total allocated space (bytes)
143 Info.uordblks = Stats[scudo::StatAllocated];
144 // Total free space (bytes)
145 Info.fordblks = Info.fsmblks;
146 return Info;
147}
148#endif
149
150INTERFACE WEAK void *SCUDO_PREFIX(malloc)(size_t size) {
151 void *Ptr = Allocator.allocate(Size: size, Origin: scudo::Chunk::Origin::Malloc,
152 SCUDO_MALLOC_ALIGNMENT);
153 reportAllocation(ptr: Ptr, size);
154 return scudo::setErrnoOnNull(Ptr);
155}
156
157#if SCUDO_ANDROID
158INTERFACE WEAK size_t SCUDO_PREFIX(malloc_usable_size)(const void *ptr) {
159#else
160INTERFACE WEAK size_t SCUDO_PREFIX(malloc_usable_size)(void *ptr) {
161#endif
162 return Allocator.getUsableSize(Ptr: ptr);
163}
164
165INTERFACE WEAK void *SCUDO_PREFIX(memalign)(size_t alignment, size_t size) {
166 // Android rounds up the alignment to a power of two if it isn't one.
167 if (SCUDO_ANDROID) {
168 if (UNLIKELY(!alignment)) {
169 alignment = 1U;
170 } else {
171 if (UNLIKELY(!scudo::isPowerOfTwo(alignment)))
172 alignment = scudo::roundUpPowerOfTwo(Size: alignment);
173 }
174 } else {
175 if (UNLIKELY(!scudo::isPowerOfTwo(alignment))) {
176 if (Allocator.canReturnNull()) {
177 errno = EINVAL;
178 return nullptr;
179 }
180 scudo::reportAlignmentNotPowerOfTwo(Alignment: alignment);
181 }
182 }
183 void *Ptr =
184 Allocator.allocate(Size: size, Origin: scudo::Chunk::Origin::Memalign, Alignment: alignment);
185 reportAllocation(ptr: Ptr, size);
186 return Ptr;
187}
188
189INTERFACE WEAK int SCUDO_PREFIX(posix_memalign)(void **memptr, size_t alignment,
190 size_t size) {
191 if (UNLIKELY(scudo::checkPosixMemalignAlignment(alignment))) {
192 if (!Allocator.canReturnNull())
193 scudo::reportInvalidPosixMemalignAlignment(Alignment: alignment);
194 return EINVAL;
195 }
196 void *Ptr =
197 Allocator.allocate(Size: size, Origin: scudo::Chunk::Origin::Memalign, Alignment: alignment);
198 if (UNLIKELY(!Ptr))
199 return ENOMEM;
200 reportAllocation(ptr: Ptr, size);
201
202 *memptr = Ptr;
203 return 0;
204}
205
206INTERFACE WEAK void *SCUDO_PREFIX(pvalloc)(size_t size) {
207 const scudo::uptr PageSize = scudo::getPageSizeCached();
208 if (UNLIKELY(scudo::checkForPvallocOverflow(size, PageSize))) {
209 if (Allocator.canReturnNull()) {
210 errno = ENOMEM;
211 return nullptr;
212 }
213 scudo::reportPvallocOverflow(Size: size);
214 }
215 // pvalloc(0) should allocate one page.
216 void *Ptr =
217 Allocator.allocate(Size: size ? scudo::roundUp(X: size, Boundary: PageSize) : PageSize,
218 Origin: scudo::Chunk::Origin::Memalign, Alignment: PageSize);
219 reportAllocation(ptr: Ptr, size: scudo::roundUp(X: size, Boundary: PageSize));
220
221 return scudo::setErrnoOnNull(Ptr);
222}
223
224INTERFACE WEAK void *SCUDO_PREFIX(realloc)(void *ptr, size_t size) {
225 if (!ptr) {
226 void *Ptr = Allocator.allocate(Size: size, Origin: scudo::Chunk::Origin::Malloc,
227 SCUDO_MALLOC_ALIGNMENT);
228 reportAllocation(ptr: Ptr, size);
229 return scudo::setErrnoOnNull(Ptr);
230 }
231 if (size == 0) {
232 reportDeallocation(ptr);
233 Allocator.deallocate(Ptr: ptr, Origin: scudo::Chunk::Origin::Malloc);
234 return nullptr;
235 }
236
237 // Given that the reporting of deallocation and allocation are not atomic, we
238 // always pretend the old pointer will be released so that the user doesn't
239 // need to worry about the false double-use case from the view of hooks.
240 //
241 // For example, assume that `realloc` releases the old pointer and allocates a
242 // new pointer. Before the reporting of both operations has been done, another
243 // thread may get the old pointer from `malloc`. It may be misinterpreted as
244 // double-use if it's not handled properly on the hook side.
245 reportReallocDeallocation(old_ptr: ptr);
246 void *NewPtr = Allocator.reallocate(OldPtr: ptr, NewSize: size, SCUDO_MALLOC_ALIGNMENT);
247 if (NewPtr != nullptr) {
248 // Note that even if NewPtr == ptr, the size has changed. We still need to
249 // report the new size.
250 reportReallocAllocation(/*OldPtr=*/old_ptr: ptr, new_ptr: NewPtr, size);
251 } else {
252 // If `realloc` fails, the old pointer is not released. Report the old
253 // pointer as allocated again.
254 reportReallocAllocation(/*OldPtr=*/old_ptr: ptr, /*NewPtr=*/new_ptr: ptr,
255 size: Allocator.getAllocSize(Ptr: ptr));
256 }
257
258 return scudo::setErrnoOnNull(NewPtr);
259}
260
261INTERFACE WEAK void *SCUDO_PREFIX(reallocarray)(void *ptr, size_t nmemb,
262 size_t size) {
263 scudo::uptr Product;
264 if (UNLIKELY(scudo::checkForCallocOverflow(size, nmemb, &Product))) {
265 if (Allocator.canReturnNull()) {
266 errno = ENOMEM;
267 return nullptr;
268 }
269 scudo::reportReallocarrayOverflow(Count: nmemb, Size: size);
270 }
271 return SCUDO_PREFIX(realloc)(ptr, size: Product);
272}
273
274INTERFACE WEAK void *SCUDO_PREFIX(valloc)(size_t size) {
275 void *Ptr = Allocator.allocate(Size: size, Origin: scudo::Chunk::Origin::Memalign,
276 Alignment: scudo::getPageSizeCached());
277 reportAllocation(ptr: Ptr, size);
278
279 return scudo::setErrnoOnNull(Ptr);
280}
281
282INTERFACE WEAK int SCUDO_PREFIX(malloc_iterate)(
283 uintptr_t base, size_t size,
284 void (*callback)(uintptr_t base, size_t size, void *arg), void *arg) {
285 Allocator.iterateOverChunks(Base: base, Size: size, Callback: callback, Arg: arg);
286 return 0;
287}
288
289INTERFACE WEAK void SCUDO_PREFIX(malloc_enable)() {
290 Allocator.enable(/*IsChild*/ false);
291}
292INTERFACE WEAK void SCUDO_PREFIX(malloc_disable)() { Allocator.disable(); }
293
294void SCUDO_PREFIX(malloc_postinit)() {
295 Allocator.initGwpAsan();
296 pthread_atfork(SCUDO_PREFIX(malloc_disable), SCUDO_PREFIX(malloc_enable),
297 child: enableInChildAfterFork);
298}
299
300INTERFACE WEAK int SCUDO_PREFIX(mallopt)(int param, int value) {
301 if (param == M_DECAY_TIME) {
302 if (SCUDO_ANDROID) {
303 // Before changing the interval, reset the memory usage status by doing a
304 // M_PURGE call so that we can minimize the impact of any unreleased pages
305 // introduced by interval transition.
306 Allocator.releaseToOS(ReleaseType: scudo::ReleaseToOS::Force);
307
308 // The values allowed on Android are {-1, 0, 1}. "1" means the longest
309 // interval.
310 CHECK(value >= -1 && value <= 1);
311 if (value == 1)
312 value = INT32_MAX;
313 }
314
315 Allocator.setOption(O: scudo::Option::ReleaseInterval,
316 Value: static_cast<scudo::sptr>(value));
317 return 1;
318 } else if (param == M_PURGE) {
319 Allocator.releaseToOS(ReleaseType: scudo::ReleaseToOS::Force);
320 return 1;
321 } else if (param == M_PURGE_FAST) {
322 Allocator.releaseToOS(ReleaseType: scudo::ReleaseToOS::ForceFast);
323 return 1;
324 } else if (param == M_PURGE_ALL) {
325 Allocator.releaseToOS(ReleaseType: scudo::ReleaseToOS::ForceAll);
326 return 1;
327 } else if (param == M_LOG_STATS) {
328 Allocator.printStats();
329 Allocator.printFragmentationInfo();
330 return 1;
331 } else {
332 scudo::Option option;
333 switch (param) {
334 case M_MEMTAG_TUNING:
335 option = scudo::Option::MemtagTuning;
336 break;
337 case M_THREAD_DISABLE_MEM_INIT:
338 option = scudo::Option::ThreadDisableMemInit;
339 break;
340 case M_CACHE_COUNT_MAX:
341 option = scudo::Option::MaxCacheEntriesCount;
342 break;
343 case M_CACHE_SIZE_MAX:
344 option = scudo::Option::MaxCacheEntrySize;
345 break;
346 case M_TSDS_COUNT_MAX:
347 option = scudo::Option::MaxTSDsCount;
348 break;
349 default:
350 return 0;
351 }
352 return Allocator.setOption(O: option, Value: static_cast<scudo::sptr>(value));
353 }
354}
355
356INTERFACE WEAK void *SCUDO_PREFIX(aligned_alloc)(size_t alignment,
357 size_t size) {
358 if (UNLIKELY(scudo::checkAlignedAllocAlignmentAndSize(alignment, size))) {
359 if (Allocator.canReturnNull()) {
360 errno = EINVAL;
361 return nullptr;
362 }
363 scudo::reportInvalidAlignedAllocAlignment(Size: alignment, Alignment: size);
364 }
365
366 void *Ptr =
367 Allocator.allocate(Size: size, Origin: scudo::Chunk::Origin::Memalign, Alignment: alignment);
368 reportAllocation(ptr: Ptr, size);
369
370 return scudo::setErrnoOnNull(Ptr);
371}
372
373INTERFACE WEAK int SCUDO_PREFIX(malloc_info)(UNUSED int options, FILE *stream) {
374 const scudo::uptr max_size =
375 decltype(Allocator)::PrimaryT::SizeClassMap::MaxSize;
376 auto *sizes = static_cast<scudo::uptr *>(
377 SCUDO_PREFIX(calloc)(nmemb: max_size, size: sizeof(scudo::uptr)));
378 auto callback = [](uintptr_t, size_t size, void *arg) {
379 auto *sizes = reinterpret_cast<scudo::uptr *>(arg);
380 if (size < max_size)
381 sizes[size]++;
382 };
383
384 Allocator.disable();
385 Allocator.iterateOverChunks(Base: 0, Size: -1ul, Callback: callback, Arg: sizes);
386 Allocator.enable(/*IsChild*/ false);
387
388 fputs(s: "<malloc version=\"scudo-1\">\n", stream: stream);
389 for (scudo::uptr i = 0; i != max_size; ++i)
390 if (sizes[i])
391 fprintf(stream: stream, format: "<alloc size=\"%zu\" count=\"%zu\"/>\n", i, sizes[i]);
392 fputs(s: "</malloc>\n", stream: stream);
393 SCUDO_PREFIX(free)(ptr: sizes);
394 return 0;
395}
396
397// Disable memory tagging for the heap. The caller must disable memory tag
398// checks globally (e.g. by clearing TCF0 on aarch64) before calling this
399// function, and may not re-enable them after calling the function.
400INTERFACE WEAK void SCUDO_PREFIX(malloc_disable_memory_tagging)() {
401 Allocator.disableMemoryTagging();
402}
403
404// Sets whether scudo records stack traces and other metadata for allocations
405// and deallocations. This function only has an effect if the allocator and
406// hardware support memory tagging.
407INTERFACE WEAK void
408SCUDO_PREFIX(malloc_set_track_allocation_stacks)(int track) {
409 Allocator.setTrackAllocationStacks(track);
410}
411
412// Sets whether scudo zero-initializes all allocated memory.
413INTERFACE WEAK void SCUDO_PREFIX(malloc_set_zero_contents)(int zero_contents) {
414 Allocator.setFillContents(zero_contents ? scudo::ZeroFill : scudo::NoFill);
415}
416
417// Sets whether scudo pattern-initializes all allocated memory.
418INTERFACE WEAK void
419SCUDO_PREFIX(malloc_set_pattern_fill_contents)(int pattern_fill_contents) {
420 Allocator.setFillContents(pattern_fill_contents ? scudo::PatternOrZeroFill
421 : scudo::NoFill);
422}
423
424// Sets whether scudo adds a small amount of slack at the end of large
425// allocations, before the guard page. This can be enabled to work around buggy
426// applications that read a few bytes past the end of their allocation.
427INTERFACE WEAK void
428SCUDO_PREFIX(malloc_set_add_large_allocation_slack)(int add_slack) {
429 Allocator.setAddLargeAllocationSlack(add_slack);
430}
431
432// Extra Internal functions.
433INTERFACE void __scudo_print_stats(void) { Allocator.printStats(); }
434
435#if !SCUDO_FUCHSIA
436
437INTERFACE void
438__scudo_get_fault_error_info(uintptr_t fault_address,
439 struct scudo_error_info *error_info) {
440 Allocator.getErrorInfo(FaultAddr: fault_address, ErrorInfo: error_info);
441}
442
443#endif
444
445} // extern "C"
446