1//===-- wrappers_c.cpp ------------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "platform.h"
10
11#include "allocator_config.h"
12#include "internal_defs.h"
13#include "scudo/interface.h"
14#include "wrappers_c.h"
15#include "wrappers_c_checks.h"
16
17#include "string_utils.h"
18
19#include <stdint.h>
20#include <stdio.h>
21
22#include <sys/uio.h>
23
24#if defined(SCUDO_PREFIX_NAME)
25#define SCUDO_PREFIX(name) CONCATENATE(SCUDO_PREFIX_NAME, name)
26#define SCUDO_ALLOCATOR_STATIC static
27#else
28#define SCUDO_PREFIX(name) name
29// Export the static allocator so that the C++ wrappers can access it.
30// Technically we could have a completely separated heap for C & C++ but in
31// reality the amount of cross pollination between the two is staggering.
32#define SCUDO_ALLOCATOR_STATIC
33#endif
34
35// malloc-type functions have to be aligned to std::max_align_t. This is
36// distinct from (1U << SCUDO_MIN_ALIGNMENT_LOG), since C++ new-type functions
37// do not have to abide by the same requirement.
38#ifndef SCUDO_MALLOC_ALIGNMENT
39#define SCUDO_MALLOC_ALIGNMENT FIRST_32_SECOND_64(8U, 16U)
40#endif
41
42extern "C" void SCUDO_PREFIX(malloc_postinit)();
43SCUDO_REQUIRE_CONSTANT_INITIALIZATION
44SCUDO_ALLOCATOR_STATIC
45scudo::Allocator<scudo::Config, SCUDO_PREFIX(malloc_postinit)> Allocator;
46
47static void reportAllocation(void *ptr, size_t size) {
48 if (SCUDO_ENABLE_HOOKS)
49 if (__scudo_allocate_hook && ptr)
50 __scudo_allocate_hook(ptr, size);
51}
52static void reportDeallocation(void *ptr) {
53 if (SCUDO_ENABLE_HOOKS)
54 if (__scudo_deallocate_hook)
55 __scudo_deallocate_hook(ptr);
56}
57static void reportReallocAllocation(void *old_ptr, void *new_ptr, size_t size) {
58 DCHECK_NE(new_ptr, nullptr);
59
60 if (SCUDO_ENABLE_HOOKS) {
61 if (__scudo_realloc_allocate_hook)
62 __scudo_realloc_allocate_hook(old_ptr, new_ptr, size);
63 else if (__scudo_allocate_hook)
64 __scudo_allocate_hook(ptr: new_ptr, size);
65 }
66}
67static void reportReallocDeallocation(void *old_ptr) {
68 if (SCUDO_ENABLE_HOOKS) {
69 if (__scudo_realloc_deallocate_hook)
70 __scudo_realloc_deallocate_hook(old_ptr);
71 else if (__scudo_deallocate_hook)
72 __scudo_deallocate_hook(ptr: old_ptr);
73 }
74}
75
76extern "C" {
77
78INTERFACE WEAK void *SCUDO_PREFIX(calloc)(size_t nmemb, size_t size) {
79 scudo::uptr Product;
80 if (UNLIKELY(scudo::checkForCallocOverflow(size, nmemb, &Product))) {
81 if (Allocator.canReturnNull()) {
82 errno = ENOMEM;
83 return nullptr;
84 }
85 scudo::reportCallocOverflow(Count: nmemb, Size: size);
86 }
87 void *Ptr = Allocator.allocate(Size: Product, Origin: scudo::Chunk::Origin::Malloc,
88 SCUDO_MALLOC_ALIGNMENT, ZeroContents: true);
89 reportAllocation(ptr: Ptr, size: Product);
90 return scudo::setErrnoOnNull(Ptr);
91}
92
93INTERFACE WEAK void SCUDO_PREFIX(free)(void *ptr) {
94 reportDeallocation(ptr);
95 Allocator.deallocate(Ptr: ptr, Origin: scudo::Chunk::Origin::Malloc);
96}
97
98INTERFACE WEAK void SCUDO_PREFIX(free_sized)(void *ptr, size_t size) {
99 reportDeallocation(ptr);
100 Allocator.deallocateSized(Ptr: ptr, Origin: scudo::Chunk::Origin::Malloc, DeleteSize: size);
101}
102
103INTERFACE WEAK void
104SCUDO_PREFIX(free_aligned_sized)(void *ptr, size_t alignment, size_t size) {
105 reportDeallocation(ptr);
106 Allocator.deallocateSizedAligned(Ptr: ptr, Origin: scudo::Chunk::Origin::Malloc, DeleteSize: size,
107 DeleteAlignment: alignment);
108}
109
110INTERFACE WEAK struct SCUDO_MALLINFO SCUDO_PREFIX(mallinfo)(void) {
111 struct SCUDO_MALLINFO Info = {};
112 scudo::StatCounters Stats;
113 Allocator.getStats(S: Stats);
114 // Space allocated in mmapped regions (bytes)
115 Info.hblkhd = static_cast<__scudo_mallinfo_data_t>(Stats[scudo::StatMapped]);
116 // Maximum total allocated space (bytes)
117 Info.usmblks = Info.hblkhd;
118 // Space in freed fastbin blocks (bytes)
119 Info.fsmblks = static_cast<__scudo_mallinfo_data_t>(Stats[scudo::StatFree]);
120 // Total allocated space (bytes)
121 Info.uordblks =
122 static_cast<__scudo_mallinfo_data_t>(Stats[scudo::StatAllocated]);
123 // Total free space (bytes)
124 Info.fordblks = Info.fsmblks;
125 return Info;
126}
127
128// On Android, mallinfo2 is an alias of mallinfo, so don't define both.
129#if !SCUDO_ANDROID
130INTERFACE WEAK struct __scudo_mallinfo2 SCUDO_PREFIX(mallinfo2)(void) {
131 struct __scudo_mallinfo2 Info = {};
132 scudo::StatCounters Stats;
133 Allocator.getStats(S: Stats);
134 // Space allocated in mmapped regions (bytes)
135 Info.hblkhd = Stats[scudo::StatMapped];
136 // Maximum total allocated space (bytes)
137 Info.usmblks = Info.hblkhd;
138 // Space in freed fastbin blocks (bytes)
139 Info.fsmblks = Stats[scudo::StatFree];
140 // Total allocated space (bytes)
141 Info.uordblks = Stats[scudo::StatAllocated];
142 // Total free space (bytes)
143 Info.fordblks = Info.fsmblks;
144 return Info;
145}
146#endif
147
148INTERFACE WEAK void *SCUDO_PREFIX(malloc)(size_t size) {
149 void *Ptr = Allocator.allocate(Size: size, Origin: scudo::Chunk::Origin::Malloc,
150 SCUDO_MALLOC_ALIGNMENT);
151 reportAllocation(ptr: Ptr, size);
152 return scudo::setErrnoOnNull(Ptr);
153}
154
155#if SCUDO_ANDROID
156INTERFACE WEAK size_t SCUDO_PREFIX(malloc_usable_size)(const void *ptr) {
157#else
158INTERFACE WEAK size_t SCUDO_PREFIX(malloc_usable_size)(void *ptr) {
159#endif
160 return Allocator.getUsableSize(Ptr: ptr);
161}
162
163INTERFACE WEAK void *SCUDO_PREFIX(memalign)(size_t alignment, size_t size) {
164 // Android rounds up the alignment to a power of two if it isn't one.
165 if (SCUDO_ANDROID) {
166 if (UNLIKELY(!alignment)) {
167 alignment = 1U;
168 } else {
169 if (UNLIKELY(!scudo::isPowerOfTwo(alignment)))
170 alignment = scudo::roundUpPowerOfTwo(Size: alignment);
171 }
172 } else {
173 if (UNLIKELY(!scudo::isPowerOfTwo(alignment))) {
174 if (Allocator.canReturnNull()) {
175 errno = EINVAL;
176 return nullptr;
177 }
178 scudo::reportAlignmentNotPowerOfTwo(Alignment: alignment);
179 }
180 }
181 void *Ptr =
182 Allocator.allocate(Size: size, Origin: scudo::Chunk::Origin::Memalign, Alignment: alignment);
183 reportAllocation(ptr: Ptr, size);
184 return Ptr;
185}
186
187INTERFACE WEAK int SCUDO_PREFIX(posix_memalign)(void **memptr, size_t alignment,
188 size_t size) {
189 if (UNLIKELY(scudo::checkPosixMemalignAlignment(alignment))) {
190 if (!Allocator.canReturnNull())
191 scudo::reportInvalidPosixMemalignAlignment(Alignment: alignment);
192 return EINVAL;
193 }
194 void *Ptr =
195 Allocator.allocate(Size: size, Origin: scudo::Chunk::Origin::Memalign, Alignment: alignment);
196 if (UNLIKELY(!Ptr))
197 return ENOMEM;
198 reportAllocation(ptr: Ptr, size);
199
200 *memptr = Ptr;
201 return 0;
202}
203
204INTERFACE WEAK void *SCUDO_PREFIX(pvalloc)(size_t size) {
205 const scudo::uptr PageSize = scudo::getPageSizeCached();
206 if (UNLIKELY(scudo::checkForPvallocOverflow(size, PageSize))) {
207 if (Allocator.canReturnNull()) {
208 errno = ENOMEM;
209 return nullptr;
210 }
211 scudo::reportPvallocOverflow(Size: size);
212 }
213 // pvalloc(0) should allocate one page.
214 void *Ptr =
215 Allocator.allocate(Size: size ? scudo::roundUp(X: size, Boundary: PageSize) : PageSize,
216 Origin: scudo::Chunk::Origin::Memalign, Alignment: PageSize);
217 reportAllocation(ptr: Ptr, size: scudo::roundUp(X: size, Boundary: PageSize));
218
219 return scudo::setErrnoOnNull(Ptr);
220}
221
222INTERFACE WEAK void *SCUDO_PREFIX(realloc)(void *ptr, size_t size) {
223 if (!ptr) {
224 void *Ptr = Allocator.allocate(Size: size, Origin: scudo::Chunk::Origin::Malloc,
225 SCUDO_MALLOC_ALIGNMENT);
226 reportAllocation(ptr: Ptr, size);
227 return scudo::setErrnoOnNull(Ptr);
228 }
229 if (size == 0) {
230 reportDeallocation(ptr);
231 Allocator.deallocate(Ptr: ptr, Origin: scudo::Chunk::Origin::Malloc);
232 return nullptr;
233 }
234
235 // Given that the reporting of deallocation and allocation are not atomic, we
236 // always pretend the old pointer will be released so that the user doesn't
237 // need to worry about the false double-use case from the view of hooks.
238 //
239 // For example, assume that `realloc` releases the old pointer and allocates a
240 // new pointer. Before the reporting of both operations has been done, another
241 // thread may get the old pointer from `malloc`. It may be misinterpreted as
242 // double-use if it's not handled properly on the hook side.
243 reportReallocDeallocation(old_ptr: ptr);
244 void *NewPtr = Allocator.reallocate(OldPtr: ptr, NewSize: size, SCUDO_MALLOC_ALIGNMENT);
245 if (NewPtr != nullptr) {
246 // Note that even if NewPtr == ptr, the size has changed. We still need to
247 // report the new size.
248 reportReallocAllocation(/*OldPtr=*/old_ptr: ptr, new_ptr: NewPtr, size);
249 } else {
250 // If `realloc` fails, the old pointer is not released. Report the old
251 // pointer as allocated again.
252 reportReallocAllocation(/*OldPtr=*/old_ptr: ptr, /*NewPtr=*/new_ptr: ptr,
253 size: Allocator.getAllocSize(Ptr: ptr));
254 }
255
256 return scudo::setErrnoOnNull(NewPtr);
257}
258
259INTERFACE WEAK void *SCUDO_PREFIX(reallocarray)(void *ptr, size_t nmemb,
260 size_t size) {
261 scudo::uptr Product;
262 if (UNLIKELY(scudo::checkForCallocOverflow(size, nmemb, &Product))) {
263 if (Allocator.canReturnNull()) {
264 errno = ENOMEM;
265 return nullptr;
266 }
267 scudo::reportReallocarrayOverflow(Count: nmemb, Size: size);
268 }
269 return SCUDO_PREFIX(realloc)(ptr, size: Product);
270}
271
272INTERFACE WEAK void *SCUDO_PREFIX(valloc)(size_t size) {
273 void *Ptr = Allocator.allocate(Size: size, Origin: scudo::Chunk::Origin::Memalign,
274 Alignment: scudo::getPageSizeCached());
275 reportAllocation(ptr: Ptr, size);
276
277 return scudo::setErrnoOnNull(Ptr);
278}
279
280INTERFACE WEAK int SCUDO_PREFIX(malloc_iterate)(
281 uintptr_t base, size_t size,
282 void (*callback)(uintptr_t base, size_t size, void *arg), void *arg) {
283 Allocator.iterateOverChunks(Base: base, Size: size, Callback: callback, Arg: arg);
284 return 0;
285}
286
287INTERFACE WEAK void SCUDO_PREFIX(malloc_enable)() { Allocator.enable(); }
288
289INTERFACE WEAK void SCUDO_PREFIX(malloc_disable)() { Allocator.disable(); }
290
291void SCUDO_PREFIX(malloc_postinit)() {
292 Allocator.initGwpAsan();
293 pthread_atfork(SCUDO_PREFIX(malloc_disable), SCUDO_PREFIX(malloc_enable),
294 SCUDO_PREFIX(malloc_enable));
295}
296
297INTERFACE WEAK int SCUDO_PREFIX(mallopt)(int param, int value) {
298 if (param == M_DECAY_TIME) {
299 if (SCUDO_ANDROID) {
300 // Before changing the interval, reset the memory usage status by doing a
301 // M_PURGE call so that we can minimize the impact of any unreleased pages
302 // introduced by interval transition.
303 Allocator.releaseToOS(ReleaseType: scudo::ReleaseToOS::Force);
304
305 // The values allowed on Android are {-1, 0, 1}. "1" means the longest
306 // interval.
307 CHECK(value >= -1 && value <= 1);
308 if (value == 1)
309 value = INT32_MAX;
310 }
311
312 Allocator.setOption(O: scudo::Option::ReleaseInterval,
313 Value: static_cast<scudo::sptr>(value));
314 return 1;
315 } else if (param == M_PURGE) {
316 Allocator.releaseToOS(ReleaseType: scudo::ReleaseToOS::Force);
317 return 1;
318 } else if (param == M_PURGE_FAST) {
319 Allocator.releaseToOS(ReleaseType: scudo::ReleaseToOS::ForceFast);
320 return 1;
321 } else if (param == M_PURGE_ALL) {
322 Allocator.releaseToOS(ReleaseType: scudo::ReleaseToOS::ForceAll);
323 return 1;
324 } else if (param == M_LOG_STATS) {
325 Allocator.printStats();
326 Allocator.printFragmentationInfo();
327 return 1;
328 } else {
329 scudo::Option option;
330 switch (param) {
331 case M_MEMTAG_TUNING:
332 option = scudo::Option::MemtagTuning;
333 break;
334 case M_THREAD_DISABLE_MEM_INIT:
335 option = scudo::Option::ThreadDisableMemInit;
336 break;
337 case M_CACHE_COUNT_MAX:
338 option = scudo::Option::MaxCacheEntriesCount;
339 break;
340 case M_CACHE_SIZE_MAX:
341 option = scudo::Option::MaxCacheEntrySize;
342 break;
343 case M_TSDS_COUNT_MAX:
344 option = scudo::Option::MaxTSDsCount;
345 break;
346 default:
347 return 0;
348 }
349 return Allocator.setOption(O: option, Value: static_cast<scudo::sptr>(value));
350 }
351}
352
353INTERFACE WEAK void *SCUDO_PREFIX(aligned_alloc)(size_t alignment,
354 size_t size) {
355 if (UNLIKELY(scudo::checkAlignedAllocAlignmentAndSize(alignment, size))) {
356 if (Allocator.canReturnNull()) {
357 errno = EINVAL;
358 return nullptr;
359 }
360 scudo::reportInvalidAlignedAllocAlignment(Size: alignment, Alignment: size);
361 }
362
363 void *Ptr =
364 Allocator.allocate(Size: size, Origin: scudo::Chunk::Origin::Memalign, Alignment: alignment);
365 reportAllocation(ptr: Ptr, size);
366
367 return scudo::setErrnoOnNull(Ptr);
368}
369
370INTERFACE WEAK int SCUDO_PREFIX(malloc_info)(UNUSED int options, FILE *stream) {
371 const scudo::uptr max_size =
372 decltype(Allocator)::PrimaryT::SizeClassMap::MaxSize;
373 auto *sizes = static_cast<scudo::uptr *>(
374 SCUDO_PREFIX(calloc)(nmemb: max_size, size: sizeof(scudo::uptr)));
375 auto callback = [](uintptr_t, size_t size, void *arg) {
376 auto *sizes = reinterpret_cast<scudo::uptr *>(arg);
377 if (size < max_size)
378 sizes[size]++;
379 };
380
381 Allocator.disable();
382 Allocator.iterateOverChunks(Base: 0, Size: -1ul, Callback: callback, Arg: sizes);
383 Allocator.enable();
384
385 fputs(s: "<malloc version=\"scudo-1\">\n", stream: stream);
386 for (scudo::uptr i = 0; i != max_size; ++i)
387 if (sizes[i])
388 fprintf(stream: stream, format: "<alloc size=\"%zu\" count=\"%zu\"/>\n", i, sizes[i]);
389 fputs(s: "</malloc>\n", stream: stream);
390 SCUDO_PREFIX(free)(ptr: sizes);
391 return 0;
392}
393
394// Disable memory tagging for the heap. The caller must disable memory tag
395// checks globally (e.g. by clearing TCF0 on aarch64) before calling this
396// function, and may not re-enable them after calling the function.
397INTERFACE WEAK void SCUDO_PREFIX(malloc_disable_memory_tagging)() {
398 Allocator.disableMemoryTagging();
399}
400
401// Sets whether scudo records stack traces and other metadata for allocations
402// and deallocations. This function only has an effect if the allocator and
403// hardware support memory tagging.
404INTERFACE WEAK void
405SCUDO_PREFIX(malloc_set_track_allocation_stacks)(int track) {
406 Allocator.setTrackAllocationStacks(track);
407}
408
409// Sets whether scudo zero-initializes all allocated memory.
410INTERFACE WEAK void SCUDO_PREFIX(malloc_set_zero_contents)(int zero_contents) {
411 Allocator.setFillContents(zero_contents ? scudo::ZeroFill : scudo::NoFill);
412}
413
414// Sets whether scudo pattern-initializes all allocated memory.
415INTERFACE WEAK void
416SCUDO_PREFIX(malloc_set_pattern_fill_contents)(int pattern_fill_contents) {
417 Allocator.setFillContents(pattern_fill_contents ? scudo::PatternOrZeroFill
418 : scudo::NoFill);
419}
420
421// Sets whether scudo adds a small amount of slack at the end of large
422// allocations, before the guard page. This can be enabled to work around buggy
423// applications that read a few bytes past the end of their allocation.
424INTERFACE WEAK void
425SCUDO_PREFIX(malloc_set_add_large_allocation_slack)(int add_slack) {
426 Allocator.setAddLargeAllocationSlack(add_slack);
427}
428
429// Extra Internal functions.
430INTERFACE void __scudo_print_stats(void) { Allocator.printStats(); }
431
432#if !SCUDO_FUCHSIA
433
434INTERFACE void
435__scudo_get_fault_error_info(uintptr_t fault_address,
436 struct scudo_error_info *error_info) {
437 Allocator.getErrorInfo(FaultAddr: fault_address, ErrorInfo: error_info);
438}
439
440#endif
441
442} // extern "C"
443