1//===-- wrappers_c.inc ------------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#ifndef SCUDO_PREFIX
10#error "Define SCUDO_PREFIX prior to including this file!"
11#endif
12
13// malloc-type functions have to be aligned to std::max_align_t. This is
14// distinct from (1U << SCUDO_MIN_ALIGNMENT_LOG), since C++ new-type functions
15// do not have to abide by the same requirement.
16#ifndef SCUDO_MALLOC_ALIGNMENT
17#define SCUDO_MALLOC_ALIGNMENT FIRST_32_SECOND_64(8U, 16U)
18#endif
19
20static void reportAllocation(void *ptr, size_t size) {
21 if (SCUDO_ENABLE_HOOKS)
22 if (__scudo_allocate_hook && ptr)
23 __scudo_allocate_hook(ptr, size);
24}
25static void reportDeallocation(void *ptr) {
26 if (SCUDO_ENABLE_HOOKS)
27 if (__scudo_deallocate_hook)
28 __scudo_deallocate_hook(ptr);
29}
30static void reportReallocAllocation(void *old_ptr, void *new_ptr, size_t size) {
31 DCHECK_NE(new_ptr, nullptr);
32
33 if (SCUDO_ENABLE_HOOKS) {
34 if (__scudo_realloc_allocate_hook)
35 __scudo_realloc_allocate_hook(old_ptr, new_ptr, size);
36 else if (__scudo_allocate_hook)
37 __scudo_allocate_hook(ptr: new_ptr, size);
38 }
39}
40static void reportReallocDeallocation(void *old_ptr) {
41 if (SCUDO_ENABLE_HOOKS) {
42 if (__scudo_realloc_deallocate_hook)
43 __scudo_realloc_deallocate_hook(old_ptr);
44 else if (__scudo_deallocate_hook)
45 __scudo_deallocate_hook(ptr: old_ptr);
46 }
47}
48
49extern "C" {
50
51INTERFACE WEAK void *SCUDO_PREFIX(calloc)(size_t nmemb, size_t size) {
52 scudo::uptr Product;
53 if (UNLIKELY(scudo::checkForCallocOverflow(size, nmemb, &Product))) {
54 if (SCUDO_ALLOCATOR.canReturnNull()) {
55 errno = ENOMEM;
56 return nullptr;
57 }
58 scudo::reportCallocOverflow(Count: nmemb, Size: size);
59 }
60 void *Ptr = SCUDO_ALLOCATOR.allocate(Size: Product, Origin: scudo::Chunk::Origin::Malloc,
61 SCUDO_MALLOC_ALIGNMENT, ZeroContents: true);
62 reportAllocation(ptr: Ptr, size: Product);
63 return scudo::setErrnoOnNull(Ptr);
64}
65
66INTERFACE WEAK void SCUDO_PREFIX(free)(void *ptr) {
67 reportDeallocation(ptr);
68 SCUDO_ALLOCATOR.deallocate(Ptr: ptr, Origin: scudo::Chunk::Origin::Malloc);
69}
70
71INTERFACE WEAK struct SCUDO_MALLINFO SCUDO_PREFIX(mallinfo)(void) {
72 struct SCUDO_MALLINFO Info = {};
73 scudo::StatCounters Stats;
74 SCUDO_ALLOCATOR.getStats(S: Stats);
75 // Space allocated in mmapped regions (bytes)
76 Info.hblkhd = static_cast<__scudo_mallinfo_data_t>(Stats[scudo::StatMapped]);
77 // Maximum total allocated space (bytes)
78 Info.usmblks = Info.hblkhd;
79 // Space in freed fastbin blocks (bytes)
80 Info.fsmblks = static_cast<__scudo_mallinfo_data_t>(Stats[scudo::StatFree]);
81 // Total allocated space (bytes)
82 Info.uordblks =
83 static_cast<__scudo_mallinfo_data_t>(Stats[scudo::StatAllocated]);
84 // Total free space (bytes)
85 Info.fordblks = Info.fsmblks;
86 return Info;
87}
88
89// On Android, mallinfo2 is an alias of mallinfo, so don't define both.
90#if !SCUDO_ANDROID
91INTERFACE WEAK struct __scudo_mallinfo2 SCUDO_PREFIX(mallinfo2)(void) {
92 struct __scudo_mallinfo2 Info = {};
93 scudo::StatCounters Stats;
94 SCUDO_ALLOCATOR.getStats(S: Stats);
95 // Space allocated in mmapped regions (bytes)
96 Info.hblkhd = Stats[scudo::StatMapped];
97 // Maximum total allocated space (bytes)
98 Info.usmblks = Info.hblkhd;
99 // Space in freed fastbin blocks (bytes)
100 Info.fsmblks = Stats[scudo::StatFree];
101 // Total allocated space (bytes)
102 Info.uordblks = Stats[scudo::StatAllocated];
103 // Total free space (bytes)
104 Info.fordblks = Info.fsmblks;
105 return Info;
106}
107#endif
108
109INTERFACE WEAK void *SCUDO_PREFIX(malloc)(size_t size) {
110 void *Ptr = SCUDO_ALLOCATOR.allocate(Size: size, Origin: scudo::Chunk::Origin::Malloc,
111 SCUDO_MALLOC_ALIGNMENT);
112 reportAllocation(ptr: Ptr, size);
113 return scudo::setErrnoOnNull(Ptr);
114}
115
116#if SCUDO_ANDROID
117INTERFACE WEAK size_t SCUDO_PREFIX(malloc_usable_size)(const void *ptr) {
118#else
119INTERFACE WEAK size_t SCUDO_PREFIX(malloc_usable_size)(void *ptr) {
120#endif
121 return SCUDO_ALLOCATOR.getUsableSize(Ptr: ptr);
122}
123
124INTERFACE WEAK void *SCUDO_PREFIX(memalign)(size_t alignment, size_t size) {
125 // Android rounds up the alignment to a power of two if it isn't one.
126 if (SCUDO_ANDROID) {
127 if (UNLIKELY(!alignment)) {
128 alignment = 1U;
129 } else {
130 if (UNLIKELY(!scudo::isPowerOfTwo(alignment)))
131 alignment = scudo::roundUpPowerOfTwo(Size: alignment);
132 }
133 } else {
134 if (UNLIKELY(!scudo::isPowerOfTwo(alignment))) {
135 if (SCUDO_ALLOCATOR.canReturnNull()) {
136 errno = EINVAL;
137 return nullptr;
138 }
139 scudo::reportAlignmentNotPowerOfTwo(Alignment: alignment);
140 }
141 }
142 void *Ptr =
143 SCUDO_ALLOCATOR.allocate(Size: size, Origin: scudo::Chunk::Origin::Memalign, Alignment: alignment);
144 reportAllocation(ptr: Ptr, size);
145 return Ptr;
146}
147
148INTERFACE WEAK int SCUDO_PREFIX(posix_memalign)(void **memptr, size_t alignment,
149 size_t size) {
150 if (UNLIKELY(scudo::checkPosixMemalignAlignment(alignment))) {
151 if (!SCUDO_ALLOCATOR.canReturnNull())
152 scudo::reportInvalidPosixMemalignAlignment(Alignment: alignment);
153 return EINVAL;
154 }
155 void *Ptr =
156 SCUDO_ALLOCATOR.allocate(Size: size, Origin: scudo::Chunk::Origin::Memalign, Alignment: alignment);
157 if (UNLIKELY(!Ptr))
158 return ENOMEM;
159 reportAllocation(ptr: Ptr, size);
160
161 *memptr = Ptr;
162 return 0;
163}
164
165INTERFACE WEAK void *SCUDO_PREFIX(pvalloc)(size_t size) {
166 const scudo::uptr PageSize = scudo::getPageSizeCached();
167 if (UNLIKELY(scudo::checkForPvallocOverflow(size, PageSize))) {
168 if (SCUDO_ALLOCATOR.canReturnNull()) {
169 errno = ENOMEM;
170 return nullptr;
171 }
172 scudo::reportPvallocOverflow(Size: size);
173 }
174 // pvalloc(0) should allocate one page.
175 void *Ptr =
176 SCUDO_ALLOCATOR.allocate(Size: size ? scudo::roundUp(X: size, Boundary: PageSize) : PageSize,
177 Origin: scudo::Chunk::Origin::Memalign, Alignment: PageSize);
178 reportAllocation(ptr: Ptr, size: scudo::roundUp(X: size, Boundary: PageSize));
179
180 return scudo::setErrnoOnNull(Ptr);
181}
182
183INTERFACE WEAK void *SCUDO_PREFIX(realloc)(void *ptr, size_t size) {
184 if (!ptr) {
185 void *Ptr = SCUDO_ALLOCATOR.allocate(Size: size, Origin: scudo::Chunk::Origin::Malloc,
186 SCUDO_MALLOC_ALIGNMENT);
187 reportAllocation(ptr: Ptr, size);
188 return scudo::setErrnoOnNull(Ptr);
189 }
190 if (size == 0) {
191 reportDeallocation(ptr);
192 SCUDO_ALLOCATOR.deallocate(Ptr: ptr, Origin: scudo::Chunk::Origin::Malloc);
193 return nullptr;
194 }
195
196 // Given that the reporting of deallocation and allocation are not atomic, we
197 // always pretend the old pointer will be released so that the user doesn't
198 // need to worry about the false double-use case from the view of hooks.
199 //
200 // For example, assume that `realloc` releases the old pointer and allocates a
201 // new pointer. Before the reporting of both operations has been done, another
202 // thread may get the old pointer from `malloc`. It may be misinterpreted as
203 // double-use if it's not handled properly on the hook side.
204 reportReallocDeallocation(old_ptr: ptr);
205 void *NewPtr = SCUDO_ALLOCATOR.reallocate(OldPtr: ptr, NewSize: size, SCUDO_MALLOC_ALIGNMENT);
206 if (NewPtr != nullptr) {
207 // Note that even if NewPtr == ptr, the size has changed. We still need to
208 // report the new size.
209 reportReallocAllocation(/*OldPtr=*/old_ptr: ptr, new_ptr: NewPtr, size);
210 } else {
211 // If `realloc` fails, the old pointer is not released. Report the old
212 // pointer as allocated again.
213 reportReallocAllocation(/*OldPtr=*/old_ptr: ptr, /*NewPtr=*/new_ptr: ptr,
214 SCUDO_ALLOCATOR.getAllocSize(Ptr: ptr));
215 }
216
217 return scudo::setErrnoOnNull(NewPtr);
218}
219
220INTERFACE WEAK void *SCUDO_PREFIX(valloc)(size_t size) {
221 void *Ptr = SCUDO_ALLOCATOR.allocate(Size: size, Origin: scudo::Chunk::Origin::Memalign,
222 Alignment: scudo::getPageSizeCached());
223 reportAllocation(ptr: Ptr, size);
224
225 return scudo::setErrnoOnNull(Ptr);
226}
227
228INTERFACE WEAK int SCUDO_PREFIX(malloc_iterate)(
229 uintptr_t base, size_t size,
230 void (*callback)(uintptr_t base, size_t size, void *arg), void *arg) {
231 SCUDO_ALLOCATOR.iterateOverChunks(Base: base, Size: size, Callback: callback, Arg: arg);
232 return 0;
233}
234
235INTERFACE WEAK void SCUDO_PREFIX(malloc_enable)() { SCUDO_ALLOCATOR.enable(); }
236
237INTERFACE WEAK void SCUDO_PREFIX(malloc_disable)() {
238 SCUDO_ALLOCATOR.disable();
239}
240
241void SCUDO_PREFIX(malloc_postinit)() {
242 SCUDO_ALLOCATOR.initGwpAsan();
243 pthread_atfork(SCUDO_PREFIX(malloc_disable), SCUDO_PREFIX(malloc_enable),
244 SCUDO_PREFIX(malloc_enable));
245}
246
247INTERFACE WEAK int SCUDO_PREFIX(mallopt)(int param, int value) {
248 if (param == M_DECAY_TIME) {
249 if (SCUDO_ANDROID) {
250 // Before changing the interval, reset the memory usage status by doing a
251 // M_PURGE call so that we can minimize the impact of any unreleased pages
252 // introduced by interval transition.
253 SCUDO_ALLOCATOR.releaseToOS(ReleaseType: scudo::ReleaseToOS::Force);
254
255 // The values allowed on Android are {-1, 0, 1}. "1" means the longest
256 // interval.
257 CHECK(value >= -1 && value <= 1);
258 if (value == 1)
259 value = INT32_MAX;
260 }
261
262 SCUDO_ALLOCATOR.setOption(O: scudo::Option::ReleaseInterval,
263 Value: static_cast<scudo::sptr>(value));
264 return 1;
265 } else if (param == M_PURGE) {
266 SCUDO_ALLOCATOR.releaseToOS(ReleaseType: scudo::ReleaseToOS::Force);
267 return 1;
268 } else if (param == M_PURGE_ALL) {
269 SCUDO_ALLOCATOR.releaseToOS(ReleaseType: scudo::ReleaseToOS::ForceAll);
270 return 1;
271 } else if (param == M_LOG_STATS) {
272 SCUDO_ALLOCATOR.printStats();
273 SCUDO_ALLOCATOR.printFragmentationInfo();
274 return 1;
275 } else {
276 scudo::Option option;
277 switch (param) {
278 case M_MEMTAG_TUNING:
279 option = scudo::Option::MemtagTuning;
280 break;
281 case M_THREAD_DISABLE_MEM_INIT:
282 option = scudo::Option::ThreadDisableMemInit;
283 break;
284 case M_CACHE_COUNT_MAX:
285 option = scudo::Option::MaxCacheEntriesCount;
286 break;
287 case M_CACHE_SIZE_MAX:
288 option = scudo::Option::MaxCacheEntrySize;
289 break;
290 case M_TSDS_COUNT_MAX:
291 option = scudo::Option::MaxTSDsCount;
292 break;
293 default:
294 return 0;
295 }
296 return SCUDO_ALLOCATOR.setOption(O: option, Value: static_cast<scudo::sptr>(value));
297 }
298}
299
300INTERFACE WEAK void *SCUDO_PREFIX(aligned_alloc)(size_t alignment,
301 size_t size) {
302 if (UNLIKELY(scudo::checkAlignedAllocAlignmentAndSize(alignment, size))) {
303 if (SCUDO_ALLOCATOR.canReturnNull()) {
304 errno = EINVAL;
305 return nullptr;
306 }
307 scudo::reportInvalidAlignedAllocAlignment(Size: alignment, Alignment: size);
308 }
309
310 void *Ptr =
311 SCUDO_ALLOCATOR.allocate(Size: size, Origin: scudo::Chunk::Origin::Malloc, Alignment: alignment);
312 reportAllocation(ptr: Ptr, size);
313
314 return scudo::setErrnoOnNull(Ptr);
315}
316
317INTERFACE WEAK int SCUDO_PREFIX(malloc_info)(UNUSED int options, FILE *stream) {
318 const scudo::uptr max_size =
319 decltype(SCUDO_ALLOCATOR)::PrimaryT::SizeClassMap::MaxSize;
320 auto *sizes = static_cast<scudo::uptr *>(
321 SCUDO_PREFIX(calloc)(nmemb: max_size, size: sizeof(scudo::uptr)));
322 auto callback = [](uintptr_t, size_t size, void *arg) {
323 auto *sizes = reinterpret_cast<scudo::uptr *>(arg);
324 if (size < max_size)
325 sizes[size]++;
326 };
327
328 SCUDO_ALLOCATOR.disable();
329 SCUDO_ALLOCATOR.iterateOverChunks(Base: 0, Size: -1ul, Callback: callback, Arg: sizes);
330 SCUDO_ALLOCATOR.enable();
331
332 fputs(s: "<malloc version=\"scudo-1\">\n", stream: stream);
333 for (scudo::uptr i = 0; i != max_size; ++i)
334 if (sizes[i])
335 fprintf(stream: stream, format: "<alloc size=\"%zu\" count=\"%zu\"/>\n", i, sizes[i]);
336 fputs(s: "</malloc>\n", stream: stream);
337 SCUDO_PREFIX(free)(ptr: sizes);
338 return 0;
339}
340
341// Disable memory tagging for the heap. The caller must disable memory tag
342// checks globally (e.g. by clearing TCF0 on aarch64) before calling this
343// function, and may not re-enable them after calling the function.
344INTERFACE WEAK void SCUDO_PREFIX(malloc_disable_memory_tagging)() {
345 SCUDO_ALLOCATOR.disableMemoryTagging();
346}
347
348// Sets whether scudo records stack traces and other metadata for allocations
349// and deallocations. This function only has an effect if the allocator and
350// hardware support memory tagging.
351INTERFACE WEAK void
352SCUDO_PREFIX(malloc_set_track_allocation_stacks)(int track) {
353 SCUDO_ALLOCATOR.setTrackAllocationStacks(track);
354}
355
356// Sets whether scudo zero-initializes all allocated memory.
357INTERFACE WEAK void SCUDO_PREFIX(malloc_set_zero_contents)(int zero_contents) {
358 SCUDO_ALLOCATOR.setFillContents(zero_contents ? scudo::ZeroFill
359 : scudo::NoFill);
360}
361
362// Sets whether scudo pattern-initializes all allocated memory.
363INTERFACE WEAK void
364SCUDO_PREFIX(malloc_set_pattern_fill_contents)(int pattern_fill_contents) {
365 SCUDO_ALLOCATOR.setFillContents(
366 pattern_fill_contents ? scudo::PatternOrZeroFill : scudo::NoFill);
367}
368
369// Sets whether scudo adds a small amount of slack at the end of large
370// allocations, before the guard page. This can be enabled to work around buggy
371// applications that read a few bytes past the end of their allocation.
372INTERFACE WEAK void
373SCUDO_PREFIX(malloc_set_add_large_allocation_slack)(int add_slack) {
374 SCUDO_ALLOCATOR.setAddLargeAllocationSlack(add_slack);
375}
376
377} // extern "C"
378