1//===-- hwasan_allocator.cpp ------------------------ ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is a part of HWAddressSanitizer.
10//
11// HWAddressSanitizer allocator.
12//===----------------------------------------------------------------------===//
13
14#include "sanitizer_common/sanitizer_atomic.h"
15#include "sanitizer_common/sanitizer_errno.h"
16#include "sanitizer_common/sanitizer_stackdepot.h"
17#include "hwasan.h"
18#include "hwasan_allocator.h"
19#include "hwasan_checks.h"
20#include "hwasan_mapping.h"
21#include "hwasan_malloc_bisect.h"
22#include "hwasan_thread.h"
23#include "hwasan_report.h"
24#include "lsan/lsan_common.h"
25
26namespace __hwasan {
27
28static Allocator allocator;
29static AllocatorCache fallback_allocator_cache;
30static SpinMutex fallback_mutex;
31static atomic_uint8_t hwasan_allocator_tagging_enabled;
32
33static constexpr tag_t kFallbackAllocTag = 0xBB & kTagMask;
34static constexpr tag_t kFallbackFreeTag = 0xBC;
35
36enum {
37 // Either just allocated by underlying allocator, but AsanChunk is not yet
38 // ready, or almost returned to undelying allocator and AsanChunk is already
39 // meaningless.
40 CHUNK_INVALID = 0,
41 // The chunk is allocated and not yet freed.
42 CHUNK_ALLOCATED = 1,
43};
44
45
46// Initialized in HwasanAllocatorInit, an never changed.
47alignas(16) static u8 tail_magic[kShadowAlignment - 1];
48static uptr max_malloc_size;
49static unsigned hwasan_tag_bits;
50static tag_t fallback_alloc_tag;
51
52bool HwasanChunkView::IsAllocated() const {
53 return metadata_ && metadata_->IsAllocated();
54}
55
56uptr HwasanChunkView::Beg() const {
57 return block_;
58}
59uptr HwasanChunkView::End() const {
60 return Beg() + UsedSize();
61}
62uptr HwasanChunkView::UsedSize() const {
63 return metadata_->GetRequestedSize();
64}
65u32 HwasanChunkView::GetAllocStackId() const {
66 return metadata_->GetAllocStackId();
67}
68
69u32 HwasanChunkView::GetAllocThreadId() const {
70 return metadata_->GetAllocThreadId();
71}
72
73uptr HwasanChunkView::ActualSize() const {
74 return allocator.GetActuallyAllocatedSize(p: reinterpret_cast<void *>(block_));
75}
76
77bool HwasanChunkView::FromSmallHeap() const {
78 return allocator.FromPrimary(p: reinterpret_cast<void *>(block_));
79}
80
81bool HwasanChunkView::AddrIsInside(uptr addr) const {
82 return (addr >= Beg()) && (addr < Beg() + UsedSize());
83}
84
85inline void Metadata::SetAllocated(u32 stack, u64 size) {
86 Thread *t = GetCurrentThread();
87 u64 context = t ? t->unique_id() : kMainTid;
88 context <<= 32;
89 context += stack;
90 requested_size_low = size & ((1ul << 32) - 1);
91 requested_size_high = size >> 32;
92 atomic_store(a: &alloc_context_id, v: context, mo: memory_order_relaxed);
93 atomic_store(a: &chunk_state, v: CHUNK_ALLOCATED, mo: memory_order_release);
94}
95
96inline void Metadata::SetUnallocated() {
97 atomic_store(a: &chunk_state, v: CHUNK_INVALID, mo: memory_order_release);
98 requested_size_low = 0;
99 requested_size_high = 0;
100 atomic_store(a: &alloc_context_id, v: 0, mo: memory_order_relaxed);
101}
102
103inline bool Metadata::IsAllocated() const {
104 return atomic_load(a: &chunk_state, mo: memory_order_relaxed) == CHUNK_ALLOCATED;
105}
106
107inline u64 Metadata::GetRequestedSize() const {
108 return (static_cast<u64>(requested_size_high) << 32) + requested_size_low;
109}
110
111inline u32 Metadata::GetAllocStackId() const {
112 return atomic_load(a: &alloc_context_id, mo: memory_order_relaxed);
113}
114
115inline u32 Metadata::GetAllocThreadId() const {
116 u64 context = atomic_load(a: &alloc_context_id, mo: memory_order_relaxed);
117 u32 tid = context >> 32;
118 return tid;
119}
120
121void GetAllocatorStats(AllocatorStatCounters s) {
122 allocator.GetStats(s);
123}
124
125inline void Metadata::SetLsanTag(__lsan::ChunkTag tag) {
126 lsan_tag = tag;
127}
128
129inline __lsan::ChunkTag Metadata::GetLsanTag() const {
130 return static_cast<__lsan::ChunkTag>(lsan_tag);
131}
132
133uptr GetAliasRegionStart() {
134#if defined(HWASAN_ALIASING_MODE)
135 constexpr uptr kAliasRegionOffset = 1ULL << (kTaggableRegionCheckShift - 1);
136 uptr AliasRegionStart =
137 __hwasan_shadow_memory_dynamic_address + kAliasRegionOffset;
138
139 CHECK_EQ(AliasRegionStart >> kTaggableRegionCheckShift,
140 __hwasan_shadow_memory_dynamic_address >> kTaggableRegionCheckShift);
141 CHECK_EQ(
142 (AliasRegionStart + kAliasRegionOffset - 1) >> kTaggableRegionCheckShift,
143 __hwasan_shadow_memory_dynamic_address >> kTaggableRegionCheckShift);
144 return AliasRegionStart;
145#else
146 return 0;
147#endif
148}
149
150void HwasanAllocatorInit() {
151 atomic_store_relaxed(a: &hwasan_allocator_tagging_enabled,
152 v: !flags()->disable_allocator_tagging);
153 int flags_tag_bits = flags()->tag_bits;
154 if (flags_tag_bits < static_cast<int>(kTagBits) && flags_tag_bits > 0)
155 hwasan_tag_bits = flags_tag_bits;
156 else
157 hwasan_tag_bits = kTagBits;
158 // With flags_tag_bits we want to restrict the number of bits in the
159 // pointer. That's why we don't need to mask out the kFallbackFreeTag,
160 // because that one is only used for the memory tag, never the pointer
161 // tag.
162 fallback_alloc_tag = kFallbackAllocTag & ((1 << hwasan_tag_bits) - 1);
163 SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
164 allocator.InitLinkerInitialized(
165 release_to_os_interval_ms: common_flags()->allocator_release_to_os_interval_ms,
166 heap_start: GetAliasRegionStart());
167 for (uptr i = 0; i < sizeof(tail_magic); i++)
168 tail_magic[i] = GetCurrentThread()->GenerateRandomTag(num_bits: hwasan_tag_bits);
169 if (common_flags()->max_allocation_size_mb) {
170 max_malloc_size = common_flags()->max_allocation_size_mb << 20;
171 max_malloc_size = Min(a: max_malloc_size, b: kMaxAllowedMallocSize);
172 } else {
173 max_malloc_size = kMaxAllowedMallocSize;
174 }
175}
176
177void HwasanAllocatorLock() { allocator.ForceLock(); }
178
179void HwasanAllocatorUnlock() { allocator.ForceUnlock(); }
180
181void AllocatorThreadStart(AllocatorCache *cache) { allocator.InitCache(cache); }
182
183void AllocatorThreadFinish(AllocatorCache *cache) {
184 allocator.SwallowCache(cache);
185 allocator.DestroyCache(cache);
186}
187
188static uptr TaggedSize(uptr size) {
189 if (!size) size = 1;
190 uptr new_size = RoundUpTo(size, boundary: kShadowAlignment);
191 CHECK_GE(new_size, size);
192 return new_size;
193}
194
195static void *HwasanAllocate(StackTrace *stack, uptr orig_size, uptr alignment,
196 bool zeroise) {
197 // Keep this consistent with LSAN and ASAN behavior.
198 if (UNLIKELY(orig_size == 0))
199 orig_size = 1;
200 if (UNLIKELY(orig_size > max_malloc_size)) {
201 if (AllocatorMayReturnNull()) {
202 Report(format: "WARNING: HWAddressSanitizer failed to allocate 0x%zx bytes\n",
203 orig_size);
204 return nullptr;
205 }
206 ReportAllocationSizeTooBig(user_size: orig_size, max_size: max_malloc_size, stack);
207 }
208 if (UNLIKELY(IsRssLimitExceeded())) {
209 if (AllocatorMayReturnNull())
210 return nullptr;
211 ReportRssLimitExceeded(stack);
212 }
213
214 alignment = Max(a: alignment, b: kShadowAlignment);
215 uptr size = TaggedSize(size: orig_size);
216 Thread *t = GetCurrentThread();
217 void *allocated;
218 if (t) {
219 allocated = allocator.Allocate(cache: t->allocator_cache(), size, alignment);
220 } else {
221 SpinMutexLock l(&fallback_mutex);
222 AllocatorCache *cache = &fallback_allocator_cache;
223 allocated = allocator.Allocate(cache, size, alignment);
224 }
225 if (UNLIKELY(!allocated)) {
226 SetAllocatorOutOfMemory();
227 if (AllocatorMayReturnNull())
228 return nullptr;
229 ReportOutOfMemory(requested_size: size, stack);
230 }
231 if (zeroise) {
232 // The secondary allocator mmaps memory, which should be zero-inited so we
233 // don't need to explicitly clear it.
234 if (allocator.FromPrimary(p: allocated))
235 internal_memset(s: allocated, c: 0, n: size);
236 } else if (flags()->max_malloc_fill_size > 0) {
237 uptr fill_size = Min(a: size, b: (uptr)flags()->max_malloc_fill_size);
238 internal_memset(s: allocated, c: flags()->malloc_fill_byte, n: fill_size);
239 }
240 if (size != orig_size) {
241 u8 *tail = reinterpret_cast<u8 *>(allocated) + orig_size;
242 uptr tail_length = size - orig_size;
243 internal_memcpy(dest: tail, src: tail_magic, n: tail_length - 1);
244 // Short granule is excluded from magic tail, so we explicitly untag.
245 tail[tail_length - 1] = 0;
246 }
247
248 void *user_ptr = allocated;
249 if (InTaggableRegion(addr: reinterpret_cast<uptr>(user_ptr)) &&
250 atomic_load_relaxed(a: &hwasan_allocator_tagging_enabled) &&
251 flags()->tag_in_malloc && malloc_bisect(stack, orig_size)) {
252 tag_t tag = t ? t->GenerateRandomTag(num_bits: hwasan_tag_bits) : fallback_alloc_tag;
253 uptr tag_size = orig_size ? orig_size : 1;
254 uptr full_granule_size = RoundDownTo(x: tag_size, boundary: kShadowAlignment);
255 user_ptr = (void *)TagMemoryAligned(p: (uptr)user_ptr, size: full_granule_size, tag);
256 if (full_granule_size != tag_size) {
257 u8 *short_granule = reinterpret_cast<u8 *>(allocated) + full_granule_size;
258 TagMemoryAligned(p: (uptr)short_granule, size: kShadowAlignment,
259 tag: tag_size % kShadowAlignment);
260 short_granule[kShadowAlignment - 1] = tag;
261 }
262 } else {
263 // Tagging can not be completely skipped. If it's disabled, we need to tag
264 // with zeros.
265 user_ptr = (void *)TagMemoryAligned(p: (uptr)user_ptr, size, tag: 0);
266 }
267
268 Metadata *meta =
269 reinterpret_cast<Metadata *>(allocator.GetMetaData(p: allocated));
270#if CAN_SANITIZE_LEAKS
271 meta->SetLsanTag(__lsan::DisabledInThisThread() ? __lsan::kIgnored
272 : __lsan::kDirectlyLeaked);
273#endif
274 meta->SetAllocated(stack: StackDepotPut(stack: *stack), size: orig_size);
275 RunMallocHooks(ptr: user_ptr, size: orig_size);
276 return user_ptr;
277}
278
279static bool PointerAndMemoryTagsMatch(void *tagged_ptr) {
280 CHECK(tagged_ptr);
281 uptr tagged_uptr = reinterpret_cast<uptr>(tagged_ptr);
282 if (!InTaggableRegion(addr: tagged_uptr))
283 return true;
284 tag_t mem_tag = *reinterpret_cast<tag_t *>(
285 MemToShadow(untagged_addr: reinterpret_cast<uptr>(UntagPtr(tagged_ptr))));
286 return PossiblyShortTagMatches(mem_tag, ptr: tagged_uptr, sz: 1);
287}
288
289static bool CheckInvalidFree(StackTrace *stack, void *untagged_ptr,
290 void *tagged_ptr) {
291 // This function can return true if halt_on_error is false.
292 if (!MemIsApp(p: reinterpret_cast<uptr>(untagged_ptr)) ||
293 !PointerAndMemoryTagsMatch(tagged_ptr)) {
294 ReportInvalidFree(stack, addr: reinterpret_cast<uptr>(tagged_ptr));
295 return true;
296 }
297 return false;
298}
299
300static void HwasanDeallocate(StackTrace *stack, void *tagged_ptr) {
301 CHECK(tagged_ptr);
302 void *untagged_ptr = UntagPtr(tagged_ptr);
303
304 if (RunFreeHooks(ptr: tagged_ptr))
305 return;
306
307 if (CheckInvalidFree(stack, untagged_ptr, tagged_ptr))
308 return;
309
310 void *aligned_ptr = reinterpret_cast<void *>(
311 RoundDownTo(x: reinterpret_cast<uptr>(untagged_ptr), boundary: kShadowAlignment));
312 tag_t pointer_tag = GetTagFromPointer(p: reinterpret_cast<uptr>(tagged_ptr));
313 Metadata *meta =
314 reinterpret_cast<Metadata *>(allocator.GetMetaData(p: aligned_ptr));
315 if (!meta) {
316 ReportInvalidFree(stack, addr: reinterpret_cast<uptr>(tagged_ptr));
317 return;
318 }
319
320 uptr orig_size = meta->GetRequestedSize();
321 u32 free_context_id = StackDepotPut(stack: *stack);
322 u32 alloc_context_id = meta->GetAllocStackId();
323 u32 alloc_thread_id = meta->GetAllocThreadId();
324
325 bool in_taggable_region =
326 InTaggableRegion(addr: reinterpret_cast<uptr>(tagged_ptr));
327
328 // Check tail magic.
329 uptr tagged_size = TaggedSize(size: orig_size);
330 if (flags()->free_checks_tail_magic && orig_size &&
331 tagged_size != orig_size) {
332 uptr tail_size = tagged_size - orig_size - 1;
333 CHECK_LT(tail_size, kShadowAlignment);
334 void *tail_beg = reinterpret_cast<void *>(
335 reinterpret_cast<uptr>(aligned_ptr) + orig_size);
336 tag_t short_granule_memtag = *(reinterpret_cast<tag_t *>(
337 reinterpret_cast<uptr>(tail_beg) + tail_size));
338 if (tail_size &&
339 (internal_memcmp(s1: tail_beg, s2: tail_magic, n: tail_size) ||
340 (in_taggable_region && pointer_tag != short_granule_memtag)))
341 ReportTailOverwritten(stack, addr: reinterpret_cast<uptr>(tagged_ptr),
342 orig_size, expected: tail_magic);
343 }
344
345 // TODO(kstoimenov): consider meta->SetUnallocated(free_context_id).
346 meta->SetUnallocated();
347 // This memory will not be reused by anyone else, so we are free to keep it
348 // poisoned.
349 Thread *t = GetCurrentThread();
350 if (flags()->max_free_fill_size > 0) {
351 uptr fill_size =
352 Min(a: TaggedSize(size: orig_size), b: (uptr)flags()->max_free_fill_size);
353 internal_memset(s: aligned_ptr, c: flags()->free_fill_byte, n: fill_size);
354 }
355 if (in_taggable_region && flags()->tag_in_free && malloc_bisect(stack, orig_size: 0) &&
356 atomic_load_relaxed(a: &hwasan_allocator_tagging_enabled) &&
357 allocator.FromPrimary(p: untagged_ptr) /* Secondary 0-tag and unmap.*/) {
358 // Always store full 8-bit tags on free to maximize UAF detection.
359 tag_t tag;
360 if (t) {
361 // Make sure we are not using a short granule tag as a poison tag. This
362 // would make us attempt to read the memory on a UaF.
363 // The tag can be zero if tagging is disabled on this thread.
364 do {
365 tag = t->GenerateRandomTag(/*num_bits=*/8);
366 } while (
367 UNLIKELY((tag < kShadowAlignment || tag == pointer_tag) && tag != 0));
368 } else {
369 static_assert(kFallbackFreeTag >= kShadowAlignment,
370 "fallback tag must not be a short granule tag.");
371 tag = kFallbackFreeTag;
372 }
373 TagMemoryAligned(p: reinterpret_cast<uptr>(aligned_ptr), size: TaggedSize(size: orig_size),
374 tag);
375 }
376 if (t) {
377 allocator.Deallocate(cache: t->allocator_cache(), p: aligned_ptr);
378 if (auto *ha = t->heap_allocations())
379 ha->push(t: {.tagged_addr: reinterpret_cast<uptr>(tagged_ptr), .alloc_thread_id: alloc_thread_id,
380 .alloc_context_id: alloc_context_id, .free_context_id: free_context_id,
381 .requested_size: static_cast<u32>(orig_size)});
382 } else {
383 SpinMutexLock l(&fallback_mutex);
384 AllocatorCache *cache = &fallback_allocator_cache;
385 allocator.Deallocate(cache, p: aligned_ptr);
386 }
387}
388
389static void *HwasanReallocate(StackTrace *stack, void *tagged_ptr_old,
390 uptr new_size, uptr alignment) {
391 void *untagged_ptr_old = UntagPtr(tagged_ptr: tagged_ptr_old);
392 if (CheckInvalidFree(stack, untagged_ptr: untagged_ptr_old, tagged_ptr: tagged_ptr_old))
393 return nullptr;
394 void *tagged_ptr_new =
395 HwasanAllocate(stack, orig_size: new_size, alignment, zeroise: false /*zeroise*/);
396 if (tagged_ptr_old && tagged_ptr_new) {
397 Metadata *meta =
398 reinterpret_cast<Metadata *>(allocator.GetMetaData(p: untagged_ptr_old));
399 void *untagged_ptr_new = UntagPtr(tagged_ptr: tagged_ptr_new);
400 internal_memcpy(dest: untagged_ptr_new, src: untagged_ptr_old,
401 n: Min(a: new_size, b: static_cast<uptr>(meta->GetRequestedSize())));
402 HwasanDeallocate(stack, tagged_ptr: tagged_ptr_old);
403 }
404 return tagged_ptr_new;
405}
406
407static void *HwasanCalloc(StackTrace *stack, uptr nmemb, uptr size) {
408 if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
409 if (AllocatorMayReturnNull())
410 return nullptr;
411 ReportCallocOverflow(count: nmemb, size, stack);
412 }
413 return HwasanAllocate(stack, orig_size: nmemb * size, alignment: sizeof(u64), zeroise: true);
414}
415
416HwasanChunkView FindHeapChunkByAddress(uptr address) {
417 if (!allocator.PointerIsMine(p: reinterpret_cast<void *>(address)))
418 return HwasanChunkView();
419 void *block = allocator.GetBlockBegin(p: reinterpret_cast<void*>(address));
420 if (!block)
421 return HwasanChunkView();
422 Metadata *metadata =
423 reinterpret_cast<Metadata*>(allocator.GetMetaData(p: block));
424 return HwasanChunkView(reinterpret_cast<uptr>(block), metadata);
425}
426
427static const void *AllocationBegin(const void *p) {
428 const void *untagged_ptr = UntagPtr(tagged_ptr: p);
429 if (!untagged_ptr)
430 return nullptr;
431
432 const void *beg = allocator.GetBlockBegin(p: untagged_ptr);
433 if (!beg)
434 return nullptr;
435
436 Metadata *b = (Metadata *)allocator.GetMetaData(p: beg);
437 if (b->GetRequestedSize() == 0)
438 return nullptr;
439
440 tag_t tag = GetTagFromPointer(p: (uptr)p);
441 return (const void *)AddTagToPointer(p: (uptr)beg, tag);
442}
443
444static uptr AllocationSize(const void *p) {
445 const void *untagged_ptr = UntagPtr(tagged_ptr: p);
446 if (!untagged_ptr) return 0;
447 const void *beg = allocator.GetBlockBegin(p: untagged_ptr);
448 if (!beg)
449 return 0;
450 Metadata *b = (Metadata *)allocator.GetMetaData(p: beg);
451 return b->GetRequestedSize();
452}
453
454static uptr AllocationSizeFast(const void *p) {
455 const void *untagged_ptr = UntagPtr(tagged_ptr: p);
456 void *aligned_ptr = reinterpret_cast<void *>(
457 RoundDownTo(x: reinterpret_cast<uptr>(untagged_ptr), boundary: kShadowAlignment));
458 Metadata *meta =
459 reinterpret_cast<Metadata *>(allocator.GetMetaData(p: aligned_ptr));
460 return meta->GetRequestedSize();
461}
462
463void *hwasan_malloc(uptr size, StackTrace *stack) {
464 return SetErrnoOnNull(HwasanAllocate(stack, orig_size: size, alignment: sizeof(u64), zeroise: false));
465}
466
467void *hwasan_calloc(uptr nmemb, uptr size, StackTrace *stack) {
468 return SetErrnoOnNull(HwasanCalloc(stack, nmemb, size));
469}
470
471void *hwasan_realloc(void *ptr, uptr size, StackTrace *stack) {
472 if (!ptr)
473 return SetErrnoOnNull(HwasanAllocate(stack, orig_size: size, alignment: sizeof(u64), zeroise: false));
474 if (size == 0) {
475 HwasanDeallocate(stack, tagged_ptr: ptr);
476 return nullptr;
477 }
478 return SetErrnoOnNull(HwasanReallocate(stack, tagged_ptr_old: ptr, new_size: size, alignment: sizeof(u64)));
479}
480
481void *hwasan_reallocarray(void *ptr, uptr nmemb, uptr size, StackTrace *stack) {
482 if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
483 errno = errno_ENOMEM;
484 if (AllocatorMayReturnNull())
485 return nullptr;
486 ReportReallocArrayOverflow(count: nmemb, size, stack);
487 }
488 return hwasan_realloc(ptr, size: nmemb * size, stack);
489}
490
491void *hwasan_valloc(uptr size, StackTrace *stack) {
492 return SetErrnoOnNull(
493 HwasanAllocate(stack, orig_size: size, alignment: GetPageSizeCached(), zeroise: false));
494}
495
496void *hwasan_pvalloc(uptr size, StackTrace *stack) {
497 uptr PageSize = GetPageSizeCached();
498 if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) {
499 errno = errno_ENOMEM;
500 if (AllocatorMayReturnNull())
501 return nullptr;
502 ReportPvallocOverflow(size, stack);
503 }
504 // pvalloc(0) should allocate one page.
505 size = size ? RoundUpTo(size, boundary: PageSize) : PageSize;
506 return SetErrnoOnNull(HwasanAllocate(stack, orig_size: size, alignment: PageSize, zeroise: false));
507}
508
509void *hwasan_aligned_alloc(uptr alignment, uptr size, StackTrace *stack) {
510 if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) {
511 errno = errno_EINVAL;
512 if (AllocatorMayReturnNull())
513 return nullptr;
514 ReportInvalidAlignedAllocAlignment(size, alignment, stack);
515 }
516 return SetErrnoOnNull(HwasanAllocate(stack, orig_size: size, alignment, zeroise: false));
517}
518
519void *hwasan_memalign(uptr alignment, uptr size, StackTrace *stack) {
520 if (UNLIKELY(!IsPowerOfTwo(alignment))) {
521 errno = errno_EINVAL;
522 if (AllocatorMayReturnNull())
523 return nullptr;
524 ReportInvalidAllocationAlignment(alignment, stack);
525 }
526 return SetErrnoOnNull(HwasanAllocate(stack, orig_size: size, alignment, zeroise: false));
527}
528
529int hwasan_posix_memalign(void **memptr, uptr alignment, uptr size,
530 StackTrace *stack) {
531 if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) {
532 if (AllocatorMayReturnNull())
533 return errno_EINVAL;
534 ReportInvalidPosixMemalignAlignment(alignment, stack);
535 }
536 void *ptr = HwasanAllocate(stack, orig_size: size, alignment, zeroise: false);
537 if (UNLIKELY(!ptr))
538 // OOM error is already taken care of by HwasanAllocate.
539 return errno_ENOMEM;
540 CHECK(IsAligned((uptr)ptr, alignment));
541 *memptr = ptr;
542 return 0;
543}
544
545void hwasan_free(void *ptr, StackTrace *stack) {
546 return HwasanDeallocate(stack, tagged_ptr: ptr);
547}
548
549} // namespace __hwasan
550
551// --- Implementation of LSan-specific functions --- {{{1
552namespace __lsan {
553
554void LockAllocator() {
555 __hwasan::HwasanAllocatorLock();
556}
557
558void UnlockAllocator() {
559 __hwasan::HwasanAllocatorUnlock();
560}
561
562void GetAllocatorGlobalRange(uptr *begin, uptr *end) {
563 *begin = (uptr)&__hwasan::allocator;
564 *end = *begin + sizeof(__hwasan::allocator);
565}
566
567uptr PointsIntoChunk(void *p) {
568 p = UntagPtr(tagged_ptr: p);
569 uptr addr = reinterpret_cast<uptr>(p);
570 uptr chunk =
571 reinterpret_cast<uptr>(__hwasan::allocator.GetBlockBeginFastLocked(p));
572 if (!chunk)
573 return 0;
574 __hwasan::Metadata *metadata = reinterpret_cast<__hwasan::Metadata *>(
575 __hwasan::allocator.GetMetaData(p: reinterpret_cast<void *>(chunk)));
576 if (!metadata || !metadata->IsAllocated())
577 return 0;
578 if (addr < chunk + metadata->GetRequestedSize())
579 return chunk;
580 if (IsSpecialCaseOfOperatorNew0(chunk_beg: chunk, chunk_size: metadata->GetRequestedSize(), addr))
581 return chunk;
582 return 0;
583}
584
585uptr GetUserBegin(uptr chunk) {
586 CHECK_EQ(UntagAddr(chunk), chunk);
587 void *block = __hwasan::allocator.GetBlockBeginFastLocked(
588 p: reinterpret_cast<void *>(chunk));
589 if (!block)
590 return 0;
591 __hwasan::Metadata *metadata = reinterpret_cast<__hwasan::Metadata *>(
592 __hwasan::allocator.GetMetaData(p: block));
593 if (!metadata || !metadata->IsAllocated())
594 return 0;
595
596 return reinterpret_cast<uptr>(block);
597}
598
599uptr GetUserAddr(uptr chunk) {
600 if (!InTaggableRegion(addr: chunk))
601 return chunk;
602 tag_t mem_tag = *(tag_t *)__hwasan::MemToShadow(untagged_addr: chunk);
603 return AddTagToPointer(p: chunk, tag: mem_tag);
604}
605
606LsanMetadata::LsanMetadata(uptr chunk) {
607 CHECK_EQ(UntagAddr(chunk), chunk);
608 metadata_ =
609 chunk ? __hwasan::allocator.GetMetaData(p: reinterpret_cast<void *>(chunk))
610 : nullptr;
611}
612
613bool LsanMetadata::allocated() const {
614 if (!metadata_)
615 return false;
616 __hwasan::Metadata *m = reinterpret_cast<__hwasan::Metadata *>(metadata_);
617 return m->IsAllocated();
618}
619
620ChunkTag LsanMetadata::tag() const {
621 __hwasan::Metadata *m = reinterpret_cast<__hwasan::Metadata *>(metadata_);
622 return m->GetLsanTag();
623}
624
625void LsanMetadata::set_tag(ChunkTag value) {
626 __hwasan::Metadata *m = reinterpret_cast<__hwasan::Metadata *>(metadata_);
627 m->SetLsanTag(value);
628}
629
630uptr LsanMetadata::requested_size() const {
631 __hwasan::Metadata *m = reinterpret_cast<__hwasan::Metadata *>(metadata_);
632 return m->GetRequestedSize();
633}
634
635u32 LsanMetadata::stack_trace_id() const {
636 __hwasan::Metadata *m = reinterpret_cast<__hwasan::Metadata *>(metadata_);
637 return m->GetAllocStackId();
638}
639
640void ForEachChunk(ForEachChunkCallback callback, void *arg) {
641 __hwasan::allocator.ForEachChunk(callback, arg);
642}
643
644IgnoreObjectResult IgnoreObject(const void *p) {
645 p = UntagPtr(tagged_ptr: p);
646 uptr addr = reinterpret_cast<uptr>(p);
647 uptr chunk = reinterpret_cast<uptr>(__hwasan::allocator.GetBlockBegin(p));
648 if (!chunk)
649 return kIgnoreObjectInvalid;
650 __hwasan::Metadata *metadata = reinterpret_cast<__hwasan::Metadata *>(
651 __hwasan::allocator.GetMetaData(p: reinterpret_cast<void *>(chunk)));
652 if (!metadata || !metadata->IsAllocated())
653 return kIgnoreObjectInvalid;
654 if (addr >= chunk + metadata->GetRequestedSize())
655 return kIgnoreObjectInvalid;
656 if (metadata->GetLsanTag() == kIgnored)
657 return kIgnoreObjectAlreadyIgnored;
658
659 metadata->SetLsanTag(kIgnored);
660 return kIgnoreObjectSuccess;
661}
662
663} // namespace __lsan
664
665using namespace __hwasan;
666
667void __hwasan_enable_allocator_tagging() {
668 atomic_store_relaxed(a: &hwasan_allocator_tagging_enabled, v: 1);
669}
670
671void __hwasan_disable_allocator_tagging() {
672 atomic_store_relaxed(a: &hwasan_allocator_tagging_enabled, v: 0);
673}
674
675uptr __sanitizer_get_current_allocated_bytes() {
676 uptr stats[AllocatorStatCount];
677 allocator.GetStats(s: stats);
678 return stats[AllocatorStatAllocated];
679}
680
681uptr __sanitizer_get_heap_size() {
682 uptr stats[AllocatorStatCount];
683 allocator.GetStats(s: stats);
684 return stats[AllocatorStatMapped];
685}
686
687uptr __sanitizer_get_free_bytes() { return 1; }
688
689uptr __sanitizer_get_unmapped_bytes() { return 1; }
690
691uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; }
692
693int __sanitizer_get_ownership(const void *p) { return AllocationSize(p) != 0; }
694
695const void *__sanitizer_get_allocated_begin(const void *p) {
696 return AllocationBegin(p);
697}
698
699uptr __sanitizer_get_allocated_size(const void *p) { return AllocationSize(p); }
700
701uptr __sanitizer_get_allocated_size_fast(const void *p) {
702 DCHECK_EQ(p, __sanitizer_get_allocated_begin(p));
703 uptr ret = AllocationSizeFast(p);
704 DCHECK_EQ(ret, __sanitizer_get_allocated_size(p));
705 return ret;
706}
707
708void __sanitizer_purge_allocator() { allocator.ForceReleaseToOS(); }
709