1//===-- hwasan_linux.cpp ----------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file is a part of HWAddressSanitizer and contains Linux-, NetBSD- and
11/// FreeBSD-specific code.
12///
13//===----------------------------------------------------------------------===//
14
15#include "sanitizer_common/sanitizer_platform.h"
16#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD
17
18# include <dlfcn.h>
19# include <elf.h>
20# include <errno.h>
21# include <link.h>
22# include <pthread.h>
23# include <signal.h>
24# include <stdio.h>
25# include <stdlib.h>
26# include <sys/prctl.h>
27# include <sys/resource.h>
28# include <sys/time.h>
29# include <unistd.h>
30# include <unwind.h>
31
32# include "hwasan.h"
33# include "hwasan_dynamic_shadow.h"
34# include "hwasan_interface_internal.h"
35# include "hwasan_mapping.h"
36# include "hwasan_report.h"
37# include "hwasan_thread.h"
38# include "hwasan_thread_list.h"
39# include "sanitizer_common/sanitizer_common.h"
40# include "sanitizer_common/sanitizer_dl.h"
41# include "sanitizer_common/sanitizer_procmaps.h"
42# include "sanitizer_common/sanitizer_stackdepot.h"
43
44// Configurations of HWASAN_WITH_INTERCEPTORS and SANITIZER_ANDROID.
45//
46// HWASAN_WITH_INTERCEPTORS=OFF, SANITIZER_ANDROID=OFF
47// Not currently tested.
48// HWASAN_WITH_INTERCEPTORS=OFF, SANITIZER_ANDROID=ON
49// Integration tests downstream exist.
50// HWASAN_WITH_INTERCEPTORS=ON, SANITIZER_ANDROID=OFF
51// Tested with check-hwasan on x86_64-linux.
52// HWASAN_WITH_INTERCEPTORS=ON, SANITIZER_ANDROID=ON
53// Tested with check-hwasan on aarch64-linux-android.
54# if !SANITIZER_ANDROID
55SANITIZER_INTERFACE_ATTRIBUTE
56THREADLOCAL uptr __hwasan_tls;
57# endif
58
59namespace __hwasan {
60
61// With the zero shadow base we can not actually map pages starting from 0.
62// This constant is somewhat arbitrary.
63constexpr uptr kZeroBaseShadowStart = 0;
64constexpr uptr kZeroBaseMaxShadowStart = 1 << 18;
65
66static void ProtectGap(uptr addr, uptr size) {
67 __sanitizer::ProtectGap(addr, size, zero_base_shadow_start: kZeroBaseShadowStart,
68 zero_base_max_shadow_start: kZeroBaseMaxShadowStart);
69}
70
71uptr kLowMemStart;
72uptr kLowMemEnd;
73uptr kHighMemStart;
74uptr kHighMemEnd;
75
76static void PrintRange(uptr start, uptr end, const char *name) {
77 Printf(format: "|| [%p, %p] || %.*s ||\n", (void *)start, (void *)end, 10, name);
78}
79
80static void PrintAddressSpaceLayout() {
81 PrintRange(start: kHighMemStart, end: kHighMemEnd, name: "HighMem");
82 if (kHighShadowEnd + 1 < kHighMemStart)
83 PrintRange(start: kHighShadowEnd + 1, end: kHighMemStart - 1, name: "ShadowGap");
84 else
85 CHECK_EQ(kHighShadowEnd + 1, kHighMemStart);
86 PrintRange(start: kHighShadowStart, end: kHighShadowEnd, name: "HighShadow");
87 if (kLowShadowEnd + 1 < kHighShadowStart)
88 PrintRange(start: kLowShadowEnd + 1, end: kHighShadowStart - 1, name: "ShadowGap");
89 else
90 CHECK_EQ(kLowMemEnd + 1, kHighShadowStart);
91 PrintRange(start: kLowShadowStart, end: kLowShadowEnd, name: "LowShadow");
92 if (kLowMemEnd + 1 < kLowShadowStart)
93 PrintRange(start: kLowMemEnd + 1, end: kLowShadowStart - 1, name: "ShadowGap");
94 else
95 CHECK_EQ(kLowMemEnd + 1, kLowShadowStart);
96 PrintRange(start: kLowMemStart, end: kLowMemEnd, name: "LowMem");
97 CHECK_EQ(0, kLowMemStart);
98}
99
100static uptr GetHighMemEnd() {
101 // HighMem covers the upper part of the address space.
102 uptr max_address = GetMaxUserVirtualAddress();
103 // Adjust max address to make sure that kHighMemEnd and kHighMemStart are
104 // properly aligned:
105 max_address |= (GetMmapGranularity() << kShadowScale) - 1;
106 return max_address;
107}
108
109static void InitializeShadowBaseAddress(uptr shadow_size_bytes) {
110 // FIXME: Android should init flags before shadow.
111 if (!SANITIZER_ANDROID && flags()->fixed_shadow_base != (uptr)-1) {
112 __hwasan_shadow_memory_dynamic_address = flags()->fixed_shadow_base;
113 uptr beg = __hwasan_shadow_memory_dynamic_address;
114 uptr end = beg + shadow_size_bytes;
115 if (!MemoryRangeIsAvailable(range_start: beg, range_end: end)) {
116 Report(
117 format: "FATAL: HWAddressSanitizer: Shadow range %p-%p is not available.\n",
118 (void *)beg, (void *)end);
119 DumpProcessMap();
120 CHECK(MemoryRangeIsAvailable(beg, end));
121 }
122 } else {
123 __hwasan_shadow_memory_dynamic_address =
124 FindDynamicShadowStart(shadow_size_bytes);
125 }
126}
127
128static void MaybeDieIfNoTaggingAbi(const char *message) {
129 if (!flags()->fail_without_syscall_abi)
130 return;
131 Printf(format: "FATAL: %s\n", message);
132 Die();
133}
134
135# define PR_SET_TAGGED_ADDR_CTRL 55
136# define PR_GET_TAGGED_ADDR_CTRL 56
137# define PR_TAGGED_ADDR_ENABLE (1UL << 0)
138# define PR_PMLEN_SHIFT 24
139# define ARCH_GET_UNTAG_MASK 0x4001
140# define ARCH_ENABLE_TAGGED_ADDR 0x4002
141# define ARCH_GET_MAX_TAG_BITS 0x4003
142
143static bool CanUseTaggingAbi() {
144# if defined(__x86_64__)
145 unsigned long num_bits = 0;
146 // Check for x86 LAM support. This API is based on a currently unsubmitted
147 // patch to the Linux kernel (as of August 2022) and is thus subject to
148 // change. The patch is here:
149 // https://lore.kernel.org/all/20220815041803.17954-1-kirill.shutemov@linux.intel.com/
150 //
151 // arch_prctl(ARCH_GET_MAX_TAG_BITS, &bits) returns the maximum number of tag
152 // bits the user can request, or zero if LAM is not supported by the hardware.
153 if (internal_iserror(retval: internal_arch_prctl(ARCH_GET_MAX_TAG_BITS,
154 arg2: reinterpret_cast<uptr>(&num_bits))))
155 return false;
156 // The platform must provide enough bits for HWASan tags.
157 if (num_bits < kTagBits)
158 return false;
159 return true;
160# else
161 // Check for ARM TBI support.
162 return !internal_iserror(internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0));
163# endif // __x86_64__
164}
165
166static bool EnableTaggingAbi() {
167# if defined(__x86_64__)
168 // Enable x86 LAM tagging for the process.
169 //
170 // arch_prctl(ARCH_ENABLE_TAGGED_ADDR, bits) enables tagging if the number of
171 // tag bits requested by the user does not exceed that provided by the system.
172 // arch_prctl(ARCH_GET_UNTAG_MASK, &mask) returns the mask of significant
173 // address bits. It is ~0ULL if either LAM is disabled for the process or LAM
174 // is not supported by the hardware.
175 if (internal_iserror(retval: internal_arch_prctl(ARCH_ENABLE_TAGGED_ADDR, arg2: kTagBits)))
176 return false;
177 unsigned long mask = 0;
178 // Make sure the tag bits are where we expect them to be.
179 if (internal_iserror(retval: internal_arch_prctl(ARCH_GET_UNTAG_MASK,
180 arg2: reinterpret_cast<uptr>(&mask))))
181 return false;
182 // @mask has ones for non-tag bits, whereas @kAddressTagMask has ones for tag
183 // bits. Therefore these masks must not overlap.
184 if (mask & kAddressTagMask)
185 return false;
186 return true;
187# elif defined(__aarch64__)
188 // Enable ARM TBI tagging for the process. If for some reason tagging is not
189 // supported, prctl(PR_SET_TAGGED_ADDR_CTRL, PR_TAGGED_ADDR_ENABLE) returns
190 // -EINVAL.
191 if (internal_iserror(internal_prctl(PR_SET_TAGGED_ADDR_CTRL,
192 PR_TAGGED_ADDR_ENABLE, 0, 0, 0)))
193 return false;
194 // Ensure that TBI is enabled.
195 if (internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0) !=
196 PR_TAGGED_ADDR_ENABLE)
197 return false;
198 return true;
199# elif SANITIZER_RISCV64
200 // Enable RISC-V address tagging via pointer masking.
201 uptr req = kTagBits << PR_PMLEN_SHIFT | PR_TAGGED_ADDR_ENABLE;
202 if (internal_iserror(internal_prctl(PR_SET_TAGGED_ADDR_CTRL, req, 0, 0, 0)))
203 return false;
204 uptr rsp = internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0);
205 if (internal_iserror(rsp))
206 return false;
207 return rsp & PR_TAGGED_ADDR_ENABLE;
208# else
209# error Architecture not supported
210# endif // __x86_64__
211}
212
213void InitializeOsSupport() {
214 // Check we're running on a kernel that can use the tagged address ABI.
215 bool has_abi = CanUseTaggingAbi();
216
217 if (!has_abi) {
218# if SANITIZER_ANDROID || defined(HWASAN_ALIASING_MODE)
219 // Some older Android kernels have the tagged pointer ABI on
220 // unconditionally, and hence don't have the tagged-addr prctl while still
221 // allow the ABI.
222 // If targeting Android and the prctl is not around we assume this is the
223 // case.
224 return;
225# else
226 MaybeDieIfNoTaggingAbi(
227 message: "HWAddressSanitizer requires a kernel with tagged address ABI.");
228# endif
229 }
230
231 if (EnableTaggingAbi())
232 return;
233
234# if SANITIZER_ANDROID
235 MaybeDieIfNoTaggingAbi(
236 "HWAddressSanitizer failed to enable tagged address syscall ABI.\n"
237 "Check the `sysctl abi.tagged_addr_disabled` configuration.");
238# else
239 MaybeDieIfNoTaggingAbi(
240 message: "HWAddressSanitizer failed to enable tagged address syscall ABI.\n");
241# endif
242}
243
244bool InitShadow() {
245 // Define the entire memory range.
246 kHighMemEnd = GetHighMemEnd();
247
248 // Determine shadow memory base offset.
249 InitializeShadowBaseAddress(shadow_size_bytes: MemToShadowSize(size: kHighMemEnd));
250
251 // Place the low memory first.
252 kLowMemEnd = __hwasan_shadow_memory_dynamic_address - 1;
253 kLowMemStart = 0;
254
255 // Define the low shadow based on the already placed low memory.
256 kLowShadowEnd = MemToShadow(untagged_addr: kLowMemEnd);
257 kLowShadowStart = __hwasan_shadow_memory_dynamic_address;
258
259 // High shadow takes whatever memory is left up there (making sure it is not
260 // interfering with low memory in the fixed case).
261 kHighShadowEnd = MemToShadow(untagged_addr: kHighMemEnd);
262 kHighShadowStart = Max(a: kLowMemEnd, b: MemToShadow(untagged_addr: kHighShadowEnd)) + 1;
263
264 // High memory starts where allocated shadow allows.
265 kHighMemStart = ShadowToMem(shadow_addr: kHighShadowStart);
266
267 // Check the sanity of the defined memory ranges (there might be gaps).
268 CHECK_EQ(kHighMemStart % GetMmapGranularity(), 0);
269 CHECK_GT(kHighMemStart, kHighShadowEnd);
270 CHECK_GT(kHighShadowEnd, kHighShadowStart);
271 CHECK_GT(kHighShadowStart, kLowMemEnd);
272 CHECK_GT(kLowMemEnd, kLowMemStart);
273 CHECK_GT(kLowShadowEnd, kLowShadowStart);
274 CHECK_GT(kLowShadowStart, kLowMemEnd);
275
276 // Reserve shadow memory.
277 ReserveShadowMemoryRange(beg: kLowShadowStart, end: kLowShadowEnd, name: "low shadow");
278 ReserveShadowMemoryRange(beg: kHighShadowStart, end: kHighShadowEnd, name: "high shadow");
279
280 // Protect all the gaps.
281 ProtectGap(addr: 0, size: Min(a: kLowMemStart, b: kLowShadowStart));
282 if (kLowMemEnd + 1 < kLowShadowStart)
283 ProtectGap(addr: kLowMemEnd + 1, size: kLowShadowStart - kLowMemEnd - 1);
284 if (kLowShadowEnd + 1 < kHighShadowStart)
285 ProtectGap(addr: kLowShadowEnd + 1, size: kHighShadowStart - kLowShadowEnd - 1);
286 if (kHighShadowEnd + 1 < kHighMemStart)
287 ProtectGap(addr: kHighShadowEnd + 1, size: kHighMemStart - kHighShadowEnd - 1);
288
289 if (Verbosity())
290 PrintAddressSpaceLayout();
291
292 return true;
293}
294
295void InitThreads() {
296 CHECK(__hwasan_shadow_memory_dynamic_address);
297 uptr guard_page_size = GetMmapGranularity();
298 uptr thread_space_start =
299 __hwasan_shadow_memory_dynamic_address - (1ULL << kShadowBaseAlignment);
300 uptr thread_space_end =
301 __hwasan_shadow_memory_dynamic_address - guard_page_size;
302 ReserveShadowMemoryRange(beg: thread_space_start, end: thread_space_end - 1,
303 name: "hwasan threads", /*madvise_shadow*/ false);
304 ProtectGap(addr: thread_space_end,
305 size: __hwasan_shadow_memory_dynamic_address - thread_space_end);
306 InitThreadList(storage: thread_space_start, size: thread_space_end - thread_space_start);
307 hwasanThreadList().CreateCurrentThread();
308}
309
310bool MemIsApp(uptr p) {
311// Memory outside the alias range has non-zero tags.
312# if !defined(HWASAN_ALIASING_MODE)
313 CHECK_EQ(GetTagFromPointer(p), 0);
314# endif
315
316 return (p >= kHighMemStart && p <= kHighMemEnd) ||
317 (p >= kLowMemStart && p <= kLowMemEnd);
318}
319
320void InstallAtExitHandler() { atexit(func: HwasanAtExit); }
321
322// ---------------------- TSD ---------------- {{{1
323
324# if HWASAN_WITH_INTERCEPTORS
325static pthread_key_t tsd_key;
326static bool tsd_key_inited = false;
327
328void HwasanTSDThreadInit() {
329 if (tsd_key_inited)
330 CHECK_EQ(0, pthread_setspecific(tsd_key,
331 (void *)GetPthreadDestructorIterations()));
332}
333
334void HwasanTSDDtor(void *tsd) {
335 uptr iterations = (uptr)tsd;
336 if (iterations > 1) {
337 CHECK_EQ(0, pthread_setspecific(tsd_key, (void *)(iterations - 1)));
338 return;
339 }
340 __hwasan_thread_exit();
341}
342
343void HwasanTSDInit() {
344 CHECK(!tsd_key_inited);
345 tsd_key_inited = true;
346 CHECK_EQ(0, pthread_key_create(&tsd_key, HwasanTSDDtor));
347}
348# else
349void HwasanTSDInit() {}
350void HwasanTSDThreadInit() {}
351# endif
352
353# if SANITIZER_ANDROID
354uptr *GetCurrentThreadLongPtr() { return (uptr *)get_android_tls_ptr(); }
355# else
356uptr *GetCurrentThreadLongPtr() { return &__hwasan_tls; }
357# endif
358
359# if SANITIZER_ANDROID
360void AndroidTestTlsSlot() {
361 uptr kMagicValue = 0x010203040A0B0C0D;
362 uptr *tls_ptr = GetCurrentThreadLongPtr();
363 uptr old_value = *tls_ptr;
364 *tls_ptr = kMagicValue;
365 dlerror();
366 if (*(uptr *)get_android_tls_ptr() != kMagicValue) {
367 Printf(
368 "ERROR: Incompatible version of Android: TLS_SLOT_SANITIZER(6) is used "
369 "for dlerror().\n");
370 Die();
371 }
372 *tls_ptr = old_value;
373}
374# else
375void AndroidTestTlsSlot() {}
376# endif
377
378static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) {
379 // Access type is passed in a platform dependent way (see below) and encoded
380 // as 0xXY, where X&1 is 1 for store, 0 for load, and X&2 is 1 if the error is
381 // recoverable. Valid values of Y are 0 to 4, which are interpreted as
382 // log2(access_size), and 0xF, which means that access size is passed via
383 // platform dependent register (see below).
384# if defined(__aarch64__)
385 // Access type is encoded in BRK immediate as 0x900 + 0xXY. For Y == 0xF,
386 // access size is stored in X1 register. Access address is always in X0
387 // register.
388 uptr pc = (uptr)info->si_addr;
389 const unsigned code = ((*(u32 *)pc) >> 5) & 0xffff;
390 if ((code & 0xff00) != 0x900)
391 return AccessInfo{}; // Not ours.
392
393 const bool is_store = code & 0x10;
394 const bool recover = code & 0x20;
395 const uptr addr = uc->uc_mcontext.regs[0];
396 const unsigned size_log = code & 0xf;
397 if (size_log > 4 && size_log != 0xf)
398 return AccessInfo{}; // Not ours.
399 const uptr size = size_log == 0xf ? uc->uc_mcontext.regs[1] : 1U << size_log;
400
401# elif defined(__x86_64__)
402 // Access type is encoded in the instruction following INT3 as
403 // NOP DWORD ptr [EAX + 0x40 + 0xXY]. For Y == 0xF, access size is stored in
404 // RSI register. Access address is always in RDI register.
405 uptr pc = (uptr)uc->uc_mcontext.gregs[REG_RIP];
406 uint8_t *nop = (uint8_t *)pc;
407 if (*nop != 0x0f || *(nop + 1) != 0x1f || *(nop + 2) != 0x40 ||
408 *(nop + 3) < 0x40)
409 return AccessInfo{}; // Not ours.
410 const unsigned code = *(nop + 3);
411
412 const bool is_store = code & 0x10;
413 const bool recover = code & 0x20;
414 const uptr addr = uc->uc_mcontext.gregs[REG_RDI];
415 const unsigned size_log = code & 0xf;
416 if (size_log > 4 && size_log != 0xf)
417 return AccessInfo{}; // Not ours.
418 const uptr size =
419 size_log == 0xf ? uc->uc_mcontext.gregs[REG_RSI] : 1U << size_log;
420
421# elif SANITIZER_RISCV64
422 // Access type is encoded in the instruction following EBREAK as
423 // ADDI x0, x0, [0x40 + 0xXY]. For Y == 0xF, access size is stored in
424 // X11 register. Access address is always in X10 register.
425 uptr pc = (uptr)uc->uc_mcontext.__gregs[REG_PC];
426 uint8_t byte1 = *((u8 *)(pc + 0));
427 uint8_t byte2 = *((u8 *)(pc + 1));
428 uint8_t byte3 = *((u8 *)(pc + 2));
429 uint8_t byte4 = *((u8 *)(pc + 3));
430 uint32_t ebreak = (byte1 | (byte2 << 8) | (byte3 << 16) | (byte4 << 24));
431 bool isFaultShort = false;
432 bool isEbreak = (ebreak == 0x100073);
433 bool isShortEbreak = false;
434# if defined(__riscv_compressed)
435 isFaultShort = ((ebreak & 0x3) != 0x3);
436 isShortEbreak = ((ebreak & 0xffff) == 0x9002);
437# endif
438 // faulted insn is not ebreak, not our case
439 if (!(isEbreak || isShortEbreak))
440 return AccessInfo{};
441 // advance pc to point after ebreak and reconstruct addi instruction
442 pc += isFaultShort ? 2 : 4;
443 byte1 = *((u8 *)(pc + 0));
444 byte2 = *((u8 *)(pc + 1));
445 byte3 = *((u8 *)(pc + 2));
446 byte4 = *((u8 *)(pc + 3));
447 // reconstruct instruction
448 uint32_t instr = (byte1 | (byte2 << 8) | (byte3 << 16) | (byte4 << 24));
449 // check if this is really 32 bit instruction
450 // code is encoded in top 12 bits, since instruction is supposed to be with
451 // imm
452 const unsigned code = (instr >> 20) & 0xffff;
453 const uptr addr = uc->uc_mcontext.__gregs[10];
454 const bool is_store = code & 0x10;
455 const bool recover = code & 0x20;
456 const unsigned size_log = code & 0xf;
457 if (size_log > 4 && size_log != 0xf)
458 return AccessInfo{}; // Not our case
459 const uptr size =
460 size_log == 0xf ? uc->uc_mcontext.__gregs[11] : 1U << size_log;
461
462# else
463# error Unsupported architecture
464# endif
465
466 return AccessInfo{.addr: addr, .size: size, .is_store: is_store, .is_load: !is_store, .recover: recover};
467}
468
469static bool HwasanOnSIGTRAP(int signo, siginfo_t *info, ucontext_t *uc) {
470 AccessInfo ai = GetAccessInfo(info, uc);
471 if (!ai.is_store && !ai.is_load)
472 return false;
473
474 SignalContext sig{info, uc};
475 HandleTagMismatch(ai, pc: StackTrace::GetNextInstructionPc(pc: sig.pc), frame: sig.bp, uc);
476
477# if defined(__aarch64__)
478 uc->uc_mcontext.pc += 4;
479# elif defined(__x86_64__)
480# elif SANITIZER_RISCV64
481 // pc points to EBREAK which is 2 bytes long
482 uint8_t *exception_source = (uint8_t *)(uc->uc_mcontext.__gregs[REG_PC]);
483 uint8_t byte1 = (uint8_t)(*(exception_source + 0));
484 uint8_t byte2 = (uint8_t)(*(exception_source + 1));
485 uint8_t byte3 = (uint8_t)(*(exception_source + 2));
486 uint8_t byte4 = (uint8_t)(*(exception_source + 3));
487 uint32_t faulted = (byte1 | (byte2 << 8) | (byte3 << 16) | (byte4 << 24));
488 bool isFaultShort = false;
489# if defined(__riscv_compressed)
490 isFaultShort = ((faulted & 0x3) != 0x3);
491# endif
492 uc->uc_mcontext.__gregs[REG_PC] += isFaultShort ? 2 : 4;
493# else
494# error Unsupported architecture
495# endif
496 return true;
497}
498
499static void OnStackUnwind(const SignalContext &sig, const void *,
500 BufferedStackTrace *stack) {
501 stack->Unwind(pc: StackTrace::GetNextInstructionPc(pc: sig.pc), bp: sig.bp, context: sig.context,
502 request_fast: common_flags()->fast_unwind_on_fatal);
503}
504
505void HwasanOnDeadlySignal(int signo, void *info, void *context) {
506 // Probably a tag mismatch.
507 if (signo == SIGTRAP)
508 if (HwasanOnSIGTRAP(signo, info: (siginfo_t *)info, uc: (ucontext_t *)context))
509 return;
510
511 HandleDeadlySignal(siginfo: info, context, tid: GetTid(), unwind: &OnStackUnwind, unwind_context: nullptr);
512}
513
514void Thread::InitStackAndTls(const InitState *) {
515 GetThreadStackAndTls(main: IsMainThread(), stk_begin: &stack_bottom_, stk_end: &stack_top_, tls_begin: &tls_begin_,
516 tls_end: &tls_end_);
517}
518
519uptr TagMemoryAligned(uptr p, uptr size, tag_t tag) {
520 CHECK(IsAligned(p, kShadowAlignment));
521 CHECK(IsAligned(size, kShadowAlignment));
522 uptr shadow_start = MemToShadow(untagged_addr: p);
523 uptr shadow_size = MemToShadowSize(size);
524
525 uptr page_size = GetPageSizeCached();
526 uptr page_start = RoundUpTo(size: shadow_start, boundary: page_size);
527 uptr page_end = RoundDownTo(x: shadow_start + shadow_size, boundary: page_size);
528 uptr threshold = common_flags()->clear_shadow_mmap_threshold;
529 if (SANITIZER_LINUX &&
530 UNLIKELY(page_end >= page_start + threshold && tag == 0)) {
531 internal_memset(s: (void *)shadow_start, c: tag, n: page_start - shadow_start);
532 internal_memset(s: (void *)page_end, c: tag,
533 n: shadow_start + shadow_size - page_end);
534 // For an anonymous private mapping MADV_DONTNEED will return a zero page on
535 // Linux.
536 ReleaseMemoryPagesToOSAndZeroFill(beg: page_start, end: page_end);
537 } else {
538 internal_memset(s: (void *)shadow_start, c: tag, n: shadow_size);
539 }
540 return AddTagToPointer(p, tag);
541}
542
543static void BeforeFork() {
544 VReport(2, "BeforeFork tid: %llu\n", GetTid());
545 if (CAN_SANITIZE_LEAKS) {
546 __lsan::LockGlobal();
547 }
548 // `_lsan` functions defined regardless of `CAN_SANITIZE_LEAKS` and lock the
549 // stuff we need.
550 __lsan::LockThreads();
551 __lsan::LockAllocator();
552 StackDepotLockBeforeFork();
553}
554
555static void AfterFork(bool fork_child) {
556 StackDepotUnlockAfterFork(fork_child);
557 // `_lsan` functions defined regardless of `CAN_SANITIZE_LEAKS` and unlock
558 // the stuff we need.
559 __lsan::UnlockAllocator();
560 __lsan::UnlockThreads();
561 if (CAN_SANITIZE_LEAKS) {
562 __lsan::UnlockGlobal();
563 }
564 VReport(2, "AfterFork tid: %llu\n", GetTid());
565}
566
567void HwasanInstallAtForkHandler() {
568 pthread_atfork(
569 prepare: &BeforeFork, parent: []() { AfterFork(/* fork_child= */ false); },
570 child: []() { AfterFork(/* fork_child= */ true); });
571}
572
573void InstallAtExitCheckLeaks() {
574 if (CAN_SANITIZE_LEAKS) {
575 if (common_flags()->detect_leaks && common_flags()->leak_check_at_exit) {
576 if (flags()->halt_on_error)
577 Atexit(function: __lsan::DoLeakCheck);
578 else
579 Atexit(function: __lsan::DoRecoverableLeakCheckVoid);
580 }
581 }
582}
583
584} // namespace __hwasan
585
586using namespace __hwasan;
587
588extern "C" void __hwasan_thread_enter() {
589 hwasanThreadList().CreateCurrentThread()->EnsureRandomStateInited();
590}
591
592extern "C" void __hwasan_thread_exit() {
593 Thread *t = GetCurrentThread();
594 // Make sure that signal handler can not see a stale current thread pointer.
595 atomic_signal_fence(mo: memory_order_seq_cst);
596 if (t) {
597 // Block async signals on the thread as the handler can be instrumented.
598 // After this point instrumented code can't access essential data from TLS
599 // and will crash.
600 // Bionic already calls __hwasan_thread_exit with blocked signals.
601 if (SANITIZER_GLIBC)
602 BlockSignals();
603 ClearDlerror();
604 hwasanThreadList().ReleaseThread(t);
605 }
606}
607
608#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD
609