1//===-- tsan_platform_linux.cpp -------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is a part of ThreadSanitizer (TSan), a race detector.
10//
11// Linux- and BSD-specific code.
12//===----------------------------------------------------------------------===//
13
14#include "sanitizer_common/sanitizer_platform.h"
15#if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD
16
17#include "sanitizer_common/sanitizer_common.h"
18#include "sanitizer_common/sanitizer_libc.h"
19#include "sanitizer_common/sanitizer_linux.h"
20#include "sanitizer_common/sanitizer_platform_limits_netbsd.h"
21#include "sanitizer_common/sanitizer_platform_limits_posix.h"
22#include "sanitizer_common/sanitizer_posix.h"
23#include "sanitizer_common/sanitizer_procmaps.h"
24#include "sanitizer_common/sanitizer_stackdepot.h"
25#include "sanitizer_common/sanitizer_stoptheworld.h"
26#include "tsan_flags.h"
27#include "tsan_platform.h"
28#include "tsan_rtl.h"
29
30#if SANITIZER_NETBSD
31# // for __lwp_gettcb_fast() / __lwp_getprivate_fast()
32# define _RTLD_SOURCE
33# include <sys/types.h>
34# include <machine/mcontext.h>
35# undef _RTLD_SOURCE
36# include <sys/param.h>
37# if __NetBSD_Version__ >= 1099001200
38# include <machine/lwp_private.h>
39# endif
40#endif
41
42#include <fcntl.h>
43#include <pthread.h>
44#include <signal.h>
45#include <stdio.h>
46#include <stdlib.h>
47#include <string.h>
48#include <stdarg.h>
49#include <sys/mman.h>
50#if SANITIZER_LINUX
51#include <sys/personality.h>
52#include <setjmp.h>
53#endif
54#include <sys/syscall.h>
55#include <sys/socket.h>
56#include <sys/time.h>
57#include <sys/types.h>
58#include <sys/resource.h>
59#include <sys/stat.h>
60#include <unistd.h>
61#include <sched.h>
62#include <dlfcn.h>
63#if SANITIZER_LINUX
64#define __need_res_state
65#include <resolv.h>
66#endif
67
68#ifdef sa_handler
69# undef sa_handler
70#endif
71
72#ifdef sa_sigaction
73# undef sa_sigaction
74#endif
75
76#if SANITIZER_FREEBSD
77extern "C" void *__libc_stack_end;
78void *__libc_stack_end = 0;
79#endif
80
81# if SANITIZER_LINUX && !SANITIZER_GO
82# define CALIBRATE_LONGJMP_SP 1
83# else
84# define CALIBRATE_LONGJMP_SP 0
85# endif
86
87# if CALIBRATE_LONGJMP_SP
88# include "interception/interception.h"
89// Must be declared outside of other namespaces.
90DECLARE_REAL(int, _setjmp, void *env)
91#endif
92
93namespace __tsan {
94
95# if CALIBRATE_LONGJMP_SP
96static void InitializeLongjmpSpMangling();
97// libc stores rotl(sp ^ longjmp_xor_key, longjmp_rot) in jmp_buf.
98// Both values are zero when the stack pointer is stored unchanged.
99static atomic_uint8_t longjmp_sp_inited;
100static atomic_uintptr_t longjmp_xor_key;
101static atomic_uintptr_t longjmp_rot;
102# endif
103
104// Runtime detected VMA size.
105uptr vmaSize;
106
107enum {
108 MemTotal,
109 MemShadow,
110 MemMeta,
111 MemFile,
112 MemMmap,
113 MemHeap,
114 MemOther,
115 MemCount,
116};
117
118void FillProfileCallback(uptr p, uptr rss, bool file, uptr *mem) {
119 mem[MemTotal] += rss;
120 if (p >= ShadowBeg() && p < ShadowEnd())
121 mem[MemShadow] += rss;
122 else if (p >= MetaShadowBeg() && p < MetaShadowEnd())
123 mem[MemMeta] += rss;
124 else if ((p >= LoAppMemBeg() && p < LoAppMemEnd()) ||
125 (p >= MidAppMemBeg() && p < MidAppMemEnd()) ||
126 (p >= HiAppMemBeg() && p < HiAppMemEnd()))
127 mem[file ? MemFile : MemMmap] += rss;
128 else if (p >= HeapMemBeg() && p < HeapMemEnd())
129 mem[MemHeap] += rss;
130 else
131 mem[MemOther] += rss;
132}
133
134void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns) {
135 uptr mem[MemCount];
136 internal_memset(s: mem, c: 0, n: sizeof(mem));
137 GetMemoryProfile(cb: FillProfileCallback, stats: mem);
138 auto meta = ctx->metamap.GetMemoryStats();
139 StackDepotStats stacks = StackDepotGetStats();
140 uptr nthread, nlive;
141 ctx->thread_registry.GetNumberOfThreads(total: &nthread, running: &nlive);
142 uptr trace_mem;
143 {
144 Lock l(&ctx->slot_mtx);
145 trace_mem = ctx->trace_part_total_allocated * sizeof(TracePart);
146 }
147 uptr internal_stats[AllocatorStatCount];
148 internal_allocator()->GetStats(s: internal_stats);
149 // All these are allocated from the common mmap region.
150 mem[MemMmap] -= meta.mem_block + meta.sync_obj + trace_mem +
151 stacks.allocated + internal_stats[AllocatorStatMapped];
152 if (s64(mem[MemMmap]) < 0)
153 mem[MemMmap] = 0;
154 internal_snprintf(
155 buffer: buf, length: buf_size,
156 format: "==%zu== %llus [%zu]: RSS %zd MB: shadow:%zd meta:%zd file:%zd"
157 " mmap:%zd heap:%zd other:%zd intalloc:%zd memblocks:%zd syncobj:%zu"
158 " trace:%zu stacks=%zd threads=%zu/%zu\n",
159 internal_getpid(), uptime_ns / (1000 * 1000 * 1000), ctx->global_epoch,
160 mem[MemTotal] >> 20, mem[MemShadow] >> 20, mem[MemMeta] >> 20,
161 mem[MemFile] >> 20, mem[MemMmap] >> 20, mem[MemHeap] >> 20,
162 mem[MemOther] >> 20, internal_stats[AllocatorStatMapped] >> 20,
163 meta.mem_block >> 20, meta.sync_obj >> 20, trace_mem >> 20,
164 stacks.allocated >> 20, nlive, nthread);
165}
166
167#if !SANITIZER_GO
168// Mark shadow for .rodata sections with the special Shadow::kRodata marker.
169// Accesses to .rodata can't race, so this saves time, memory and trace space.
170static NOINLINE void MapRodata(char* buffer, uptr size) {
171 // First create temp file.
172 const char *tmpdir = GetEnv(name: "TMPDIR");
173 if (tmpdir == 0)
174 tmpdir = GetEnv(name: "TEST_TMPDIR");
175#ifdef P_tmpdir
176 if (tmpdir == 0)
177 tmpdir = P_tmpdir;
178#endif
179 if (tmpdir == 0)
180 return;
181 internal_snprintf(buffer, length: size, format: "%s/tsan.rodata.%d",
182 tmpdir, (int)internal_getpid());
183 uptr openrv = internal_open(filename: buffer, O_RDWR | O_CREAT | O_EXCL, mode: 0600);
184 if (internal_iserror(retval: openrv))
185 return;
186 internal_unlink(path: buffer); // Unlink it now, so that we can reuse the buffer.
187 fd_t fd = openrv;
188 // Fill the file with Shadow::kRodata.
189 const uptr kMarkerSize = 512 * 1024 / sizeof(RawShadow);
190 InternalMmapVector<RawShadow> marker(kMarkerSize);
191 // volatile to prevent insertion of memset
192 for (volatile RawShadow *p = marker.data(); p < marker.data() + kMarkerSize;
193 p++)
194 *p = Shadow::kRodata;
195 internal_write(fd, buf: marker.data(), count: marker.size() * sizeof(RawShadow));
196 // Map the file into memory.
197 uptr page = internal_mmap(addr: 0, length: GetPageSizeCached(), PROT_READ | PROT_WRITE,
198 MAP_PRIVATE | MAP_ANONYMOUS, fd, offset: 0);
199 if (internal_iserror(retval: page)) {
200 internal_close(fd);
201 return;
202 }
203 // Map the file into shadow of .rodata sections.
204 MemoryMappingLayout proc_maps(/*cache_enabled*/true);
205 // Reusing the buffer 'buffer'.
206 MemoryMappedSegment segment(buffer, size);
207 while (proc_maps.Next(segment: &segment)) {
208 if (segment.filename[0] != 0 && segment.filename[0] != '[' &&
209 segment.IsReadable() && segment.IsExecutable() &&
210 !segment.IsWritable() && IsAppMem(mem: segment.start)) {
211 // Assume it's .rodata
212 char *shadow_start = (char *)MemToShadow(x: segment.start);
213 char *shadow_end = (char *)MemToShadow(x: segment.end);
214 for (char *p = shadow_start; p < shadow_end;
215 p += marker.size() * sizeof(RawShadow)) {
216 internal_mmap(
217 addr: p, length: Min<uptr>(a: marker.size() * sizeof(RawShadow), b: shadow_end - p),
218 PROT_READ, MAP_PRIVATE | MAP_FIXED, fd, offset: 0);
219 }
220 }
221 }
222 internal_close(fd);
223}
224
225void InitializeShadowMemoryPlatform() {
226 char buffer[256]; // Keep in a different frame.
227 MapRodata(buffer, size: sizeof(buffer));
228}
229
230#endif // #if !SANITIZER_GO
231
232# if !SANITIZER_GO
233static void ReExecIfNeeded(bool ignore_heap) {
234 // Go maps shadow memory lazily and works fine with limited address space.
235 // Unlimited stack is not a problem as well, because the executable
236 // is not compiled with -pie.
237 bool reexec = false;
238 // TSan doesn't play well with unlimited stack size (as stack
239 // overlaps with shadow memory). If we detect unlimited stack size,
240 // we re-exec the program with limited stack size as a best effort.
241 if (StackSizeIsUnlimited()) {
242 const uptr kMaxStackSize = 32 * 1024 * 1024;
243 VReport(1,
244 "Program is run with unlimited stack size, which wouldn't "
245 "work with ThreadSanitizer.\n"
246 "Re-execing with stack size limited to %zd bytes.\n",
247 kMaxStackSize);
248 SetStackSizeLimitInBytes(kMaxStackSize);
249 reexec = true;
250 }
251
252 if (!AddressSpaceIsUnlimited()) {
253 Report(
254 format: "WARNING: Program is run with limited virtual address space,"
255 " which wouldn't work with ThreadSanitizer.\n");
256 Report(format: "Re-execing with unlimited virtual address space.\n");
257 SetAddressSpaceUnlimited();
258 reexec = true;
259 }
260
261# if SANITIZER_LINUX
262# if SANITIZER_ANDROID && (defined(__aarch64__) || defined(__x86_64__))
263 // ASLR personality check.
264 int old_personality = personality(0xffffffff);
265 bool aslr_on =
266 (old_personality != -1) && ((old_personality & ADDR_NO_RANDOMIZE) == 0);
267
268 // After patch "arm64: mm: support ARCH_MMAP_RND_BITS." is introduced in
269 // linux kernel, the random gap between stack and mapped area is increased
270 // from 128M to 36G on 39-bit aarch64. As it is almost impossible to cover
271 // this big range, we should disable randomized virtual space on aarch64.
272 if (aslr_on) {
273 VReport(1,
274 "WARNING: Program is run with randomized virtual address "
275 "space, which wouldn't work with ThreadSanitizer on Android.\n"
276 "Re-execing with fixed virtual address space.\n");
277
278 if (personality(old_personality | ADDR_NO_RANDOMIZE) == -1) {
279 Printf(
280 "FATAL: ThreadSanitizer: unable to disable ASLR (perhaps "
281 "sandboxing is enabled?).\n");
282 Printf("FATAL: Please rerun without sandboxing and/or ASLR.\n");
283 Die();
284 }
285
286 reexec = true;
287 }
288# endif
289
290 if (reexec) {
291 // Don't check the address space since we're going to re-exec anyway.
292 } else if (!CheckAndProtect(protect: false, ignore_heap, print_warnings: false)) {
293 // ASLR personality check.
294 // N.B. 'personality' is sometimes forbidden by sandboxes, so we only call
295 // this as a last resort (when the memory mapping is incompatible and TSan
296 // would fail anyway).
297 int old_personality = personality(persona: 0xffffffff);
298 bool aslr_on =
299 (old_personality != -1) && ((old_personality & ADDR_NO_RANDOMIZE) == 0);
300
301 if (aslr_on) {
302 // Disable ASLR if the memory layout was incompatible.
303 // Alternatively, we could just keep re-execing until we get lucky
304 // with a compatible randomized layout, but the risk is that if it's
305 // not an ASLR-related issue, we will be stuck in an infinite loop of
306 // re-execing (unless we change ReExec to pass a parameter of the
307 // number of retries allowed.)
308 VReport(1,
309 "WARNING: ThreadSanitizer: memory layout is incompatible, "
310 "possibly due to high-entropy ASLR.\n"
311 "Re-execing with fixed virtual address space.\n"
312 "N.B. reducing ASLR entropy is preferable.\n");
313
314 if (personality(persona: old_personality | ADDR_NO_RANDOMIZE) == -1) {
315 Printf(
316 format: "FATAL: ThreadSanitizer: encountered an incompatible memory "
317 "layout but was unable to disable ASLR (perhaps sandboxing is "
318 "enabled?).\n");
319 Printf(
320 format: "FATAL: Please rerun with lower ASLR entropy, ASLR disabled, "
321 "and/or sandboxing disabled.\n");
322 Die();
323 }
324
325 reexec = true;
326 } else {
327 Printf(
328 format: "FATAL: ThreadSanitizer: memory layout is incompatible, "
329 "even though ASLR is disabled.\n"
330 "Please file a bug.\n");
331 DumpProcessMap();
332 Die();
333 }
334 }
335# endif // SANITIZER_LINUX
336
337 if (reexec)
338 ReExec();
339}
340# endif
341
342void InitializePlatformEarly() {
343 vmaSize =
344 (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1);
345#if defined(__aarch64__)
346# if !SANITIZER_GO
347 if (vmaSize != 39 && vmaSize != 42 && vmaSize != 47 && vmaSize != 48) {
348 Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
349 Printf("FATAL: Found %zd - Supported 39, 42, 47 and 48\n", vmaSize);
350 Die();
351 }
352# else
353 if (vmaSize != 47 && vmaSize != 48) {
354 Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
355 Printf("FATAL: Found %zd - Supported 47 and 48\n", vmaSize);
356 Die();
357 }
358# endif
359# elif SANITIZER_LOONGARCH64
360# if !SANITIZER_GO
361 if (vmaSize != 47) {
362 Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
363 Printf("FATAL: Found %zd - Supported 47\n", vmaSize);
364 Die();
365 }
366# else
367 if (vmaSize != 47) {
368 Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
369 Printf("FATAL: Found %zd - Supported 47\n", vmaSize);
370 Die();
371 }
372# endif
373# elif defined(__powerpc64__)
374# if !SANITIZER_GO
375 if (vmaSize != 44 && vmaSize != 46 && vmaSize != 47) {
376 Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
377 Printf("FATAL: Found %zd - Supported 44, 46, and 47\n", vmaSize);
378 Die();
379 }
380# else
381 if (vmaSize != 46 && vmaSize != 47) {
382 Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
383 Printf("FATAL: Found %zd - Supported 46, and 47\n", vmaSize);
384 Die();
385 }
386# endif
387# elif defined(__mips64)
388# if !SANITIZER_GO
389 if (vmaSize != 40) {
390 Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
391 Printf("FATAL: Found %zd - Supported 40\n", vmaSize);
392 Die();
393 }
394# else
395 if (vmaSize != 47) {
396 Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
397 Printf("FATAL: Found %zd - Supported 47\n", vmaSize);
398 Die();
399 }
400# endif
401# elif SANITIZER_RISCV64
402 // the bottom half of vma is allocated for userspace
403 vmaSize = vmaSize + 1;
404# if !SANITIZER_GO
405 if (vmaSize != 39 && vmaSize != 48) {
406 Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
407 Printf("FATAL: Found %zd - Supported 39 and 48\n", vmaSize);
408 Die();
409 }
410# else
411 if (vmaSize != 39 && vmaSize != 48) {
412 Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
413 Printf("FATAL: Found %zd - Supported 39 and 48\n", vmaSize);
414 Die();
415 }
416# endif
417# endif
418
419# if !SANITIZER_GO
420 // Heap has not been allocated yet
421 ReExecIfNeeded(ignore_heap: false);
422# endif
423}
424
425void InitializePlatform() {
426 DisableCoreDumperIfNecessary();
427
428 // Go maps shadow memory lazily and works fine with limited address space.
429 // Unlimited stack is not a problem as well, because the executable
430 // is not compiled with -pie.
431#if !SANITIZER_GO
432 // We called ReExecIfNeeded() in InitializePlatformEarly(), but there are
433 // intervening allocations that result in an edge case:
434 // 1) InitializePlatformEarly(): memory layout is compatible
435 // 2) Intervening allocations happen
436 // 3) InitializePlatform(): memory layout is incompatible and fails
437 // CheckAndProtect()
438# if !SANITIZER_GO
439 // Heap has already been allocated
440 ReExecIfNeeded(ignore_heap: true);
441# endif
442
443 // Earlier initialization steps already re-exec'ed until we got a compatible
444 // memory layout, so we don't expect any more issues here.
445 if (!CheckAndProtect(protect: true, ignore_heap: true, print_warnings: true)) {
446 Printf(
447 format: "FATAL: ThreadSanitizer: unexpectedly found incompatible memory "
448 "layout.\n");
449 Printf(format: "FATAL: Please file a bug.\n");
450 DumpProcessMap();
451 Die();
452 }
453
454#endif // !SANITIZER_GO
455}
456
457#if !SANITIZER_GO
458// Extract file descriptors passed to glibc internal __res_iclose function.
459// This is required to properly "close" the fds, because we do not see internal
460// closes within glibc. The code is a pure hack.
461int ExtractResolvFDs(void *state, int *fds, int nfd) {
462#if SANITIZER_LINUX && !SANITIZER_ANDROID
463 int cnt = 0;
464 struct __res_state *statp = (struct __res_state*)state;
465 for (int i = 0; i < MAXNS && cnt < nfd; i++) {
466 if (statp->_u._ext.nsaddrs[i] && statp->_u._ext.nssocks[i] != -1)
467 fds[cnt++] = statp->_u._ext.nssocks[i];
468 }
469 return cnt;
470#else
471 return 0;
472#endif
473}
474
475// Extract file descriptors passed via UNIX domain sockets.
476// This is required to properly handle "open" of these fds.
477// see 'man recvmsg' and 'man 3 cmsg'.
478int ExtractRecvmsgFDs(void *msgp, int *fds, int nfd) {
479 int res = 0;
480 msghdr *msg = (msghdr*)msgp;
481 struct cmsghdr *cmsg = CMSG_FIRSTHDR(msg);
482 for (; cmsg; cmsg = CMSG_NXTHDR(msg, cmsg)) {
483 if (cmsg->cmsg_level != SOL_SOCKET || cmsg->cmsg_type != SCM_RIGHTS)
484 continue;
485 int n = (cmsg->cmsg_len - CMSG_LEN(0)) / sizeof(fds[0]);
486 for (int i = 0; i < n; i++) {
487 fds[res++] = ((int*)CMSG_DATA(cmsg))[i];
488 if (res == nfd)
489 return res;
490 }
491 }
492 return res;
493}
494
495// Reverse operation of libc stack pointer mangling
496static uptr UnmangleLongJmpSp(uptr mangled_sp) {
497# if CALIBRATE_LONGJMP_SP
498 // Calibrate on first longjmp/siglongjmp: most programs never call longjmp,
499 // and bionic initializes setjmp_cookie after TSan initialization
500 // (__libc_init_globals -> __libc_init_vdso -> strcmp -> __tsan_init ->
501 // __libc_init_setjmp_cookie).
502 if (UNLIKELY(!atomic_load(&longjmp_sp_inited, memory_order_acquire)))
503 InitializeLongjmpSpMangling();
504 uptr xor_key = atomic_load(a: &longjmp_xor_key, mo: memory_order_relaxed);
505 uptr rot = atomic_load(a: &longjmp_rot, mo: memory_order_relaxed);
506 uptr sp = mangled_sp;
507 if (rot)
508 sp = (sp >> rot) | (sp << (sizeof(uptr) * 8 - rot));
509 return sp ^ xor_key;
510# else
511 // The BSDs do not mangle the stack pointer in jmp_buf.
512 return mangled_sp;
513# endif
514}
515
516#if SANITIZER_NETBSD
517# ifdef __x86_64__
518# define LONG_JMP_SP_ENV_SLOT 6
519# else
520# error unsupported
521# endif
522#elif defined(__powerpc__)
523# define LONG_JMP_SP_ENV_SLOT 0
524#elif SANITIZER_FREEBSD
525# ifdef __aarch64__
526# define LONG_JMP_SP_ENV_SLOT 1
527# else
528# define LONG_JMP_SP_ENV_SLOT 2
529# endif
530# elif SANITIZER_ANDROID
531# ifdef __aarch64__
532# define LONG_JMP_SP_ENV_SLOT 3
533# elif SANITIZER_RISCV64
534# define LONG_JMP_SP_ENV_SLOT 3
535# elif defined(__x86_64__)
536# define LONG_JMP_SP_ENV_SLOT 6
537# else
538# error unsupported
539# endif
540# elif SANITIZER_LINUX
541# ifdef __aarch64__
542# define LONG_JMP_SP_ENV_SLOT 13
543# elif defined(__loongarch__)
544# define LONG_JMP_SP_ENV_SLOT 1
545# elif defined(__mips64)
546# define LONG_JMP_SP_ENV_SLOT 1
547# elif SANITIZER_RISCV64
548# define LONG_JMP_SP_ENV_SLOT 13
549# elif defined(__s390x__)
550# define LONG_JMP_SP_ENV_SLOT 9
551# else
552# define LONG_JMP_SP_ENV_SLOT 6
553# endif
554# endif
555
556uptr ExtractLongJmpSp(uptr *env) {
557 uptr mangled_sp = env[LONG_JMP_SP_ENV_SLOT];
558 return UnmangleLongJmpSp(mangled_sp);
559}
560
561# if CALIBRATE_LONGJMP_SP
562// Read the SP immediately after _setjmp returns. This is the value libc saves.
563NOINLINE static void ProbeLongJmpSp(uptr* sp, uptr* mangled_sp) {
564 jmp_buf env;
565 REAL(_setjmp)(env);
566 uptr v;
567# if defined(__x86_64__)
568 asm volatile("mov %%rsp, %0" : "=r"(v));
569# elif defined(__aarch64__)
570 asm volatile("mov %0, sp" : "=r"(v));
571# elif defined(__loongarch__)
572 asm volatile("move %0, $sp" : "=r"(v));
573# elif defined(__powerpc64__)
574 asm volatile("mr %0, 1" : "=r"(v));
575# elif defined(__s390x__)
576 asm volatile("lgr %0, %%r15" : "=r"(v));
577# elif defined(__mips__)
578 asm volatile("move %0, $sp" : "=r"(v));
579# elif SANITIZER_RISCV64
580 asm volatile("mv %0, sp" : "=r"(v));
581# else
582# error "Unknown platform"
583# endif
584 *sp = v;
585 *mangled_sp = ((uptr*)&env)[LONG_JMP_SP_ENV_SLOT];
586}
587
588// Run the probe in another frame. Consuming its result after the call prevents
589// tail-call elimination.
590NOINLINE static void ProbeLongJmpSpDeep(uptr* sp, uptr* mangled_sp) {
591 ProbeLongJmpSp(sp, mangled_sp);
592 asm volatile("" : : "r"(*sp));
593}
594
595// Probe two stack depths to identify identity, XOR, or XOR-and-rotate without
596// depending on private guard locations. Lock-free and reentrant so concurrent
597// threads or signal handlers calling siglongjmp do not block or deadlock.
598static void InitializeLongjmpSpMangling() {
599 uptr sp1, m1, sp2, m2;
600 ProbeLongJmpSp(sp: &sp1, mangled_sp: &m1);
601 ProbeLongJmpSpDeep(sp: &sp2, mangled_sp: &m2);
602 // The rotation is coprime to the word size, so only zero and all-ones
603 // deltas are rotation-invariant.
604 const uptr d = sp1 ^ sp2;
605 CHECK_NE(d, 0);
606 CHECK_NE(d, (uptr)-1);
607 constexpr uptr kRot = 2 * sizeof(uptr) + 1;
608 constexpr uptr kBits = sizeof(uptr) * 8;
609 const uptr r1 = (m1 >> kRot) | (m1 << (kBits - kRot));
610 const uptr r2 = (m2 >> kRot) | (m2 << (kBits - kRot));
611 uptr rot;
612 uptr xor_key;
613 if ((m1 ^ sp1) == (m2 ^ sp2)) {
614 // This includes the identity scheme when the key is zero.
615 rot = 0;
616 xor_key = m1 ^ sp1;
617 } else if ((r1 ^ sp1) == (r2 ^ sp2)) {
618 rot = kRot;
619 xor_key = r1 ^ sp1;
620 } else {
621 Printf(
622 format: "FATAL: ThreadSanitizer: unrecognized jmp_buf pointer mangling "
623 "scheme\n");
624 Die();
625 }
626 if (SANITIZER_ANDROID)
627 CHECK_NE(xor_key, 0);
628 atomic_store(a: &longjmp_rot, v: rot, mo: memory_order_relaxed);
629 atomic_store(a: &longjmp_xor_key, v: xor_key, mo: memory_order_relaxed);
630 atomic_store(a: &longjmp_sp_inited, v: 1, mo: memory_order_release);
631}
632# endif
633
634extern "C" void __tsan_tls_initialization() {}
635
636void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size) {
637 // Check that the thr object is in tls;
638 const uptr thr_beg = (uptr)thr;
639 const uptr thr_end = (uptr)thr + sizeof(*thr);
640 CHECK_GE(thr_beg, tls_addr);
641 CHECK_LE(thr_beg, tls_addr + tls_size);
642 CHECK_GE(thr_end, tls_addr);
643 CHECK_LE(thr_end, tls_addr + tls_size);
644 // Since the thr object is huge, skip it.
645 const uptr pc = StackTrace::GetNextInstructionPc(
646 pc: reinterpret_cast<uptr>(__tsan_tls_initialization));
647 MemoryRangeImitateWrite(thr, pc, addr: tls_addr, size: thr_beg - tls_addr);
648 MemoryRangeImitateWrite(thr, pc, addr: thr_end, size: tls_addr + tls_size - thr_end);
649}
650
651// Note: this function runs with async signals enabled,
652// so it must not touch any tsan state.
653int call_pthread_cancel_with_cleanup(int (*fn)(void *arg),
654 void (*cleanup)(void *arg), void *arg) {
655 // pthread_cleanup_push/pop are hardcore macros mess.
656 // We can't intercept nor call them w/o including pthread.h.
657 int res;
658 pthread_cleanup_push(cleanup, arg);
659 res = fn(arg);
660 pthread_cleanup_pop(0);
661 return res;
662}
663#endif // !SANITIZER_GO
664
665#if !SANITIZER_GO
666void ReplaceSystemMalloc() { }
667#endif
668
669#if !SANITIZER_GO
670#if SANITIZER_ANDROID
671// On Android, one thread can call intercepted functions after
672// DestroyThreadState(), so add a fake thread state for "dead" threads.
673static ThreadState *dead_thread_state = nullptr;
674
675ThreadState *cur_thread() {
676 ThreadState* thr = reinterpret_cast<ThreadState*>(*get_android_tls_ptr());
677 if (thr == nullptr) {
678 __sanitizer_sigset_t emptyset;
679 internal_sigfillset(&emptyset);
680 __sanitizer_sigset_t oldset;
681 CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &emptyset, &oldset));
682 thr = reinterpret_cast<ThreadState*>(*get_android_tls_ptr());
683 if (thr == nullptr) {
684 thr = reinterpret_cast<ThreadState*>(MmapOrDie(sizeof(ThreadState),
685 "ThreadState"));
686 *get_android_tls_ptr() = reinterpret_cast<uptr>(thr);
687 if (dead_thread_state == nullptr) {
688 dead_thread_state = reinterpret_cast<ThreadState*>(
689 MmapOrDie(sizeof(ThreadState), "ThreadState"));
690 dead_thread_state->fast_state.SetIgnoreBit();
691 dead_thread_state->ignore_interceptors = 1;
692 dead_thread_state->is_dead = true;
693 *const_cast<u32*>(&dead_thread_state->tid) = -1;
694 CHECK_EQ(0, internal_mprotect(dead_thread_state, sizeof(ThreadState),
695 PROT_READ));
696 }
697 }
698 CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &oldset, nullptr));
699 }
700
701 // Skia calls mallopt(M_THREAD_DISABLE_MEM_INIT, 1), which sets the least
702 // significant bit of TLS_SLOT_SANITIZER to 1. Scudo allocator uses this bit
703 // as a flag to disable memory initialization. This is a workaround to get the
704 // correct ThreadState pointer.
705 uptr addr = reinterpret_cast<uptr>(thr);
706 return reinterpret_cast<ThreadState*>(addr & ~1ULL);
707}
708
709void set_cur_thread(ThreadState *thr) {
710 *get_android_tls_ptr() = reinterpret_cast<uptr>(thr);
711}
712
713void cur_thread_finalize() {
714 __sanitizer_sigset_t emptyset;
715 internal_sigfillset(&emptyset);
716 __sanitizer_sigset_t oldset;
717 CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &emptyset, &oldset));
718 ThreadState* thr = reinterpret_cast<ThreadState*>(*get_android_tls_ptr());
719 if (thr != dead_thread_state) {
720 *get_android_tls_ptr() = reinterpret_cast<uptr>(dead_thread_state);
721 UnmapOrDie(thr, sizeof(ThreadState));
722 }
723 CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &oldset, nullptr));
724}
725#endif // SANITIZER_ANDROID
726#endif // if !SANITIZER_GO
727
728} // namespace __tsan
729
730#endif // SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD
731