1//===-- sanitizer_linux_libcdep.cpp ---------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is shared between AddressSanitizer and ThreadSanitizer
10// run-time libraries and implements linux-specific functions from
11// sanitizer_libc.h.
12//===----------------------------------------------------------------------===//
13
14#include "sanitizer_platform.h"
15
16#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \
17 SANITIZER_SOLARIS || SANITIZER_HAIKU
18
19# include "sanitizer_allocator_internal.h"
20# include "sanitizer_atomic.h"
21# include "sanitizer_common.h"
22# include "sanitizer_file.h"
23# include "sanitizer_flags.h"
24# include "sanitizer_getauxval.h"
25# include "sanitizer_glibc_version.h"
26# include "sanitizer_linux.h"
27# include "sanitizer_placement_new.h"
28# include "sanitizer_procmaps.h"
29# include "sanitizer_solaris.h"
30
31# if SANITIZER_HAIKU
32# define _GNU_SOURCE
33# define _DEFAULT_SOURCE
34# endif
35
36# if SANITIZER_NETBSD
37# // for __lwp_gettcb_fast() / __lwp_getprivate_fast()
38# define _RTLD_SOURCE
39# include <machine/mcontext.h>
40# undef _RTLD_SOURCE
41# include <sys/param.h>
42# if __NetBSD_Version__ >= 1099001200
43# include <machine/lwp_private.h>
44# endif
45# endif
46
47# include <dlfcn.h> // for dlsym()
48# include <link.h>
49# include <pthread.h>
50# include <signal.h>
51# include <sys/mman.h>
52# include <sys/resource.h>
53# include <syslog.h>
54
55# if SANITIZER_GLIBC
56# include <gnu/libc-version.h>
57# endif
58
59# if !defined(ElfW)
60# define ElfW(type) Elf_##type
61# endif
62
63# if SANITIZER_FREEBSD
64# include <pthread_np.h>
65# include <sys/auxv.h>
66# include <sys/sysctl.h>
67# define pthread_getattr_np pthread_attr_get_np
68// The MAP_NORESERVE define has been removed in FreeBSD 11.x, and even before
69// that, it was never implemented. So just define it to zero.
70# undef MAP_NORESERVE
71# define MAP_NORESERVE 0
72extern const Elf_Auxinfo *__elf_aux_vector __attribute__((weak));
73extern "C" int __sys_sigaction(int signum, const struct sigaction *act,
74 struct sigaction *oldact);
75# endif
76
77# if SANITIZER_NETBSD
78# include <lwp.h>
79# include <sys/sysctl.h>
80# include <sys/tls.h>
81# endif
82
83# if SANITIZER_SOLARIS
84# include <stddef.h>
85# include <stdlib.h>
86# include <thread.h>
87# endif
88
89# if SANITIZER_HAIKU
90# include <kernel/OS.h>
91# include <sys/link_elf.h>
92# endif
93
94# if !SANITIZER_ANDROID
95# include <elf.h>
96# include <unistd.h>
97# endif
98
99namespace __sanitizer {
100
101SANITIZER_WEAK_ATTRIBUTE int real_sigaction(int signum, const void *act,
102 void *oldact);
103
104int internal_sigaction(int signum, const void *act, void *oldact) {
105# if SANITIZER_FREEBSD
106 // On FreeBSD, call the sigaction syscall directly (part of libsys in FreeBSD
107 // 15) since the libc version goes via a global interposing table. Due to
108 // library initialization order the table can be relocated after the call to
109 // InitializeDeadlySignals() which then crashes when dereferencing the
110 // uninitialized pointer in libc.
111 return __sys_sigaction(signum, (const struct sigaction *)act,
112 (struct sigaction *)oldact);
113# else
114# if !SANITIZER_GO
115 if (&real_sigaction)
116 return real_sigaction(signum, act, oldact);
117# endif
118 return sigaction(sig: signum, act: (const struct sigaction *)act,
119 oact: (struct sigaction *)oldact);
120# endif
121}
122
123void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top,
124 uptr *stack_bottom) {
125 CHECK(stack_top);
126 CHECK(stack_bottom);
127 if (at_initialization) {
128 // This is the main thread. Libpthread may not be initialized yet.
129 struct rlimit rl;
130 CHECK_EQ(getrlimit(RLIMIT_STACK, &rl), 0);
131
132 // Find the mapping that contains a stack variable.
133 MemoryMappingLayout proc_maps(/*cache_enabled*/ true);
134 if (proc_maps.Error()) {
135 *stack_top = *stack_bottom = 0;
136 return;
137 }
138 MemoryMappedSegment segment;
139 uptr prev_end = 0;
140 while (proc_maps.Next(segment: &segment)) {
141 if ((uptr)&rl < segment.end)
142 break;
143 prev_end = segment.end;
144 }
145 CHECK((uptr)&rl >= segment.start && (uptr)&rl < segment.end);
146
147 // Get stacksize from rlimit, but clip it so that it does not overlap
148 // with other mappings.
149 uptr stacksize = rl.rlim_cur;
150# if SANITIZER_HPPA
151 // The stack grows up from the start of the mapping, so clip against the
152 // next mapping instead.
153 (void)prev_end;
154 uptr next_start = GetMaxUserVirtualAddress();
155 MemoryMappedSegment next_segment;
156 if (proc_maps.Next(&next_segment))
157 next_start = next_segment.start;
158 if (stacksize > next_start - segment.start)
159 stacksize = next_start - segment.start;
160# else
161 if (stacksize > segment.end - prev_end)
162 stacksize = segment.end - prev_end;
163# endif
164 // When running with unlimited stack size, we still want to set some limit.
165 // The unlimited stack size is caused by 'ulimit -s unlimited'.
166 // Also, for some reason, GNU make spawns subprocesses with unlimited stack.
167 if (stacksize > kMaxThreadStackSize)
168 stacksize = kMaxThreadStackSize;
169# if SANITIZER_HPPA
170 *stack_bottom = segment.start;
171 *stack_top = segment.start + stacksize;
172# else
173 *stack_top = segment.end;
174 *stack_bottom = segment.end - stacksize;
175# endif
176
177 uptr maxAddr = GetMaxUserVirtualAddress();
178 // Edge case: the stack mapping on some systems may be off-by-one e.g.,
179 // fffffffdf000-1000000000000 rw-p 00000000 00:00 0 [stack]
180 // instead of:
181 // fffffffdf000- ffffffffffff
182 // The out-of-range stack_top can result in an invalid shadow address
183 // calculation, since those usually assume the parameters are in range.
184 if (*stack_top == maxAddr + 1)
185 *stack_top = maxAddr;
186 else
187 CHECK_LE(*stack_top, maxAddr);
188
189 return;
190 }
191 uptr stacksize = 0;
192 void *stackaddr = nullptr;
193# if SANITIZER_SOLARIS
194 stack_t ss;
195 CHECK_EQ(thr_stksegment(&ss), 0);
196 stacksize = ss.ss_size;
197 stackaddr = (char *)ss.ss_sp - stacksize;
198# else // !SANITIZER_SOLARIS
199 pthread_attr_t attr;
200 pthread_attr_init(attr: &attr);
201 CHECK_EQ(pthread_getattr_np(pthread_self(), &attr), 0);
202 internal_pthread_attr_getstack(attr: &attr, addr: &stackaddr, size: &stacksize);
203 pthread_attr_destroy(attr: &attr);
204# endif // SANITIZER_SOLARIS
205
206 *stack_top = (uptr)stackaddr + stacksize;
207 *stack_bottom = (uptr)stackaddr;
208}
209
210# if !SANITIZER_GO
211bool SetEnv(const char *name, const char *value) {
212 void *f = dlsym(RTLD_NEXT, name: "setenv");
213 if (!f)
214 return false;
215 typedef int (*setenv_ft)(const char *name, const char *value, int overwrite);
216 setenv_ft setenv_f;
217 CHECK_EQ(sizeof(setenv_f), sizeof(f));
218 internal_memcpy(dest: &setenv_f, src: &f, n: sizeof(f));
219 return setenv_f(name, value, 1) == 0;
220}
221# endif
222
223// True if we can use dlpi_tls_data. glibc before 2.25 may leave NULL (BZ
224// #19826) so dlpi_tls_data cannot be used.
225//
226// musl before 1.2.3 and FreeBSD as of 12.2 incorrectly set dlpi_tls_data to
227// the TLS initialization image
228// https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=254774
229__attribute__((unused)) static int g_use_dlpi_tls_data;
230
231# if SANITIZER_GLIBC && !SANITIZER_GO
232static void GetGLibcVersion(int *major, int *minor, int *patch) {
233 const char *p = gnu_get_libc_version();
234 *major = internal_simple_strtoll(nptr: p, endptr: &p, base: 10);
235 // Caller does not expect anything else.
236 CHECK_EQ(*major, 2);
237 *minor = (*p == '.') ? internal_simple_strtoll(nptr: p + 1, endptr: &p, base: 10) : 0;
238 *patch = (*p == '.') ? internal_simple_strtoll(nptr: p + 1, endptr: &p, base: 10) : 0;
239}
240
241static uptr ThreadDescriptorSizeFallback() {
242# if defined(__x86_64__) || defined(__i386__) || defined(__arm__) || \
243 SANITIZER_RISCV64
244 int major;
245 int minor;
246 int patch;
247 GetGLibcVersion(major: &major, minor: &minor, patch: &patch);
248# endif
249
250# if defined(__x86_64__) || defined(__i386__) || defined(__arm__)
251 /* sizeof(struct pthread) values from various glibc versions. */
252 if (SANITIZER_X32)
253 return 1728; // Assume only one particular version for x32.
254 // For ARM sizeof(struct pthread) changed in Glibc 2.23.
255 if (SANITIZER_ARM)
256 return minor <= 22 ? 1120 : 1216;
257 if (minor <= 3)
258 return FIRST_32_SECOND_64(1104, 1696);
259 if (minor == 4)
260 return FIRST_32_SECOND_64(1120, 1728);
261 if (minor == 5)
262 return FIRST_32_SECOND_64(1136, 1728);
263 if (minor <= 9)
264 return FIRST_32_SECOND_64(1136, 1712);
265 if (minor == 10)
266 return FIRST_32_SECOND_64(1168, 1776);
267 if (minor == 11 || (minor == 12 && patch == 1))
268 return FIRST_32_SECOND_64(1168, 2288);
269 if (minor <= 14)
270 return FIRST_32_SECOND_64(1168, 2304);
271 if (minor < 32) // Unknown version
272 return FIRST_32_SECOND_64(1216, 2304);
273 // minor == 32
274 return FIRST_32_SECOND_64(1344, 2496);
275# endif
276
277# if SANITIZER_RISCV64
278 // TODO: consider adding an optional runtime check for an unknown (untested)
279 // glibc version
280 if (minor <= 28) // WARNING: the highest tested version is 2.29
281 return 1772; // no guarantees for this one
282 if (minor <= 31)
283 return 1772; // tested against glibc 2.29, 2.31
284 return 1936; // tested against glibc 2.32
285# endif
286
287# if defined(__s390__) || defined(__sparc__)
288 // The size of a prefix of TCB including pthread::{specific_1stblock,specific}
289 // suffices. Just return offsetof(struct pthread, specific_used), which hasn't
290 // changed since 2007-05. Technically this applies to i386/x86_64 as well but
291 // we call _dl_get_tls_static_info and need the precise size of struct
292 // pthread.
293 return FIRST_32_SECOND_64(524, 1552);
294# endif
295
296# if defined(__mips__)
297 // TODO(sagarthakur): add more values as per different glibc versions.
298 return FIRST_32_SECOND_64(1152, 1776);
299# endif
300
301# if SANITIZER_LOONGARCH64
302 return 1856; // from glibc 2.36
303# endif
304
305# if defined(__aarch64__)
306 // The sizeof (struct pthread) is the same from GLIBC 2.17 to 2.22.
307 return 1776;
308# endif
309
310# if defined(__powerpc64__)
311 return 1776; // from glibc.ppc64le 2.20-8.fc21
312# endif
313
314# if defined(__alpha__)
315 return 1824; // from glibc 2.43
316# endif
317
318# if defined(__hppa__)
319 return 1216; // from glibc 2.43
320# endif
321}
322# endif // SANITIZER_GLIBC && !SANITIZER_GO
323
324# if SANITIZER_FREEBSD && !SANITIZER_GO
325// FIXME: Implementation is very GLIBC specific, but it's used by FreeBSD.
326static uptr ThreadDescriptorSizeFallback() {
327# if defined(__s390__) || defined(__sparc__)
328 // The size of a prefix of TCB including pthread::{specific_1stblock,specific}
329 // suffices. Just return offsetof(struct pthread, specific_used), which hasn't
330 // changed since 2007-05. Technically this applies to i386/x86_64 as well but
331 // we call _dl_get_tls_static_info and need the precise size of struct
332 // pthread.
333 return FIRST_32_SECOND_64(524, 1552);
334# endif
335
336# if defined(__mips__)
337 // TODO(sagarthakur): add more values as per different glibc versions.
338 return FIRST_32_SECOND_64(1152, 1776);
339# endif
340
341# if SANITIZER_LOONGARCH64
342 return 1856; // from glibc 2.36
343# endif
344
345# if defined(__aarch64__)
346 // The sizeof (struct pthread) is the same from GLIBC 2.17 to 2.22.
347 return 1776;
348# endif
349
350# if defined(__powerpc64__)
351 return 1776; // from glibc.ppc64le 2.20-8.fc21
352# endif
353
354 return 0;
355}
356# endif // SANITIZER_FREEBSD && !SANITIZER_GO
357
358# if (SANITIZER_FREEBSD || SANITIZER_GLIBC) && !SANITIZER_GO
359// On glibc x86_64, ThreadDescriptorSize() needs to be precise due to the usage
360// of g_tls_size. On other targets, ThreadDescriptorSize() is only used by lsan
361// to get the pointer to thread-specific data keys in the thread control block.
362// sizeof(struct pthread) from glibc.
363static uptr thread_descriptor_size;
364
365uptr ThreadDescriptorSize() { return thread_descriptor_size; }
366
367# if SANITIZER_GLIBC
368__attribute__((unused)) static size_t g_tls_size;
369# endif
370
371void InitTlsSize() {
372# if SANITIZER_GLIBC
373 int major, minor, patch;
374 GetGLibcVersion(major: &major, minor: &minor, patch: &patch);
375 g_use_dlpi_tls_data = major == 2 && minor >= 25;
376
377 if (major == 2 && minor >= 34) {
378 // _thread_db_sizeof_pthread is a GLIBC_PRIVATE symbol that is exported in
379 // glibc 2.34 and later.
380 if (unsigned *psizeof = static_cast<unsigned *>(
381 dlsym(RTLD_DEFAULT, name: "_thread_db_sizeof_pthread"))) {
382 thread_descriptor_size = *psizeof;
383 }
384 }
385
386# if defined(__aarch64__) || defined(__x86_64__) || \
387 defined(__powerpc64__) || defined(__loongarch__)
388 auto *get_tls_static_info = (void (*)(size_t *, size_t *))dlsym(
389 RTLD_DEFAULT, name: "_dl_get_tls_static_info");
390 size_t tls_align;
391 // Can be null if static link.
392 if (get_tls_static_info)
393 get_tls_static_info(&g_tls_size, &tls_align);
394# endif
395
396# endif // SANITIZER_GLIBC
397
398 if (!thread_descriptor_size)
399 thread_descriptor_size = ThreadDescriptorSizeFallback();
400}
401
402# if defined(__mips__) || defined(__powerpc64__) || SANITIZER_RISCV64 || \
403 SANITIZER_LOONGARCH64
404// TlsPreTcbSize includes size of struct pthread_descr and size of tcb
405// head structure. It lies before the static tls blocks.
406static uptr TlsPreTcbSize() {
407# if defined(__mips__)
408 const uptr kTcbHead = 16; // sizeof (tcbhead_t)
409# elif defined(__powerpc64__)
410 const uptr kTcbHead = 88; // sizeof (tcbhead_t)
411# elif SANITIZER_RISCV64
412 const uptr kTcbHead = 16; // sizeof (tcbhead_t)
413# elif SANITIZER_LOONGARCH64
414 const uptr kTcbHead = 16; // sizeof (tcbhead_t)
415# endif
416 const uptr kTlsAlign = 16;
417 const uptr kTlsPreTcbSize =
418 RoundUpTo(ThreadDescriptorSize() + kTcbHead, kTlsAlign);
419 return kTlsPreTcbSize;
420}
421# endif
422# else // (SANITIZER_FREEBSD || SANITIZER_GLIBC) && !SANITIZER_GO
423void InitTlsSize() {}
424uptr ThreadDescriptorSize() { return 0; }
425# endif // (SANITIZER_FREEBSD || SANITIZER_GLIBC) && !SANITIZER_GO
426
427# if (SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_SOLARIS) && \
428 !SANITIZER_ANDROID && !SANITIZER_GO
429namespace {
430struct TlsBlock {
431 uptr begin, end, align;
432 size_t tls_modid;
433 bool operator<(const TlsBlock &rhs) const { return begin < rhs.begin; }
434};
435} // namespace
436
437# ifdef __s390__
438extern "C" uptr __tls_get_offset(void *arg);
439
440static uptr TlsGetOffset(uptr ti_module, uptr ti_offset) {
441 // The __tls_get_offset ABI requires %r12 to point to GOT and %r2 to be an
442 // offset of a struct tls_index inside GOT. We don't possess either of the
443 // two, so violate the letter of the "ELF Handling For Thread-Local
444 // Storage" document and assume that the implementation just dereferences
445 // %r2 + %r12.
446 uptr tls_index[2] = {ti_module, ti_offset};
447 register uptr r2 asm("2") = 0;
448 register void *r12 asm("12") = tls_index;
449 asm("basr %%r14, %[__tls_get_offset]"
450 : "+r"(r2)
451 : [__tls_get_offset] "r"(__tls_get_offset), "r"(r12)
452 : "memory", "cc", "0", "1", "3", "4", "5", "14");
453 return r2;
454}
455# else
456extern "C" void *__tls_get_addr(size_t *);
457# endif
458
459static size_t main_tls_modid;
460
461static int CollectStaticTlsBlocks(struct dl_phdr_info *info, size_t size,
462 void *data) {
463 size_t tls_modid;
464# if SANITIZER_SOLARIS
465 // dlpi_tls_modid is only available since Solaris 11.4 SRU 10. Use
466 // dlinfo(RTLD_DI_LINKMAP) instead which works on all of Solaris 11.3,
467 // 11.4, and Illumos. The tlsmodid of the executable was changed to 1 in
468 // 11.4 to match other implementations.
469 if (size >= offsetof(dl_phdr_info_test, dlpi_tls_modid))
470 main_tls_modid = 1;
471 else
472 main_tls_modid = 0;
473 g_use_dlpi_tls_data = 0;
474 Rt_map *map;
475 dlinfo(RTLD_SELF, RTLD_DI_LINKMAP, &map);
476 tls_modid = map->rt_tlsmodid;
477# else
478 main_tls_modid = 1;
479 tls_modid = info->dlpi_tls_modid;
480# endif
481
482 if (tls_modid < main_tls_modid)
483 return 0;
484 uptr begin;
485# if !SANITIZER_SOLARIS
486 begin = (uptr)info->dlpi_tls_data;
487# endif
488 if (!g_use_dlpi_tls_data) {
489 // Call __tls_get_addr as a fallback. This forces TLS allocation on glibc
490 // and FreeBSD.
491# ifdef __s390__
492 begin = (uptr)__builtin_thread_pointer() + TlsGetOffset(tls_modid, 0);
493# else
494 size_t mod_and_off[2] = {tls_modid, 0};
495 begin = (uptr)__tls_get_addr(mod_and_off);
496# endif
497 }
498 for (unsigned i = 0; i != info->dlpi_phnum; ++i)
499 if (info->dlpi_phdr[i].p_type == PT_TLS) {
500 static_cast<InternalMmapVector<TlsBlock> *>(data)->push_back(
501 element: TlsBlock{.begin: begin, .end: begin + info->dlpi_phdr[i].p_memsz,
502 .align: info->dlpi_phdr[i].p_align, .tls_modid: tls_modid});
503 break;
504 }
505 return 0;
506}
507
508__attribute__((unused)) static void GetStaticTlsBoundary(uptr *addr, uptr *size,
509 uptr *align) {
510 InternalMmapVector<TlsBlock> ranges;
511 dl_iterate_phdr(callback: CollectStaticTlsBlocks, data: &ranges);
512 uptr len = ranges.size();
513 Sort(v: ranges.begin(), size: len);
514 // Find the range with tls_modid == main_tls_modid. For glibc, because
515 // libc.so uses PT_TLS, this module is guaranteed to exist and is one of
516 // the initially loaded modules.
517 uptr one = 0;
518 while (one != len && ranges[one].tls_modid != main_tls_modid) ++one;
519 if (one == len) {
520 // This may happen with musl if no module uses PT_TLS.
521 *addr = 0;
522 *size = 0;
523 *align = 1;
524 return;
525 }
526 // Find the maximum consecutive ranges. We consider two modules consecutive if
527 // the gap is smaller than the alignment of the latter range. The dynamic
528 // loader places static TLS blocks this way not to waste space.
529 uptr l = one;
530 *align = ranges[l].align;
531 while (l != 0 && ranges[l].begin <= ranges[l - 1].end + ranges[l].align)
532 *align = Max(a: *align, b: ranges[--l].align);
533 uptr r = one + 1;
534 while (r != len && ranges[r].begin <= ranges[r - 1].end + ranges[r].align)
535 *align = Max(a: *align, b: ranges[r++].align);
536 *addr = ranges[l].begin;
537 *size = ranges[r - 1].end - ranges[l].begin;
538}
539# endif // (x86_64 || i386 || mips || ...) && (SANITIZER_FREEBSD ||
540 // SANITIZER_LINUX) && !SANITIZER_ANDROID && !SANITIZER_GO
541
542# if SANITIZER_NETBSD
543static struct tls_tcb *ThreadSelfTlsTcb() {
544 struct tls_tcb *tcb = nullptr;
545# ifdef __HAVE___LWP_GETTCB_FAST
546 tcb = (struct tls_tcb *)__lwp_gettcb_fast();
547# elif defined(__HAVE___LWP_GETPRIVATE_FAST)
548 tcb = (struct tls_tcb *)__lwp_getprivate_fast();
549# endif
550 return tcb;
551}
552
553uptr ThreadSelf() { return (uptr)ThreadSelfTlsTcb()->tcb_pthread; }
554
555int GetSizeFromHdr(struct dl_phdr_info *info, size_t size, void *data) {
556 const Elf_Phdr *hdr = info->dlpi_phdr;
557 const Elf_Phdr *last_hdr = hdr + info->dlpi_phnum;
558
559 for (; hdr != last_hdr; ++hdr) {
560 if (hdr->p_type == PT_TLS && info->dlpi_tls_modid == 1) {
561 *(uptr *)data = hdr->p_memsz;
562 break;
563 }
564 }
565 return 0;
566}
567# endif // SANITIZER_NETBSD
568
569# if SANITIZER_ANDROID
570// Bionic provides this API since S.
571extern "C" SANITIZER_WEAK_ATTRIBUTE void __libc_get_static_tls_bounds(void **,
572 void **);
573# endif
574
575# if !SANITIZER_GO
576static void GetTls(uptr *addr, uptr *size) {
577# if SANITIZER_ANDROID
578 if (&__libc_get_static_tls_bounds) {
579 void *start_addr;
580 void *end_addr;
581 __libc_get_static_tls_bounds(&start_addr, &end_addr);
582 *addr = reinterpret_cast<uptr>(start_addr);
583 *size =
584 reinterpret_cast<uptr>(end_addr) - reinterpret_cast<uptr>(start_addr);
585 } else {
586 *addr = 0;
587 *size = 0;
588 }
589# elif SANITIZER_GLIBC && defined(__x86_64__)
590 // For aarch64 and x86-64, use an O(1) approach which requires relatively
591 // precise ThreadDescriptorSize. g_tls_size was initialized in InitTlsSize.
592# if SANITIZER_X32
593 asm("mov %%fs:8,%0" : "=r"(*addr));
594# else
595 asm("mov %%fs:16,%0" : "=r"(*addr));
596# endif
597 *size = g_tls_size;
598 *addr -= *size;
599 *addr += ThreadDescriptorSize();
600# elif SANITIZER_GLIBC && defined(__aarch64__)
601 *addr = reinterpret_cast<uptr>(__builtin_thread_pointer()) -
602 ThreadDescriptorSize();
603 *size = g_tls_size + ThreadDescriptorSize();
604# elif SANITIZER_GLIBC && defined(__loongarch__)
605# ifdef __clang__
606 *addr = reinterpret_cast<uptr>(__builtin_thread_pointer()) -
607 ThreadDescriptorSize();
608# else
609 asm("or %0,$tp,$zero" : "=r"(*addr));
610 *addr -= ThreadDescriptorSize();
611# endif
612 *size = g_tls_size + ThreadDescriptorSize();
613# elif SANITIZER_GLIBC && defined(__powerpc64__)
614 // Workaround for glibc<2.25(?). 2.27 is known to not need this.
615 uptr tp;
616 asm("addi %0,13,-0x7000" : "=r"(tp));
617 const uptr pre_tcb_size = TlsPreTcbSize();
618 *addr = tp - pre_tcb_size;
619 *size = g_tls_size + pre_tcb_size;
620# elif SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_SOLARIS
621 uptr align;
622 GetStaticTlsBoundary(addr, size, &align);
623# if defined(__x86_64__) || defined(__i386__) || defined(__s390__) || \
624 defined(__sparc__)
625 if (SANITIZER_GLIBC) {
626# if defined(__x86_64__) || defined(__i386__)
627 align = Max<uptr>(align, 64);
628# else
629 align = Max<uptr>(align, 16);
630# endif
631 }
632 const uptr tp = RoundUpTo(*addr + *size, align);
633
634 // lsan requires the range to additionally cover the static TLS surplus
635 // (elf/dl-tls.c defines 1664). Otherwise there may be false positives for
636 // allocations only referenced by tls in dynamically loaded modules.
637 if (SANITIZER_GLIBC)
638 *size += 1644;
639 else if (SANITIZER_FREEBSD)
640 *size += 128; // RTLD_STATIC_TLS_EXTRA
641
642 // Extend the range to include the thread control block. On glibc, lsan needs
643 // the range to include pthread::{specific_1stblock,specific} so that
644 // allocations only referenced by pthread_setspecific can be scanned. This may
645 // underestimate by at most TLS_TCB_ALIGN-1 bytes but it should be fine
646 // because the number of bytes after pthread::specific is larger.
647 *addr = tp - RoundUpTo(*size, align);
648 *size = tp - *addr + ThreadDescriptorSize();
649# else
650# if SANITIZER_GLIBC
651 *size += 1664;
652# elif SANITIZER_FREEBSD
653 *size += 128; // RTLD_STATIC_TLS_EXTRA
654# if defined(__mips__) || defined(__powerpc64__) || SANITIZER_RISCV64
655 const uptr pre_tcb_size = TlsPreTcbSize();
656 *addr -= pre_tcb_size;
657 *size += pre_tcb_size;
658# else
659 // arm and aarch64 reserve two words at TP, so this underestimates the range.
660 // However, this is sufficient for the purpose of finding the pointers to
661 // thread-specific data keys.
662 const uptr tcb_size = ThreadDescriptorSize();
663 *addr -= tcb_size;
664 *size += tcb_size;
665# endif
666# endif
667# endif
668# elif SANITIZER_NETBSD
669 struct tls_tcb *const tcb = ThreadSelfTlsTcb();
670 *addr = 0;
671 *size = 0;
672 if (tcb != 0) {
673 // Find size (p_memsz) of dlpi_tls_modid 1 (TLS block of the main program).
674 // ld.elf_so hardcodes the index 1.
675 dl_iterate_phdr(GetSizeFromHdr, size);
676
677 if (*size != 0) {
678 // The block has been found and tcb_dtv[1] contains the base address
679 *addr = (uptr)tcb->tcb_dtv[1];
680 }
681 }
682# elif SANITIZER_HAIKU
683# else
684# error "Unknown OS"
685# endif
686}
687# endif
688
689# if !SANITIZER_GO
690uptr GetTlsSize() {
691# if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \
692 SANITIZER_SOLARIS
693 uptr addr, size;
694 GetTls(addr: &addr, size: &size);
695 return size;
696# else
697 return 0;
698# endif
699}
700# endif
701
702void GetThreadStackAndTls(bool main, uptr *stk_begin, uptr *stk_end,
703 uptr *tls_begin, uptr *tls_end) {
704# if SANITIZER_GO
705 // Stub implementation for Go.
706 *stk_begin = 0;
707 *stk_end = 0;
708 *tls_begin = 0;
709 *tls_end = 0;
710# else
711 uptr tls_addr = 0;
712 uptr tls_size = 0;
713 GetTls(addr: &tls_addr, size: &tls_size);
714 *tls_begin = tls_addr;
715 *tls_end = tls_addr + tls_size;
716
717 uptr stack_top, stack_bottom;
718 GetThreadStackTopAndBottom(at_initialization: main, stack_top: &stack_top, stack_bottom: &stack_bottom);
719 *stk_begin = stack_bottom;
720 *stk_end = stack_top;
721
722 if (!main) {
723 // If stack and tls intersect, make them non-intersecting.
724 if (*tls_begin > *stk_begin && *tls_begin < *stk_end) {
725 if (*stk_end < *tls_end)
726 *tls_end = *stk_end;
727 *stk_end = *tls_begin;
728 }
729 }
730# endif
731}
732
733# if !SANITIZER_FREEBSD
734typedef ElfW(Phdr) Elf_Phdr;
735# endif
736
737struct DlIteratePhdrData {
738 InternalMmapVectorNoCtor<LoadedModule> *modules;
739 bool first;
740};
741
742static int AddModuleSegments(const char *module_name, dl_phdr_info *info,
743 InternalMmapVectorNoCtor<LoadedModule> *modules) {
744 if (module_name[0] == '\0')
745 return 0;
746 LoadedModule cur_module;
747 cur_module.set(module_name, base_address: info->dlpi_addr);
748 for (int i = 0; i < (int)info->dlpi_phnum; i++) {
749 const Elf_Phdr *phdr = &info->dlpi_phdr[i];
750 if (phdr->p_type == PT_LOAD) {
751 uptr cur_beg = info->dlpi_addr + phdr->p_vaddr;
752 uptr cur_end = cur_beg + phdr->p_memsz;
753# if SANITIZER_HAIKU
754 bool executable = phdr->p_flags & PF_EXECUTE;
755 bool writable = phdr->p_flags & PF_WRITE;
756# else
757 bool executable = phdr->p_flags & PF_X;
758 bool writable = phdr->p_flags & PF_W;
759# endif
760 cur_module.addAddressRange(beg: cur_beg, end: cur_end, executable, writable);
761 } else if (phdr->p_type == PT_NOTE) {
762# ifdef NT_GNU_BUILD_ID
763 uptr off = 0;
764 while (off + sizeof(ElfW(Nhdr)) < phdr->p_memsz) {
765 auto *nhdr = reinterpret_cast<const ElfW(Nhdr) *>(info->dlpi_addr +
766 phdr->p_vaddr + off);
767 constexpr auto kGnuNamesz = 4; // "GNU" with NUL-byte.
768 static_assert(kGnuNamesz % 4 == 0, "kGnuNameSize is aligned to 4.");
769 if (nhdr->n_type == NT_GNU_BUILD_ID && nhdr->n_namesz == kGnuNamesz) {
770 if (off + sizeof(ElfW(Nhdr)) + nhdr->n_namesz + nhdr->n_descsz >
771 phdr->p_memsz) {
772 // Something is very wrong, bail out instead of reading potentially
773 // arbitrary memory.
774 break;
775 }
776 const char *name =
777 reinterpret_cast<const char *>(nhdr) + sizeof(*nhdr);
778 if (internal_memcmp(s1: name, s2: "GNU", n: 3) == 0) {
779 const char *value = reinterpret_cast<const char *>(nhdr) +
780 sizeof(*nhdr) + kGnuNamesz;
781 cur_module.setUuid(uuid: value, size: nhdr->n_descsz);
782 break;
783 }
784 }
785 off += sizeof(*nhdr) + RoundUpTo(size: nhdr->n_namesz, boundary: 4) +
786 RoundUpTo(size: nhdr->n_descsz, boundary: 4);
787 }
788# endif
789 }
790 }
791 modules->push_back(element: cur_module);
792 return 0;
793}
794
795static int dl_iterate_phdr_cb(dl_phdr_info *info, size_t size, void *arg) {
796 DlIteratePhdrData *data = (DlIteratePhdrData *)arg;
797 if (data->first) {
798 InternalMmapVector<char> module_name(kMaxPathLength);
799 data->first = false;
800 // First module is the binary itself.
801 ReadBinaryNameCached(buf: module_name.data(), buf_len: module_name.size());
802 return AddModuleSegments(module_name: module_name.data(), info, modules: data->modules);
803 }
804
805 if (info->dlpi_name)
806 return AddModuleSegments(module_name: info->dlpi_name, info, modules: data->modules);
807
808 return 0;
809}
810
811void ListOfModules::init() {
812 clearOrInit();
813 DlIteratePhdrData data = {.modules: &modules_, .first: true};
814 dl_iterate_phdr(callback: dl_iterate_phdr_cb, data: &data);
815}
816
817void ListOfModules::fallbackInit() { clear(); }
818
819// getrusage does not give us the current RSS, only the max RSS.
820// Still, this is better than nothing if /proc/self/statm is not available
821// for some reason, e.g. due to a sandbox.
822static uptr GetRSSFromGetrusage() {
823 struct rusage usage;
824 if (getrusage(RUSAGE_SELF, usage: &usage)) // Failed, probably due to a sandbox.
825 return 0;
826 return usage.ru_maxrss << 10; // ru_maxrss is in Kb.
827}
828
829uptr GetRSS() {
830 if (!common_flags()->can_use_proc_maps_statm)
831 return GetRSSFromGetrusage();
832 fd_t fd = OpenFile(filename: "/proc/self/statm", mode: RdOnly);
833 if (fd == kInvalidFd)
834 return GetRSSFromGetrusage();
835 char buf[64];
836 uptr len = internal_read(fd, buf, count: sizeof(buf) - 1);
837 internal_close(fd);
838 if ((sptr)len <= 0)
839 return 0;
840 buf[len] = 0;
841 // The format of the file is:
842 // 1084 89 69 11 0 79 0
843 // We need the second number which is RSS in pages.
844 char *pos = buf;
845 // Skip the first number.
846 while (*pos >= '0' && *pos <= '9') pos++;
847 // Skip whitespaces.
848 while (!(*pos >= '0' && *pos <= '9') && *pos != 0) pos++;
849 // Read the number.
850 uptr rss = 0;
851 while (*pos >= '0' && *pos <= '9') rss = rss * 10 + *pos++ - '0';
852 return rss * GetPageSizeCached();
853}
854
855// sysconf(_SC_NPROCESSORS_{CONF,ONLN}) cannot be used on most platforms as
856// they allocate memory.
857u32 GetNumberOfCPUs() {
858# if SANITIZER_FREEBSD || SANITIZER_NETBSD
859 u32 ncpu;
860 int req[2];
861 uptr len = sizeof(ncpu);
862 req[0] = CTL_HW;
863# ifdef HW_NCPUONLINE
864 req[1] = HW_NCPUONLINE;
865# else
866 req[1] = HW_NCPU;
867# endif
868 CHECK_EQ(internal_sysctl(req, 2, &ncpu, &len, NULL, 0), 0);
869 return ncpu;
870# elif SANITIZER_HAIKU
871 system_info info;
872 get_system_info(&info);
873 return info.cpu_count;
874# elif SANITIZER_SOLARIS
875 return sysconf(_SC_NPROCESSORS_ONLN);
876# else
877 cpu_set_t CPUs;
878 CHECK_EQ(sched_getaffinity(0, sizeof(cpu_set_t), &CPUs), 0);
879 return CPU_COUNT(&CPUs);
880# endif
881}
882
883# if SANITIZER_LINUX
884
885# if SANITIZER_ANDROID
886static atomic_uint8_t android_log_initialized;
887
888void AndroidLogInit() {
889 openlog(GetProcessName(), 0, LOG_USER);
890 atomic_store(&android_log_initialized, 1, memory_order_release);
891}
892
893static bool ShouldLogAfterPrintf() {
894 return atomic_load(&android_log_initialized, memory_order_acquire);
895}
896
897extern "C" SANITIZER_WEAK_ATTRIBUTE int async_safe_write_log(int pri,
898 const char *tag,
899 const char *msg);
900extern "C" SANITIZER_WEAK_ATTRIBUTE int __android_log_write(int prio,
901 const char *tag,
902 const char *msg);
903
904// ANDROID_LOG_INFO is 4, but can't be resolved at runtime.
905# define SANITIZER_ANDROID_LOG_INFO 4
906
907// async_safe_write_log is a new public version of __libc_write_log that is
908// used behind syslog. It is preferable to syslog as it will not do any dynamic
909// memory allocation or formatting.
910// If the function is not available, syslog is preferred for L+ (it was broken
911// pre-L) as __android_log_write triggers a racey behavior with the strncpy
912// interceptor. Fallback to __android_log_write pre-L.
913void WriteOneLineToSyslog(const char *s) {
914 if (&async_safe_write_log) {
915 async_safe_write_log(SANITIZER_ANDROID_LOG_INFO, GetProcessName(), s);
916 } else {
917 syslog(LOG_INFO, "%s", s);
918 }
919}
920
921extern "C" SANITIZER_WEAK_ATTRIBUTE void android_set_abort_message(
922 const char *);
923
924void SetAbortMessage(const char *str) {
925 if (&android_set_abort_message)
926 android_set_abort_message(str);
927}
928# else
929void AndroidLogInit() {}
930
931static bool ShouldLogAfterPrintf() { return true; }
932
933void WriteOneLineToSyslog(const char *s) { syslog(LOG_INFO, fmt: "%s", s); }
934
935void SetAbortMessage(const char *str) {}
936# endif // SANITIZER_ANDROID
937
938void LogMessageOnPrintf(const char *str) {
939 if (common_flags()->log_to_syslog && ShouldLogAfterPrintf())
940 WriteToSyslog(buffer: str);
941}
942
943# endif // SANITIZER_LINUX
944
945# if SANITIZER_GLIBC && !SANITIZER_GO
946// glibc crashes when using clock_gettime from a preinit_array function as the
947// vDSO function pointers haven't been initialized yet. __progname is
948// initialized after the vDSO function pointers, so if it exists, is not null
949// and is not empty, we can use clock_gettime.
950extern "C" SANITIZER_WEAK_ATTRIBUTE char *__progname;
951inline bool CanUseVDSO() { return &__progname && __progname && *__progname; }
952
953// MonotonicNanoTime is a timing function that can leverage the vDSO by calling
954// clock_gettime. real_clock_gettime only exists if clock_gettime is
955// intercepted, so define it weakly and use it if available.
956extern "C" SANITIZER_WEAK_ATTRIBUTE int real_clock_gettime(u32 clk_id,
957 void *tp);
958u64 MonotonicNanoTime() {
959 timespec ts;
960 if (CanUseVDSO()) {
961 if (&real_clock_gettime)
962 real_clock_gettime(CLOCK_MONOTONIC, tp: &ts);
963 else
964 clock_gettime(CLOCK_MONOTONIC, tp: &ts);
965 } else {
966 internal_clock_gettime(CLOCK_MONOTONIC, tp: &ts);
967 }
968 return (u64)ts.tv_sec * (1000ULL * 1000 * 1000) + ts.tv_nsec;
969}
970# else
971// Non-glibc & Go always use the regular function.
972u64 MonotonicNanoTime() {
973 timespec ts;
974 clock_gettime(CLOCK_MONOTONIC, &ts);
975 return (u64)ts.tv_sec * (1000ULL * 1000 * 1000) + ts.tv_nsec;
976}
977# endif // SANITIZER_GLIBC && !SANITIZER_GO
978
979void ReExec() {
980 const char *pathname = "/proc/self/exe";
981
982# if SANITIZER_FREEBSD
983 for (const auto *aux = __elf_aux_vector; aux->a_type != AT_NULL; aux++) {
984 if (aux->a_type == AT_EXECPATH) {
985 pathname = static_cast<const char *>(aux->a_un.a_ptr);
986 break;
987 }
988 }
989# elif SANITIZER_NETBSD
990 static const int name[] = {
991 CTL_KERN,
992 KERN_PROC_ARGS,
993 -1,
994 KERN_PROC_PATHNAME,
995 };
996 char path[400];
997 uptr len;
998
999 len = sizeof(path);
1000 if (internal_sysctl(name, ARRAY_SIZE(name), path, &len, NULL, 0) != -1)
1001 pathname = path;
1002# elif SANITIZER_SOLARIS
1003 pathname = getexecname();
1004 CHECK_NE(pathname, NULL);
1005# elif SANITIZER_USE_GETAUXVAL
1006 // Calling execve with /proc/self/exe sets that as $EXEC_ORIGIN. Binaries that
1007 // rely on that will fail to load shared libraries. Query AT_EXECFN instead.
1008 pathname = reinterpret_cast<const char *>(getauxval(AT_EXECFN));
1009# endif
1010
1011 uptr rv = internal_execve(filename: pathname, argv: GetArgv(), envp: GetEnviron());
1012 int rverrno;
1013 CHECK_EQ(internal_iserror(rv, &rverrno), true);
1014 Printf(format: "execve failed, errno %d\n", rverrno);
1015 Die();
1016}
1017
1018void UnmapFromTo(uptr from, uptr to) {
1019 if (to == from)
1020 return;
1021 CHECK(to >= from);
1022 uptr res = internal_munmap(addr: reinterpret_cast<void *>(from), length: to - from);
1023 if (UNLIKELY(internal_iserror(res))) {
1024 Report(format: "ERROR: %s failed to unmap 0x%zx (%zd) bytes at address %p\n",
1025 SanitizerToolName, to - from, to - from, (void *)from);
1026 CHECK("unable to unmap" && 0);
1027 }
1028}
1029
1030uptr MapDynamicShadow(uptr shadow_size_bytes, uptr shadow_scale,
1031 uptr min_shadow_base_alignment, UNUSED uptr &high_mem_end,
1032 uptr granularity) {
1033 const uptr alignment =
1034 Max<uptr>(a: granularity << shadow_scale, b: 1ULL << min_shadow_base_alignment);
1035 const uptr left_padding =
1036 Max<uptr>(a: granularity, b: 1ULL << min_shadow_base_alignment);
1037
1038 const uptr shadow_size = RoundUpTo(size: shadow_size_bytes, boundary: granularity);
1039 const uptr map_size = shadow_size + left_padding + alignment;
1040
1041 const uptr map_start = (uptr)MmapNoAccess(size: map_size);
1042 CHECK_NE(map_start, ~(uptr)0);
1043
1044 const uptr shadow_start = RoundUpTo(size: map_start + left_padding, boundary: alignment);
1045
1046 UnmapFromTo(from: map_start, to: shadow_start - left_padding);
1047 UnmapFromTo(from: shadow_start + shadow_size, to: map_start + map_size);
1048
1049 return shadow_start;
1050}
1051
1052static uptr MmapSharedNoReserve(uptr addr, uptr size) {
1053 return internal_mmap(
1054 addr: reinterpret_cast<void *>(addr), length: size, PROT_READ | PROT_WRITE,
1055 MAP_FIXED | MAP_SHARED | MAP_ANONYMOUS | MAP_NORESERVE, fd: -1, offset: 0);
1056}
1057
1058static uptr MremapCreateAlias(uptr base_addr, uptr alias_addr,
1059 uptr alias_size) {
1060# if SANITIZER_LINUX
1061 return internal_mremap(old_address: reinterpret_cast<void *>(base_addr), old_size: 0, new_size: alias_size,
1062 MREMAP_MAYMOVE | MREMAP_FIXED,
1063 new_address: reinterpret_cast<void *>(alias_addr));
1064# else
1065 CHECK(false && "mremap is not supported outside of Linux");
1066 return 0;
1067# endif
1068}
1069
1070static void CreateAliases(uptr start_addr, uptr alias_size, uptr num_aliases) {
1071 uptr total_size = alias_size * num_aliases;
1072 uptr mapped = MmapSharedNoReserve(addr: start_addr, size: total_size);
1073 CHECK_EQ(mapped, start_addr);
1074
1075 for (uptr i = 1; i < num_aliases; ++i) {
1076 uptr alias_addr = start_addr + i * alias_size;
1077 CHECK_EQ(MremapCreateAlias(start_addr, alias_addr, alias_size), alias_addr);
1078 }
1079}
1080
1081uptr MapDynamicShadowAndAliases(uptr shadow_size, uptr alias_size,
1082 uptr num_aliases, uptr ring_buffer_size) {
1083 CHECK_EQ(alias_size & (alias_size - 1), 0);
1084 CHECK_EQ(num_aliases & (num_aliases - 1), 0);
1085 CHECK_EQ(ring_buffer_size & (ring_buffer_size - 1), 0);
1086
1087 const uptr granularity = GetMmapGranularity();
1088 shadow_size = RoundUpTo(size: shadow_size, boundary: granularity);
1089 CHECK_EQ(shadow_size & (shadow_size - 1), 0);
1090
1091 const uptr alias_region_size = alias_size * num_aliases;
1092 const uptr alignment =
1093 2 * Max(a: Max(a: shadow_size, b: alias_region_size), b: ring_buffer_size);
1094 const uptr left_padding = ring_buffer_size;
1095
1096 const uptr right_size = alignment;
1097 const uptr map_size = left_padding + 2 * alignment;
1098
1099 const uptr map_start = reinterpret_cast<uptr>(MmapNoAccess(size: map_size));
1100 CHECK_NE(map_start, static_cast<uptr>(-1));
1101 const uptr right_start = RoundUpTo(size: map_start + left_padding, boundary: alignment);
1102
1103 UnmapFromTo(from: map_start, to: right_start - left_padding);
1104 UnmapFromTo(from: right_start + right_size, to: map_start + map_size);
1105
1106 CreateAliases(start_addr: right_start + right_size / 2, alias_size, num_aliases);
1107
1108 return right_start;
1109}
1110
1111void InitializePlatformCommonFlags(CommonFlags *cf) {
1112# if SANITIZER_ANDROID
1113 if (&__libc_get_static_tls_bounds == nullptr)
1114 cf->detect_leaks = false;
1115# endif
1116}
1117
1118} // namespace __sanitizer
1119
1120#endif
1121