1//===-- sanitizer_tls_get_addr.cpp ----------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Handle the __tls_get_addr call.
10//
11//===----------------------------------------------------------------------===//
12
13#include "sanitizer_tls_get_addr.h"
14
15#include "sanitizer_allocator_interface.h"
16#include "sanitizer_atomic.h"
17#include "sanitizer_common/sanitizer_common.h"
18#include "sanitizer_common/sanitizer_internal_defs.h"
19#include "sanitizer_flags.h"
20#include "sanitizer_platform_interceptors.h"
21
22namespace __sanitizer {
23#if SANITIZER_INTERCEPT_TLS_GET_ADDR
24
25// The actual parameter that comes to __tls_get_addr
26// is a pointer to a struct with two words in it:
27struct TlsGetAddrParam {
28 uptr dso_id;
29 uptr offset;
30};
31
32// This must be static TLS
33__attribute__((tls_model("initial-exec")))
34static __thread DTLS dtls;
35
36// Make sure we properly destroy the DTLS objects:
37// this counter should never get too large.
38static atomic_uintptr_t number_of_live_dtls;
39
40static const uptr kDestroyedThread = -1;
41
42static void DTLS_Deallocate(DTLS::DTVBlock *block) {
43 VReport(2, "__tls_get_addr: DTLS_Deallocate %p\n", (void *)block);
44 UnmapOrDie(addr: block, size: sizeof(DTLS::DTVBlock));
45 atomic_fetch_sub(a: &number_of_live_dtls, v: 1, mo: memory_order_relaxed);
46}
47
48static DTLS::DTVBlock *DTLS_NextBlock(atomic_uintptr_t *cur) {
49 uptr v = atomic_load(a: cur, mo: memory_order_acquire);
50 if (v == kDestroyedThread)
51 return nullptr;
52 DTLS::DTVBlock *next = (DTLS::DTVBlock *)v;
53 if (next)
54 return next;
55 DTLS::DTVBlock *new_dtv =
56 (DTLS::DTVBlock *)MmapOrDie(size: sizeof(DTLS::DTVBlock), mem_type: "DTLS_NextBlock");
57 uptr prev = 0;
58 if (!atomic_compare_exchange_strong(a: cur, cmp: &prev, xchg: (uptr)new_dtv,
59 mo: memory_order_seq_cst)) {
60 UnmapOrDie(addr: new_dtv, size: sizeof(DTLS::DTVBlock));
61 return (DTLS::DTVBlock *)prev;
62 }
63 uptr num_live_dtls =
64 atomic_fetch_add(a: &number_of_live_dtls, v: 1, mo: memory_order_relaxed);
65 VReport(2, "__tls_get_addr: DTLS_NextBlock %p %zd\n", (void *)&dtls,
66 num_live_dtls);
67 return new_dtv;
68}
69
70static DTLS::DTV *DTLS_Find(uptr id) {
71 VReport(3, "__tls_get_addr: DTLS_Find %p %zd\n", (void *)&dtls, id);
72 static constexpr uptr kPerBlock = ARRAY_SIZE(DTLS::DTVBlock::dtvs);
73 DTLS::DTVBlock *cur = DTLS_NextBlock(cur: &dtls.dtv_block);
74 if (!cur)
75 return nullptr;
76 for (; id >= kPerBlock; id -= kPerBlock) cur = DTLS_NextBlock(cur: &cur->next);
77 return cur->dtvs + id;
78}
79
80void DTLS_Destroy() {
81 if (!common_flags()->intercept_tls_get_addr) return;
82 VReport(2, "__tls_get_addr: DTLS_Destroy %p\n", (void *)&dtls);
83 DTLS::DTVBlock *block = (DTLS::DTVBlock *)atomic_exchange(
84 a: &dtls.dtv_block, v: kDestroyedThread, mo: memory_order_release);
85 while (block) {
86 DTLS::DTVBlock *next =
87 (DTLS::DTVBlock *)atomic_load(a: &block->next, mo: memory_order_acquire);
88 DTLS_Deallocate(block);
89 block = next;
90 }
91}
92
93#if defined(__powerpc64__) || defined(__mips__)
94// This is glibc's TLS_DTV_OFFSET:
95// "Dynamic thread vector pointers point 0x8000 past the start of each
96// TLS block." (sysdeps/<arch>/dl-tls.h)
97static const uptr kDtvOffset = 0x8000;
98#elif defined(__riscv)
99// This is glibc's TLS_DTV_OFFSET:
100// "Dynamic thread vector pointers point 0x800 past the start of each
101// TLS block." (sysdeps/riscv/dl-tls.h)
102static const uptr kDtvOffset = 0x800;
103#else
104static const uptr kDtvOffset = 0;
105#endif
106
107extern "C" {
108SANITIZER_WEAK_ATTRIBUTE
109uptr __sanitizer_get_allocated_size(const void *p);
110
111SANITIZER_WEAK_ATTRIBUTE
112const void *__sanitizer_get_allocated_begin(const void *p);
113}
114
115SANITIZER_INTERFACE_WEAK_DEF(uptr, __sanitizer_get_dtls_size,
116 const void *tls_begin) {
117 const void *start = __sanitizer_get_allocated_begin(p: tls_begin);
118 if (!start)
119 return 0;
120 CHECK_LE(start, tls_begin);
121 uptr tls_size = __sanitizer_get_allocated_size(p: start);
122 VReport(2, "__tls_get_addr: glibc DTLS suspected; tls={%p,0x%zx}\n",
123 tls_begin, tls_size);
124 uptr offset =
125 (reinterpret_cast<uptr>(tls_begin) - reinterpret_cast<uptr>(start));
126 CHECK_LE(offset, tls_size);
127 return tls_size - offset;
128}
129
130DTLS::DTV *DTLS_on_tls_get_addr(void *arg_void, void *res,
131 uptr static_tls_begin, uptr static_tls_end) {
132 if (!common_flags()->intercept_tls_get_addr) return 0;
133 TlsGetAddrParam *arg = reinterpret_cast<TlsGetAddrParam *>(arg_void);
134 uptr dso_id = arg->dso_id;
135 DTLS::DTV *dtv = DTLS_Find(id: dso_id);
136 if (!dtv || dtv->beg)
137 return nullptr;
138 CHECK_LE(static_tls_begin, static_tls_end);
139 uptr tls_beg = reinterpret_cast<uptr>(res) - arg->offset - kDtvOffset;
140 VReport(2,
141 "__tls_get_addr: %p {0x%zx,0x%zx} => %p; tls_beg: %p; sp: %p "
142 "num_live_dtls %zd\n",
143 (void *)arg, arg->dso_id, arg->offset, res, (void *)tls_beg,
144 (void *)&tls_beg,
145 atomic_load(&number_of_live_dtls, memory_order_relaxed));
146 if (tls_beg >= static_tls_begin && tls_beg < static_tls_end) {
147 // This is the static TLS block which was initialized / unpoisoned at thread
148 // creation.
149 VReport(2, "__tls_get_addr: static tls: %p\n", (void *)tls_beg);
150 dtv->beg = tls_beg;
151 dtv->size = 0;
152 return nullptr;
153 }
154 if (uptr tls_size =
155 __sanitizer_get_dtls_size(tls_begin: reinterpret_cast<void *>(tls_beg))) {
156 dtv->beg = tls_beg;
157 dtv->size = tls_size;
158 return dtv;
159 }
160 VReport(2, "__tls_get_addr: Can't guess glibc version\n");
161 // This may happen inside the DTOR a thread, or async signal handlers before
162 // thread initialization, so just ignore it.
163 //
164 // If the unknown block is dynamic TLS, unlikely we will be able to recognize
165 // it in future, mark it as done with '{tls_beg, 0}'.
166 //
167 // If the block is static TLS, possible reason of failed detection is nullptr
168 // in `static_tls_begin`. Regardless of reasons, the future handling of static
169 // TLS is still '{tls_beg, 0}'.
170 dtv->beg = tls_beg;
171 dtv->size = 0;
172 return nullptr;
173}
174
175DTLS *DTLS_Get() { return &dtls; }
176
177bool DTLSInDestruction(DTLS *dtls) {
178 return atomic_load(a: &dtls->dtv_block, mo: memory_order_relaxed) ==
179 kDestroyedThread;
180}
181
182#else
183SANITIZER_INTERFACE_WEAK_DEF(uptr, __sanitizer_get_dtls_size, const void *) {
184 return 0;
185}
186DTLS::DTV *DTLS_on_tls_get_addr(void *arg, void *res,
187 unsigned long, unsigned long) { return 0; }
188DTLS *DTLS_Get() { return 0; }
189void DTLS_Destroy() {}
190bool DTLSInDestruction(DTLS *dtls) {
191 UNREACHABLE("dtls is unsupported on this platform!");
192}
193
194#endif // SANITIZER_INTERCEPT_TLS_GET_ADDR
195
196} // namespace __sanitizer
197