1//===-- hwasan_thread_list.h ------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is a part of HWAddressSanitizer.
10//
11//===----------------------------------------------------------------------===//
12
13// HwasanThreadList is a registry for live threads, as well as an allocator for
14// HwasanThread objects and their stack history ring buffers. There are
15// constraints on memory layout of the shadow region and CompactRingBuffer that
16// are part of the ABI contract between compiler-rt and llvm.
17//
18// * Start of the shadow memory region is aligned to 2**kShadowBaseAlignment.
19// * All stack ring buffers are located within (2**kShadowBaseAlignment)
20// sized region below and adjacent to the shadow region.
21// * Each ring buffer has a size of (2**N)*4096 where N is in [0, 7), and is
22// aligned to twice its size. The value of N can be different for each buffer.
23//
24// These constrains guarantee that, given an address A of any element of the
25// ring buffer,
26// A_next = (A + sizeof(uptr)) & ~((1 << (N + 13)) - 1)
27// is the address of the next element of that ring buffer (with wrap-around).
28// And, with K = kShadowBaseAlignment,
29// S = (A | ((1 << K) - 1)) + 1
30// (align up to kShadowBaseAlignment) is the start of the shadow region.
31//
32// These calculations are used in compiler instrumentation to update the ring
33// buffer and obtain the base address of shadow using only two inputs: address
34// of the current element of the ring buffer, and N (i.e. size of the ring
35// buffer). Since the value of N is very limited, we pack both inputs into a
36// single thread-local word as
37// (1 << (N + 56)) | A
38// See the implementation of class CompactRingBuffer, which is what is stored in
39// said thread-local word.
40//
41// Note the unusual way of aligning up the address of the shadow:
42// (A | ((1 << K) - 1)) + 1
43// It is only correct if A is not already equal to the shadow base address, but
44// it saves 2 instructions on AArch64.
45
46#include "hwasan.h"
47#include "hwasan_allocator.h"
48#include "hwasan_flags.h"
49#include "hwasan_thread.h"
50#include "sanitizer_common/sanitizer_thread_arg_retval.h"
51
52namespace __hwasan {
53
54static uptr RingBufferSize() {
55 uptr desired_bytes = flags()->stack_history_size * sizeof(uptr);
56 // FIXME: increase the limit to 8 once this bug is fixed:
57 // https://bugs.llvm.org/show_bug.cgi?id=39030
58 // Note that we *cannot* do that on Android, as the runtime will indefinitely
59 // have to support code that is compiled with ashr, which only works with
60 // shifts up to 6.
61 for (int shift = 0; shift < 7; ++shift) {
62 uptr size = 4096 * (1ULL << shift);
63 if (size >= desired_bytes)
64 return size;
65 }
66 Printf(format: "stack history size too large: %d\n", flags()->stack_history_size);
67 CHECK(0);
68 return 0;
69}
70
71struct ThreadStats {
72 uptr n_live_threads;
73 uptr total_stack_size;
74};
75
76class SANITIZER_MUTEX HwasanThreadList {
77 public:
78 HwasanThreadList(uptr storage, uptr size)
79 : free_space_(storage), free_space_end_(storage + size) {
80 // [storage, storage + size) is used as a vector of
81 // thread_alloc_size_-sized, ring_buffer_size_*2-aligned elements.
82 // Each element contains
83 // * a ring buffer at offset 0,
84 // * a Thread object at offset ring_buffer_size_.
85 ring_buffer_size_ = RingBufferSize();
86 thread_alloc_size_ =
87 RoundUpTo(size: ring_buffer_size_ + sizeof(Thread), boundary: ring_buffer_size_ * 2);
88 }
89
90 Thread *CreateCurrentThread(const Thread::InitState *state = nullptr)
91 SANITIZER_EXCLUDES(free_list_mutex_, live_list_mutex_) {
92 Thread *t = nullptr;
93 {
94 SpinMutexLock l(&free_list_mutex_);
95 if (!free_list_.empty()) {
96 t = free_list_.back();
97 free_list_.pop_back();
98 }
99 }
100 if (t) {
101 uptr start = (uptr)t - ring_buffer_size_;
102 internal_memset(s: (void *)start, c: 0, n: ring_buffer_size_ + sizeof(Thread));
103 } else {
104 t = AllocThread();
105 }
106 {
107 SpinMutexLock l(&live_list_mutex_);
108 live_list_.push_back(element: t);
109 }
110 t->Init(stack_buffer_start: (uptr)t - ring_buffer_size_, stack_buffer_size: ring_buffer_size_, state);
111 AddThreadStats(t);
112 return t;
113 }
114
115 void DontNeedThread(Thread *t) {
116 uptr start = (uptr)t - ring_buffer_size_;
117 ReleaseMemoryPagesToOS(beg: start, end: start + thread_alloc_size_);
118 }
119
120 void RemoveThreadFromLiveList(Thread *t)
121 SANITIZER_EXCLUDES(live_list_mutex_) {
122 SpinMutexLock l(&live_list_mutex_);
123 for (Thread *&t2 : live_list_)
124 if (t2 == t) {
125 // To remove t2, copy the last element of the list in t2's position, and
126 // pop_back(). This works even if t2 is itself the last element.
127 t2 = live_list_.back();
128 live_list_.pop_back();
129 return;
130 }
131 CHECK(0 && "thread not found in live list");
132 }
133
134 void ReleaseThread(Thread *t) SANITIZER_EXCLUDES(free_list_mutex_) {
135 RemoveThreadStats(t);
136 RemoveThreadFromLiveList(t);
137 t->Destroy();
138 DontNeedThread(t);
139 SpinMutexLock l(&free_list_mutex_);
140 free_list_.push_back(element: t);
141 }
142
143 Thread *GetThreadByBufferAddress(uptr p) {
144 return (Thread *)(RoundDownTo(x: p, boundary: ring_buffer_size_ * 2) +
145 ring_buffer_size_);
146 }
147
148 uptr MemoryUsedPerThread() {
149 uptr res = sizeof(Thread) + ring_buffer_size_;
150 if (auto sz = flags()->heap_history_size)
151 res += HeapAllocationsRingBuffer::SizeInBytes(Size: sz);
152 return res;
153 }
154
155 template <class CB>
156 void VisitAllLiveThreads(CB cb) SANITIZER_EXCLUDES(live_list_mutex_) {
157 SpinMutexLock l(&live_list_mutex_);
158 for (Thread *t : live_list_) cb(t);
159 }
160
161 template <class CB>
162 Thread *FindThreadLocked(CB cb) SANITIZER_CHECK_LOCKED(live_list_mutex_) {
163 CheckLocked();
164 for (Thread *t : live_list_)
165 if (cb(t))
166 return t;
167 return nullptr;
168 }
169
170 void AddThreadStats(Thread *t) SANITIZER_EXCLUDES(stats_mutex_) {
171 SpinMutexLock l(&stats_mutex_);
172 stats_.n_live_threads++;
173 stats_.total_stack_size += t->stack_size();
174 }
175
176 void RemoveThreadStats(Thread *t) SANITIZER_EXCLUDES(stats_mutex_) {
177 SpinMutexLock l(&stats_mutex_);
178 stats_.n_live_threads--;
179 stats_.total_stack_size -= t->stack_size();
180 }
181
182 ThreadStats GetThreadStats() SANITIZER_EXCLUDES(stats_mutex_) {
183 SpinMutexLock l(&stats_mutex_);
184 return stats_;
185 }
186
187 uptr GetRingBufferSize() const { return ring_buffer_size_; }
188
189 void Lock() SANITIZER_ACQUIRE(live_list_mutex_) { live_list_mutex_.Lock(); }
190 void CheckLocked() const SANITIZER_CHECK_LOCKED(live_list_mutex_) {
191 live_list_mutex_.CheckLocked();
192 }
193 void Unlock() SANITIZER_RELEASE(live_list_mutex_) {
194 live_list_mutex_.Unlock();
195 }
196
197 private:
198 Thread *AllocThread() {
199 SpinMutexLock l(&free_space_mutex_);
200 uptr align = ring_buffer_size_ * 2;
201 CHECK(IsAligned(free_space_, align));
202 Thread *t = (Thread *)(free_space_ + ring_buffer_size_);
203 free_space_ += thread_alloc_size_;
204 CHECK_LE(free_space_, free_space_end_);
205 return t;
206 }
207
208 SpinMutex free_space_mutex_;
209 uptr free_space_;
210 uptr free_space_end_;
211 uptr ring_buffer_size_;
212 uptr thread_alloc_size_;
213
214 SpinMutex free_list_mutex_;
215 InternalMmapVector<Thread *> free_list_
216 SANITIZER_GUARDED_BY(free_list_mutex_);
217 SpinMutex live_list_mutex_;
218 InternalMmapVector<Thread *> live_list_
219 SANITIZER_GUARDED_BY(live_list_mutex_);
220
221 SpinMutex stats_mutex_;
222 ThreadStats stats_ SANITIZER_GUARDED_BY(stats_mutex_);
223};
224
225void InitThreadList(uptr storage, uptr size);
226HwasanThreadList &hwasanThreadList();
227ThreadArgRetval &hwasanThreadArgRetval();
228
229} // namespace __hwasan
230