1//===-- tsan_rtl.h ----------------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is a part of ThreadSanitizer (TSan), a race detector.
10//
11// Main internal TSan header file.
12//
13// Ground rules:
14// - C++ run-time should not be used (static CTORs, RTTI, exceptions, static
15// function-scope locals)
16// - All functions/classes/etc reside in namespace __tsan, except for those
17// declared in tsan_interface.h.
18// - Platform-specific files should be used instead of ifdefs (*).
19// - No system headers included in header files (*).
20// - Platform specific headres included only into platform-specific files (*).
21//
22// (*) Except when inlining is critical for performance.
23//===----------------------------------------------------------------------===//
24
25#ifndef TSAN_RTL_H
26#define TSAN_RTL_H
27
28#include "sanitizer_common/sanitizer_allocator.h"
29#include "sanitizer_common/sanitizer_allocator_internal.h"
30#include "sanitizer_common/sanitizer_asm.h"
31#include "sanitizer_common/sanitizer_common.h"
32#include "sanitizer_common/sanitizer_deadlock_detector_interface.h"
33#include "sanitizer_common/sanitizer_libignore.h"
34#include "sanitizer_common/sanitizer_suppressions.h"
35#include "sanitizer_common/sanitizer_thread_registry.h"
36#include "sanitizer_common/sanitizer_vector.h"
37#include "tsan_adaptive_delay.h"
38#include "tsan_defs.h"
39#include "tsan_flags.h"
40#include "tsan_ignoreset.h"
41#include "tsan_ilist.h"
42#include "tsan_mman.h"
43#include "tsan_mutexset.h"
44#include "tsan_platform.h"
45#include "tsan_report.h"
46#include "tsan_shadow.h"
47#include "tsan_stack_trace.h"
48#include "tsan_sync.h"
49#include "tsan_trace.h"
50#include "tsan_vector_clock.h"
51
52#if SANITIZER_WORDSIZE != 64
53# error "ThreadSanitizer is supported only on 64-bit platforms"
54#endif
55
56namespace __tsan {
57
58extern bool ready_to_symbolize;
59
60#if !SANITIZER_GO
61struct MapUnmapCallback;
62# if defined(__mips64) || defined(__aarch64__) || defined(__loongarch__) || \
63 defined(__powerpc__) || SANITIZER_RISCV64
64
65struct AP32 {
66 static const uptr kSpaceBeg = SANITIZER_MMAP_BEGIN;
67 static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE;
68 static const uptr kMetadataSize = 0;
69 typedef __sanitizer::CompactSizeClassMap SizeClassMap;
70 static const uptr kRegionSizeLog = 20;
71 using AddressSpaceView = LocalAddressSpaceView;
72 typedef __tsan::MapUnmapCallback MapUnmapCallback;
73 static const uptr kFlags = 0;
74};
75typedef SizeClassAllocator32<AP32> PrimaryAllocator;
76#else
77struct AP64 { // Allocator64 parameters. Deliberately using a short name.
78# if defined(__s390x__)
79 typedef MappingS390x Mapping;
80# else
81 typedef Mapping48AddressSpace Mapping;
82# endif
83 static const uptr kSpaceBeg = Mapping::kHeapMemBeg;
84 static const uptr kSpaceSize = Mapping::kHeapMemEnd - Mapping::kHeapMemBeg;
85 static const uptr kMetadataSize = 0;
86 typedef DefaultSizeClassMap SizeClassMap;
87 typedef __tsan::MapUnmapCallback MapUnmapCallback;
88 static const uptr kFlags = 0;
89 using AddressSpaceView = LocalAddressSpaceView;
90};
91typedef SizeClassAllocator64<AP64> PrimaryAllocator;
92#endif
93typedef CombinedAllocator<PrimaryAllocator> Allocator;
94typedef Allocator::AllocatorCache AllocatorCache;
95Allocator *allocator();
96#endif
97
98struct ThreadSignalContext;
99
100struct JmpBuf {
101 uptr sp;
102 int int_signal_send;
103 bool in_blocking_func;
104 uptr oldset_stack_size;
105 uptr in_signal_handler;
106 uptr *shadow_stack_pos;
107};
108
109// A Processor represents a physical thread, or a P for Go.
110// It is used to store internal resources like allocate cache, and does not
111// participate in race-detection logic (invisible to end user).
112// In C++ it is tied to an OS thread just like ThreadState, however ideally
113// it should be tied to a CPU (this way we will have fewer allocator caches).
114// In Go it is tied to a P, so there are significantly fewer Processor's than
115// ThreadState's (which are tied to Gs).
116// A ThreadState must be wired with a Processor to handle events.
117struct Processor {
118 ThreadState *thr; // currently wired thread, or nullptr
119#if !SANITIZER_GO
120 AllocatorCache alloc_cache;
121 InternalAllocatorCache internal_alloc_cache;
122#endif
123 DenseSlabAllocCache block_cache;
124 DenseSlabAllocCache sync_cache;
125 DDPhysicalThread *dd_pt;
126};
127
128#if !SANITIZER_GO
129// ScopedGlobalProcessor temporary setups a global processor for the current
130// thread, if it does not have one. Intended for interceptors that can run
131// at the very thread end, when we already destroyed the thread processor.
132struct ScopedGlobalProcessor {
133 ScopedGlobalProcessor();
134 ~ScopedGlobalProcessor();
135};
136#endif
137
138struct TidEpoch {
139 Tid tid;
140 Epoch epoch;
141};
142
143struct alignas(SANITIZER_CACHE_LINE_SIZE) TidSlot {
144 Mutex mtx;
145 Sid sid;
146 atomic_uint32_t raw_epoch;
147 ThreadState *thr;
148 Vector<TidEpoch> journal;
149 INode node;
150
151 Epoch epoch() const {
152 return static_cast<Epoch>(atomic_load(a: &raw_epoch, mo: memory_order_relaxed));
153 }
154
155 void SetEpoch(Epoch v) {
156 atomic_store(a: &raw_epoch, v: static_cast<u32>(v), mo: memory_order_relaxed);
157 }
158
159 TidSlot();
160};
161
162// This struct is stored in TLS.
163struct alignas(SANITIZER_CACHE_LINE_SIZE) ThreadState {
164 FastState fast_state;
165 int ignore_sync;
166#if !SANITIZER_GO
167 int ignore_interceptors;
168#endif
169 uptr *shadow_stack_pos;
170
171 // Current position in tctx->trace.Back()->events (Event*).
172 atomic_uintptr_t trace_pos;
173 // PC of the last memory access, used to compute PC deltas in the trace.
174 uptr trace_prev_pc;
175
176 // Technically `current` should be a separate THREADLOCAL variable;
177 // but it is placed here in order to share cache line with previous fields.
178 ThreadState* current;
179
180 atomic_sint32_t pending_signals;
181
182 VectorClock clock;
183
184 // This is a slow path flag. On fast path, fast_state.GetIgnoreBit() is read.
185 // We do not distinguish beteween ignoring reads and writes
186 // for better performance.
187 int ignore_reads_and_writes;
188 int suppress_reports;
189 // Go does not support ignores.
190#if !SANITIZER_GO
191 IgnoreSet mop_ignore_set;
192 IgnoreSet sync_ignore_set;
193#endif
194 uptr *shadow_stack;
195 uptr *shadow_stack_end;
196#if !SANITIZER_GO
197 Vector<JmpBuf> jmp_bufs;
198 int in_symbolizer;
199 atomic_uintptr_t in_blocking_func;
200 bool in_ignored_lib;
201 bool is_inited;
202#endif
203 MutexSet mset;
204 bool is_dead;
205 const Tid tid;
206 uptr stk_addr;
207 uptr stk_size;
208 uptr tls_addr;
209 uptr tls_size;
210 ThreadContext *tctx;
211
212 DDLogicalThread *dd_lt;
213
214 TidSlot *slot;
215 uptr slot_epoch;
216 bool slot_locked;
217
218 // Current wired Processor, or nullptr. Required to handle any events.
219 Processor *proc1;
220#if !SANITIZER_GO
221 Processor *proc() { return proc1; }
222#else
223 Processor *proc();
224#endif
225
226 atomic_uintptr_t in_signal_handler;
227 atomic_uintptr_t signal_ctx;
228
229#if !SANITIZER_GO
230 StackID last_sleep_stack_id;
231 VectorClock last_sleep_clock;
232#endif
233
234 // Set in regions of runtime that must be signal-safe and fork-safe.
235 // If set, malloc must not be called.
236 int nomalloc;
237
238 const ReportDesc *current_report;
239
240#if SANITIZER_APPLE && !SANITIZER_GO
241 bool in_internal_write_call;
242#endif
243
244 AdaptiveDelayState adaptive_delay_state;
245
246 explicit ThreadState(Tid tid);
247};
248
249#if !SANITIZER_GO
250#if SANITIZER_APPLE || SANITIZER_ANDROID
251ThreadState *cur_thread();
252void set_cur_thread(ThreadState *thr);
253void cur_thread_finalize();
254inline ThreadState *cur_thread_init() { return cur_thread(); }
255# else
256__attribute__((tls_model("initial-exec")))
257extern THREADLOCAL char cur_thread_placeholder[];
258inline ThreadState *cur_thread() {
259 return reinterpret_cast<ThreadState *>(cur_thread_placeholder)->current;
260}
261inline ThreadState *cur_thread_init() {
262 ThreadState *thr = reinterpret_cast<ThreadState *>(cur_thread_placeholder);
263 if (UNLIKELY(!thr->current))
264 thr->current = thr;
265 return thr->current;
266}
267inline void set_cur_thread(ThreadState *thr) {
268 reinterpret_cast<ThreadState *>(cur_thread_placeholder)->current = thr;
269}
270inline void cur_thread_finalize() { }
271# endif // SANITIZER_APPLE || SANITIZER_ANDROID
272#endif // SANITIZER_GO
273
274class ThreadContext final : public ThreadContextBase {
275 public:
276 explicit ThreadContext(Tid tid);
277 ~ThreadContext();
278 ThreadState *thr;
279 StackID creation_stack_id;
280 VectorClock *sync;
281 uptr sync_epoch;
282 Trace trace;
283
284 // Override superclass callbacks.
285 void OnDead() override;
286 void OnJoined(void *arg) override;
287 void OnFinished() override;
288 void OnStarted(void *arg) override;
289 void OnCreated(void *arg) override;
290 void OnReset() override;
291 void OnDetached(void *arg) override;
292};
293
294struct RacyStacks {
295 MD5Hash hash[2];
296 bool operator==(const RacyStacks &other) const;
297};
298
299struct RacyAddress {
300 uptr addr_min;
301 uptr addr_max;
302};
303
304struct FiredSuppression {
305 ReportType type;
306 uptr pc_or_addr;
307 Suppression *supp;
308};
309
310struct Context {
311 Context();
312
313 bool initialized;
314#if !SANITIZER_GO
315 bool after_multithreaded_fork;
316#endif
317
318 MetaMap metamap;
319
320 atomic_uint32_t nreported;
321 atomic_uint64_t last_symbolize_time_ns;
322
323 void *background_thread;
324 atomic_uint32_t stop_background_thread;
325
326 ThreadRegistry thread_registry;
327
328 // This is used to prevent a very unlikely but very pathological behavior.
329 // Since memory access handling is not synchronized with DoReset,
330 // a thread running concurrently with DoReset can leave a bogus shadow value
331 // that will be later falsely detected as a race. For such false races
332 // RestoreStack will return false and we will not report it.
333 // However, consider that a thread leaves a whole lot of such bogus values
334 // and these values are later read by a whole lot of threads.
335 // This will cause massive amounts of ReportRace calls and lots of
336 // serialization. In very pathological cases the resulting slowdown
337 // can be >100x. This is very unlikely, but it was presumably observed
338 // in practice: https://github.com/google/sanitizers/issues/1552
339 // If this happens, previous access sid+epoch will be the same for all of
340 // these false races b/c if the thread will try to increment epoch, it will
341 // notice that DoReset has happened and will stop producing bogus shadow
342 // values. So, last_spurious_race is used to remember the last sid+epoch
343 // for which RestoreStack returned false. Then it is used to filter out
344 // races with the same sid+epoch very early and quickly.
345 // It is of course possible that multiple threads left multiple bogus shadow
346 // values and all of them are read by lots of threads at the same time.
347 // In such case last_spurious_race will only be able to deduplicate a few
348 // races from one thread, then few from another and so on. An alternative
349 // would be to hold an array of such sid+epoch, but we consider such scenario
350 // as even less likely.
351 // Note: this can lead to some rare false negatives as well:
352 // 1. When a legit access with the same sid+epoch participates in a race
353 // as the "previous" memory access, it will be wrongly filtered out.
354 // 2. When RestoreStack returns false for a legit memory access because it
355 // was already evicted from the thread trace, we will still remember it in
356 // last_spurious_race. Then if there is another racing memory access from
357 // the same thread that happened in the same epoch, but was stored in the
358 // next thread trace part (which is still preserved in the thread trace),
359 // we will also wrongly filter it out while RestoreStack would actually
360 // succeed for that second memory access.
361 RawShadow last_spurious_race;
362
363 Mutex racy_mtx;
364 Vector<RacyStacks> racy_stacks;
365 // Number of fired suppressions may be large enough.
366 Mutex fired_suppressions_mtx;
367 InternalMmapVector<FiredSuppression> fired_suppressions;
368 DDetector *dd;
369
370 Flags flags;
371 fd_t memprof_fd;
372
373 // The last slot index (kFreeSid) is used to denote freed memory.
374 TidSlot slots[kThreadSlotCount - 1];
375
376 // Protects global_epoch, slot_queue, trace_part_recycle.
377 Mutex slot_mtx;
378 uptr global_epoch; // guarded by slot_mtx and by all slot mutexes
379 bool resetting; // global reset is in progress
380 IList<TidSlot, &TidSlot::node> slot_queue SANITIZER_GUARDED_BY(slot_mtx);
381 IList<TraceHeader, &TraceHeader::global, TracePart> trace_part_recycle
382 SANITIZER_GUARDED_BY(slot_mtx);
383 uptr trace_part_total_allocated SANITIZER_GUARDED_BY(slot_mtx);
384 uptr trace_part_recycle_finished SANITIZER_GUARDED_BY(slot_mtx);
385 uptr trace_part_finished_excess SANITIZER_GUARDED_BY(slot_mtx);
386#if SANITIZER_GO
387 uptr mapped_shadow_begin;
388 uptr mapped_shadow_end;
389#endif
390};
391
392extern Context *ctx; // The one and the only global runtime context.
393
394ALWAYS_INLINE Flags *flags() {
395 return &ctx->flags;
396}
397
398struct ScopedIgnoreInterceptors {
399 ScopedIgnoreInterceptors() {
400#if !SANITIZER_GO
401 cur_thread()->ignore_interceptors++;
402#endif
403 }
404
405 ~ScopedIgnoreInterceptors() {
406#if !SANITIZER_GO
407 cur_thread()->ignore_interceptors--;
408#endif
409 }
410};
411
412const char *GetObjectTypeFromTag(uptr tag);
413const char *GetReportHeaderFromTag(uptr tag);
414uptr TagFromShadowStackFrame(uptr pc);
415
416class ScopedReport {
417 public:
418 explicit ScopedReport(ReportType typ, uptr tag = kExternalTagNone);
419 ~ScopedReport();
420
421 void AddMemoryAccess(uptr addr, uptr external_tag, Shadow s, Tid tid,
422 StackTrace stack, const MutexSet *mset);
423 void AddStack(StackTrace stack, bool suppressable = false);
424 void AddThread(const ThreadContext *tctx, bool suppressable = false);
425 void AddThread(Tid tid, bool suppressable = false);
426 void AddUniqueTid(Tid unique_tid);
427 int AddMutex(uptr addr, StackID creation_stack_id);
428 void AddLocation(uptr addr, uptr size);
429 void AddSleep(StackID stack_id);
430 void SetCount(int count);
431 void SetSigNum(int sig);
432 void SymbolizeStackElems(void);
433
434 const ReportDesc *GetReport() const;
435
436 private:
437 ReportDesc *rep_;
438 // Symbolizer makes lots of intercepted calls. If we try to process them,
439 // at best it will cause deadlocks on internal mutexes.
440 ScopedIgnoreInterceptors ignore_interceptors_;
441 ScopedErrorReportLock lock_;
442
443 void AddThreadLocked(const ThreadContext* tctx, bool suppressable = false);
444
445 ScopedReport(const ScopedReport&) = delete;
446 void operator=(const ScopedReport&) = delete;
447};
448
449bool ShouldReport(ThreadState *thr, ReportType typ);
450ThreadContext *IsThreadStackOrTls(uptr addr, bool *is_stack);
451
452// The stack could look like:
453// <start> | <main> | <foo> | tag | <bar>
454// This will extract the tag and keep:
455// <start> | <main> | <foo> | <bar>
456template<typename StackTraceTy>
457void ExtractTagFromStack(StackTraceTy *stack, uptr *tag = nullptr) {
458 if (stack->size < 2) return;
459 uptr possible_tag_pc = stack->trace[stack->size - 2];
460 uptr possible_tag = TagFromShadowStackFrame(pc: possible_tag_pc);
461 if (possible_tag == kExternalTagNone) return;
462 stack->trace_buffer[stack->size - 2] = stack->trace_buffer[stack->size - 1];
463 stack->size -= 1;
464 if (tag) *tag = possible_tag;
465}
466
467template<typename StackTraceTy>
468void ObtainCurrentStack(ThreadState *thr, uptr toppc, StackTraceTy *stack,
469 uptr *tag = nullptr) {
470 uptr size = thr->shadow_stack_pos - thr->shadow_stack;
471 uptr start = 0;
472 if (size + !!toppc > kStackTraceMax) {
473 start = size + !!toppc - kStackTraceMax;
474 size = kStackTraceMax - !!toppc;
475 }
476 stack->Init(&thr->shadow_stack[start], size, toppc);
477 ExtractTagFromStack(stack, tag);
478}
479
480#define GET_STACK_TRACE_FATAL(thr, pc) \
481 VarSizeStackTrace stack; \
482 ObtainCurrentStack(thr, pc, &stack); \
483 stack.ReverseOrder();
484
485void MapShadow(uptr addr, uptr size);
486void MapThreadTrace(uptr addr, uptr size, const char *name);
487void DontNeedShadowFor(uptr addr, uptr size);
488void UnmapShadow(ThreadState *thr, uptr addr, uptr size);
489void InitializeShadowMemory();
490void DontDumpShadow(uptr addr, uptr size);
491void InitializeInterceptors();
492void InitializeLibIgnore();
493void InitializeDynamicAnnotations();
494
495void ForkBefore(ThreadState *thr, uptr pc);
496void ForkParentAfter(ThreadState *thr, uptr pc);
497void ForkChildAfter(ThreadState *thr, uptr pc, bool start_thread);
498
499void ReportRace(ThreadState *thr, RawShadow *shadow_mem, Shadow cur, Shadow old,
500 AccessType typ);
501bool OutputReport(ThreadState *thr, ScopedReport &srep);
502bool IsFiredSuppression(Context *ctx, ReportType type, StackTrace trace);
503bool IsExpectedReport(uptr addr, uptr size);
504
505#if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 1
506# define DPrintf Printf
507#else
508# define DPrintf(...)
509#endif
510
511#if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 2
512# define DPrintf2 Printf
513#else
514# define DPrintf2(...)
515#endif
516
517StackID CurrentStackId(ThreadState *thr, uptr pc);
518ReportStack *SymbolizeStackId(StackID stack_id);
519void PrintCurrentStack(ThreadState *thr, uptr pc);
520void PrintCurrentStack(uptr pc, bool fast); // may uses libunwind
521MBlock *JavaHeapBlock(uptr addr, uptr *start);
522
523void Initialize(ThreadState *thr);
524void MaybeSpawnBackgroundThread();
525int Finalize(ThreadState *thr);
526
527void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write);
528void OnUserFree(ThreadState *thr, uptr pc, uptr p, bool write);
529
530void MemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size,
531 AccessType typ);
532void UnalignedMemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size,
533 AccessType typ);
534// This creates 2 non-inlined specialized versions of MemoryAccessRange.
535template <bool is_read>
536void MemoryAccessRangeT(ThreadState *thr, uptr pc, uptr addr, uptr size);
537
538ALWAYS_INLINE
539void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size,
540 bool is_write) {
541 if (size == 0)
542 return;
543 if (is_write)
544 MemoryAccessRangeT<false>(thr, pc, addr, size);
545 else
546 MemoryAccessRangeT<true>(thr, pc, addr, size);
547}
548
549void ShadowSet(RawShadow *p, RawShadow *end, RawShadow v);
550void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size);
551void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size);
552void MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size);
553void MemoryRangeImitateWriteOrResetRange(ThreadState *thr, uptr pc, uptr addr,
554 uptr size);
555
556void ThreadIgnoreBegin(ThreadState *thr, uptr pc);
557void ThreadIgnoreEnd(ThreadState *thr);
558void ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc);
559void ThreadIgnoreSyncEnd(ThreadState *thr);
560
561Tid ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached);
562void ThreadStart(ThreadState *thr, Tid tid, ThreadID os_id,
563 ThreadType thread_type);
564void ThreadFinish(ThreadState *thr);
565Tid ThreadConsumeTid(ThreadState *thr, uptr pc, uptr uid);
566void ThreadJoin(ThreadState *thr, uptr pc, Tid tid);
567void ThreadDetach(ThreadState *thr, uptr pc, Tid tid);
568void ThreadFinalize(ThreadState *thr);
569void ThreadSetName(ThreadState *thr, const char *name);
570int ThreadCount(ThreadState *thr);
571void ProcessPendingSignalsImpl(ThreadState *thr);
572void ThreadNotJoined(ThreadState *thr, uptr pc, Tid tid, uptr uid);
573
574Processor *ProcCreate();
575void ProcDestroy(Processor *proc);
576void ProcWire(Processor *proc, ThreadState *thr);
577void ProcUnwire(Processor *proc, ThreadState *thr);
578
579// Note: the parameter is called flagz, because flags is already taken
580// by the global function that returns flags.
581void MutexCreate(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
582void MutexDestroy(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
583void MutexPreLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
584void MutexPostLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0,
585 int rec = 1);
586int MutexUnlock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
587void MutexPreReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
588void MutexPostReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
589void MutexReadUnlock(ThreadState *thr, uptr pc, uptr addr);
590void MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr);
591void MutexRepair(ThreadState *thr, uptr pc, uptr addr); // call on EOWNERDEAD
592void MutexInvalidAccess(ThreadState *thr, uptr pc, uptr addr);
593
594void Acquire(ThreadState *thr, uptr pc, uptr addr);
595// AcquireGlobal synchronizes the current thread with all other threads.
596// In terms of happens-before relation, it draws a HB edge from all threads
597// (where they happen to execute right now) to the current thread. We use it to
598// handle Go finalizers. Namely, finalizer goroutine executes AcquireGlobal
599// right before executing finalizers. This provides a coarse, but simple
600// approximation of the actual required synchronization.
601void AcquireGlobal(ThreadState *thr);
602void Release(ThreadState *thr, uptr pc, uptr addr);
603void ReleaseStoreAcquire(ThreadState *thr, uptr pc, uptr addr);
604void ReleaseStore(ThreadState *thr, uptr pc, uptr addr);
605void AfterSleep(ThreadState *thr, uptr pc);
606void IncrementEpoch(ThreadState *thr);
607
608#if !SANITIZER_GO
609uptr ALWAYS_INLINE HeapEnd() {
610 return HeapMemEnd() + PrimaryAllocator::AdditionalSize();
611}
612#endif
613
614void SlotAttachAndLock(ThreadState *thr) SANITIZER_ACQUIRE(thr->slot->mtx);
615void SlotDetach(ThreadState *thr);
616void SlotLock(ThreadState *thr) SANITIZER_ACQUIRE(thr->slot->mtx);
617void SlotUnlock(ThreadState *thr) SANITIZER_RELEASE(thr->slot->mtx);
618void DoReset(ThreadState *thr, uptr epoch);
619void FlushShadowMemory();
620
621ThreadState *FiberCreate(ThreadState *thr, uptr pc, unsigned flags);
622void FiberDestroy(ThreadState *thr, uptr pc, ThreadState *fiber);
623void FiberSwitch(ThreadState *thr, uptr pc, ThreadState *fiber, unsigned flags);
624
625// These need to match __tsan_switch_to_fiber_* flags defined in
626// tsan_interface.h. See documentation there as well.
627enum FiberSwitchFlags {
628 FiberSwitchFlagNoSync = 1 << 0, // __tsan_switch_to_fiber_no_sync
629};
630
631class SlotLocker {
632 public:
633 ALWAYS_INLINE
634 SlotLocker(ThreadState *thr, bool recursive = false)
635 : thr_(thr), locked_(recursive ? thr->slot_locked : false) {
636#if !SANITIZER_GO
637 // We are in trouble if we are here with in_blocking_func set.
638 // If in_blocking_func is set, all signals will be delivered synchronously,
639 // which means we can't lock slots since the signal handler will try
640 // to lock it recursively and deadlock.
641 DCHECK(!atomic_load(&thr->in_blocking_func, memory_order_relaxed));
642#endif
643 if (!locked_)
644 SlotLock(thr: thr_);
645 }
646
647 ALWAYS_INLINE
648 ~SlotLocker() {
649 if (!locked_)
650 SlotUnlock(thr: thr_);
651 }
652
653 private:
654 ThreadState *thr_;
655 bool locked_;
656};
657
658class SlotUnlocker {
659 public:
660 SlotUnlocker(ThreadState *thr) : thr_(thr), locked_(thr->slot_locked) {
661 if (locked_)
662 SlotUnlock(thr: thr_);
663 }
664
665 ~SlotUnlocker() {
666 if (locked_)
667 SlotLock(thr: thr_);
668 }
669
670 private:
671 ThreadState *thr_;
672 bool locked_;
673};
674
675ALWAYS_INLINE void ProcessPendingSignals(ThreadState *thr) {
676 if (UNLIKELY(atomic_load_relaxed(&thr->pending_signals)))
677 ProcessPendingSignalsImpl(thr);
678}
679
680extern bool is_initialized;
681
682ALWAYS_INLINE
683void LazyInitialize(ThreadState *thr) {
684 // If we can use .preinit_array, assume that __tsan_init
685 // called from .preinit_array initializes runtime before
686 // any instrumented code except when tsan is used as a
687 // shared library.
688#if (!SANITIZER_CAN_USE_PREINIT_ARRAY || defined(SANITIZER_SHARED))
689 if (UNLIKELY(!is_initialized))
690 Initialize(thr);
691#endif
692}
693
694void TraceResetForTesting();
695void TraceSwitchPart(ThreadState *thr);
696void TraceSwitchPartImpl(ThreadState *thr);
697bool RestoreStack(EventType type, Sid sid, Epoch epoch, uptr addr, uptr size,
698 AccessType typ, Tid *ptid, VarSizeStackTrace *pstk,
699 MutexSet *pmset, uptr *ptag);
700
701template <typename EventT>
702ALWAYS_INLINE WARN_UNUSED_RESULT bool TraceAcquire(ThreadState *thr,
703 EventT **ev) {
704 // TraceSwitchPart accesses shadow_stack, but it's called infrequently,
705 // so we check it here proactively.
706 DCHECK(thr->shadow_stack);
707 Event *pos = reinterpret_cast<Event *>(atomic_load_relaxed(a: &thr->trace_pos));
708#if SANITIZER_DEBUG
709 // TraceSwitch acquires these mutexes,
710 // so we lock them here to detect deadlocks more reliably.
711 { Lock lock(&ctx->slot_mtx); }
712 { Lock lock(&thr->tctx->trace.mtx); }
713 TracePart *current = thr->tctx->trace.parts.Back();
714 if (current) {
715 DCHECK_GE(pos, &current->events[0]);
716 DCHECK_LE(pos, &current->events[TracePart::kSize]);
717 } else {
718 DCHECK_EQ(pos, nullptr);
719 }
720#endif
721 // TracePart is allocated with mmap and is at least 4K aligned.
722 // So the following check is a faster way to check for part end.
723 // It may have false positives in the middle of the trace,
724 // they are filtered out in TraceSwitch.
725 if (UNLIKELY(((uptr)(pos + 1) & TracePart::kAlignment) == 0))
726 return false;
727 *ev = reinterpret_cast<EventT *>(pos);
728 return true;
729}
730
731template <typename EventT>
732ALWAYS_INLINE void TraceRelease(ThreadState *thr, EventT *evp) {
733 DCHECK_LE(evp + 1, &thr->tctx->trace.parts.Back()->events[TracePart::kSize]);
734 atomic_store_relaxed(a: &thr->trace_pos, v: (uptr)(evp + 1));
735}
736
737template <typename EventT>
738void TraceEvent(ThreadState *thr, EventT ev) {
739 EventT *evp;
740 if (!TraceAcquire(thr, &evp)) {
741 TraceSwitchPart(thr);
742 UNUSED bool res = TraceAcquire(thr, &evp);
743 DCHECK(res);
744 }
745 *evp = ev;
746 TraceRelease(thr, evp);
747}
748
749ALWAYS_INLINE WARN_UNUSED_RESULT bool TryTraceFunc(ThreadState *thr,
750 uptr pc = 0) {
751 if (!kCollectHistory)
752 return true;
753 EventFunc *ev;
754 if (UNLIKELY(!TraceAcquire(thr, &ev)))
755 return false;
756 ev->is_access = 0;
757 ev->is_func = 1;
758 ev->pc = pc;
759 TraceRelease(thr, evp: ev);
760 return true;
761}
762
763WARN_UNUSED_RESULT
764bool TryTraceMemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size,
765 AccessType typ);
766WARN_UNUSED_RESULT
767bool TryTraceMemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size,
768 AccessType typ);
769void TraceMemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size,
770 AccessType typ);
771void TraceFunc(ThreadState *thr, uptr pc = 0);
772void TraceMutexLock(ThreadState *thr, EventType type, uptr pc, uptr addr,
773 StackID stk);
774void TraceMutexUnlock(ThreadState *thr, uptr addr);
775void TraceTime(ThreadState *thr);
776
777void TraceRestartFuncExit(ThreadState *thr);
778void TraceRestartFuncEntry(ThreadState *thr, uptr pc);
779
780void GrowShadowStack(ThreadState *thr);
781
782ALWAYS_INLINE
783void FuncEntry(ThreadState *thr, uptr pc) {
784 DPrintf2("#%d: FuncEntry %p\n", (int)thr->fast_state.sid(), (void *)pc);
785 if (UNLIKELY(!TryTraceFunc(thr, pc)))
786 return TraceRestartFuncEntry(thr, pc);
787 DCHECK_GE(thr->shadow_stack_pos, thr->shadow_stack);
788#if !SANITIZER_GO
789 DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);
790#else
791 if (thr->shadow_stack_pos == thr->shadow_stack_end)
792 GrowShadowStack(thr);
793#endif
794 thr->shadow_stack_pos[0] = pc;
795 thr->shadow_stack_pos++;
796}
797
798ALWAYS_INLINE
799void FuncExit(ThreadState *thr) {
800 DPrintf2("#%d: FuncExit\n", (int)thr->fast_state.sid());
801 if (UNLIKELY(!TryTraceFunc(thr, 0)))
802 return TraceRestartFuncExit(thr);
803 DCHECK_GT(thr->shadow_stack_pos, thr->shadow_stack);
804#if !SANITIZER_GO
805 DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);
806#endif
807 thr->shadow_stack_pos--;
808}
809
810#if !SANITIZER_GO
811extern void (*on_initialize)(void);
812extern int (*on_finalize)(int);
813#endif
814} // namespace __tsan
815
816#endif // TSAN_RTL_H
817