1//===-- tsan_rtl.h ----------------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is a part of ThreadSanitizer (TSan), a race detector.
10//
11// Main internal TSan header file.
12//
13// Ground rules:
14// - C++ run-time should not be used (static CTORs, RTTI, exceptions, static
15// function-scope locals)
16// - All functions/classes/etc reside in namespace __tsan, except for those
17// declared in tsan_interface.h.
18// - Platform-specific files should be used instead of ifdefs (*).
19// - No system headers included in header files (*).
20// - Platform specific headres included only into platform-specific files (*).
21//
22// (*) Except when inlining is critical for performance.
23//===----------------------------------------------------------------------===//
24
25#ifndef TSAN_RTL_H
26#define TSAN_RTL_H
27
28#include "sanitizer_common/sanitizer_allocator.h"
29#include "sanitizer_common/sanitizer_allocator_internal.h"
30#include "sanitizer_common/sanitizer_asm.h"
31#include "sanitizer_common/sanitizer_common.h"
32#include "sanitizer_common/sanitizer_deadlock_detector_interface.h"
33#include "sanitizer_common/sanitizer_libignore.h"
34#include "sanitizer_common/sanitizer_suppressions.h"
35#include "sanitizer_common/sanitizer_thread_registry.h"
36#include "sanitizer_common/sanitizer_vector.h"
37#include "tsan_defs.h"
38#include "tsan_flags.h"
39#include "tsan_ignoreset.h"
40#include "tsan_ilist.h"
41#include "tsan_mman.h"
42#include "tsan_mutexset.h"
43#include "tsan_platform.h"
44#include "tsan_report.h"
45#include "tsan_shadow.h"
46#include "tsan_stack_trace.h"
47#include "tsan_sync.h"
48#include "tsan_trace.h"
49#include "tsan_vector_clock.h"
50
51#if SANITIZER_WORDSIZE != 64
52# error "ThreadSanitizer is supported only on 64-bit platforms"
53#endif
54
55namespace __tsan {
56
57extern bool ready_to_symbolize;
58
59#if !SANITIZER_GO
60struct MapUnmapCallback;
61# if defined(__mips64) || defined(__aarch64__) || defined(__loongarch__) || \
62 defined(__powerpc__) || SANITIZER_RISCV64
63
64struct AP32 {
65 static const uptr kSpaceBeg = 0;
66 static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE;
67 static const uptr kMetadataSize = 0;
68 typedef __sanitizer::CompactSizeClassMap SizeClassMap;
69 static const uptr kRegionSizeLog = 20;
70 using AddressSpaceView = LocalAddressSpaceView;
71 typedef __tsan::MapUnmapCallback MapUnmapCallback;
72 static const uptr kFlags = 0;
73};
74typedef SizeClassAllocator32<AP32> PrimaryAllocator;
75#else
76struct AP64 { // Allocator64 parameters. Deliberately using a short name.
77# if defined(__s390x__)
78 typedef MappingS390x Mapping;
79# else
80 typedef Mapping48AddressSpace Mapping;
81# endif
82 static const uptr kSpaceBeg = Mapping::kHeapMemBeg;
83 static const uptr kSpaceSize = Mapping::kHeapMemEnd - Mapping::kHeapMemBeg;
84 static const uptr kMetadataSize = 0;
85 typedef DefaultSizeClassMap SizeClassMap;
86 typedef __tsan::MapUnmapCallback MapUnmapCallback;
87 static const uptr kFlags = 0;
88 using AddressSpaceView = LocalAddressSpaceView;
89};
90typedef SizeClassAllocator64<AP64> PrimaryAllocator;
91#endif
92typedef CombinedAllocator<PrimaryAllocator> Allocator;
93typedef Allocator::AllocatorCache AllocatorCache;
94Allocator *allocator();
95#endif
96
97struct ThreadSignalContext;
98
99struct JmpBuf {
100 uptr sp;
101 int int_signal_send;
102 bool in_blocking_func;
103 uptr oldset_stack_size;
104 uptr in_signal_handler;
105 uptr *shadow_stack_pos;
106};
107
108// A Processor represents a physical thread, or a P for Go.
109// It is used to store internal resources like allocate cache, and does not
110// participate in race-detection logic (invisible to end user).
111// In C++ it is tied to an OS thread just like ThreadState, however ideally
112// it should be tied to a CPU (this way we will have fewer allocator caches).
113// In Go it is tied to a P, so there are significantly fewer Processor's than
114// ThreadState's (which are tied to Gs).
115// A ThreadState must be wired with a Processor to handle events.
116struct Processor {
117 ThreadState *thr; // currently wired thread, or nullptr
118#if !SANITIZER_GO
119 AllocatorCache alloc_cache;
120 InternalAllocatorCache internal_alloc_cache;
121#endif
122 DenseSlabAllocCache block_cache;
123 DenseSlabAllocCache sync_cache;
124 DDPhysicalThread *dd_pt;
125};
126
127#if !SANITIZER_GO
128// ScopedGlobalProcessor temporary setups a global processor for the current
129// thread, if it does not have one. Intended for interceptors that can run
130// at the very thread end, when we already destroyed the thread processor.
131struct ScopedGlobalProcessor {
132 ScopedGlobalProcessor();
133 ~ScopedGlobalProcessor();
134};
135#endif
136
137struct TidEpoch {
138 Tid tid;
139 Epoch epoch;
140};
141
142struct alignas(SANITIZER_CACHE_LINE_SIZE) TidSlot {
143 Mutex mtx;
144 Sid sid;
145 atomic_uint32_t raw_epoch;
146 ThreadState *thr;
147 Vector<TidEpoch> journal;
148 INode node;
149
150 Epoch epoch() const {
151 return static_cast<Epoch>(atomic_load(a: &raw_epoch, mo: memory_order_relaxed));
152 }
153
154 void SetEpoch(Epoch v) {
155 atomic_store(a: &raw_epoch, v: static_cast<u32>(v), mo: memory_order_relaxed);
156 }
157
158 TidSlot();
159};
160
161// This struct is stored in TLS.
162struct alignas(SANITIZER_CACHE_LINE_SIZE) ThreadState {
163 FastState fast_state;
164 int ignore_sync;
165#if !SANITIZER_GO
166 int ignore_interceptors;
167#endif
168 uptr *shadow_stack_pos;
169
170 // Current position in tctx->trace.Back()->events (Event*).
171 atomic_uintptr_t trace_pos;
172 // PC of the last memory access, used to compute PC deltas in the trace.
173 uptr trace_prev_pc;
174
175 // Technically `current` should be a separate THREADLOCAL variable;
176 // but it is placed here in order to share cache line with previous fields.
177 ThreadState* current;
178
179 atomic_sint32_t pending_signals;
180
181 VectorClock clock;
182
183 // This is a slow path flag. On fast path, fast_state.GetIgnoreBit() is read.
184 // We do not distinguish beteween ignoring reads and writes
185 // for better performance.
186 int ignore_reads_and_writes;
187 int suppress_reports;
188 // Go does not support ignores.
189#if !SANITIZER_GO
190 IgnoreSet mop_ignore_set;
191 IgnoreSet sync_ignore_set;
192#endif
193 uptr *shadow_stack;
194 uptr *shadow_stack_end;
195#if !SANITIZER_GO
196 Vector<JmpBuf> jmp_bufs;
197 int in_symbolizer;
198 atomic_uintptr_t in_blocking_func;
199 bool in_ignored_lib;
200 bool is_inited;
201#endif
202 MutexSet mset;
203 bool is_dead;
204 const Tid tid;
205 uptr stk_addr;
206 uptr stk_size;
207 uptr tls_addr;
208 uptr tls_size;
209 ThreadContext *tctx;
210
211 DDLogicalThread *dd_lt;
212
213 TidSlot *slot;
214 uptr slot_epoch;
215 bool slot_locked;
216
217 // Current wired Processor, or nullptr. Required to handle any events.
218 Processor *proc1;
219#if !SANITIZER_GO
220 Processor *proc() { return proc1; }
221#else
222 Processor *proc();
223#endif
224
225 atomic_uintptr_t in_signal_handler;
226 atomic_uintptr_t signal_ctx;
227
228#if !SANITIZER_GO
229 StackID last_sleep_stack_id;
230 VectorClock last_sleep_clock;
231#endif
232
233 // Set in regions of runtime that must be signal-safe and fork-safe.
234 // If set, malloc must not be called.
235 int nomalloc;
236
237 const ReportDesc *current_report;
238
239 explicit ThreadState(Tid tid);
240};
241
242#if !SANITIZER_GO
243#if SANITIZER_APPLE || SANITIZER_ANDROID
244ThreadState *cur_thread();
245void set_cur_thread(ThreadState *thr);
246void cur_thread_finalize();
247inline ThreadState *cur_thread_init() { return cur_thread(); }
248# else
249__attribute__((tls_model("initial-exec")))
250extern THREADLOCAL char cur_thread_placeholder[];
251inline ThreadState *cur_thread() {
252 return reinterpret_cast<ThreadState *>(cur_thread_placeholder)->current;
253}
254inline ThreadState *cur_thread_init() {
255 ThreadState *thr = reinterpret_cast<ThreadState *>(cur_thread_placeholder);
256 if (UNLIKELY(!thr->current))
257 thr->current = thr;
258 return thr->current;
259}
260inline void set_cur_thread(ThreadState *thr) {
261 reinterpret_cast<ThreadState *>(cur_thread_placeholder)->current = thr;
262}
263inline void cur_thread_finalize() { }
264# endif // SANITIZER_APPLE || SANITIZER_ANDROID
265#endif // SANITIZER_GO
266
267class ThreadContext final : public ThreadContextBase {
268 public:
269 explicit ThreadContext(Tid tid);
270 ~ThreadContext();
271 ThreadState *thr;
272 StackID creation_stack_id;
273 VectorClock *sync;
274 uptr sync_epoch;
275 Trace trace;
276
277 // Override superclass callbacks.
278 void OnDead() override;
279 void OnJoined(void *arg) override;
280 void OnFinished() override;
281 void OnStarted(void *arg) override;
282 void OnCreated(void *arg) override;
283 void OnReset() override;
284 void OnDetached(void *arg) override;
285};
286
287struct RacyStacks {
288 MD5Hash hash[2];
289 bool operator==(const RacyStacks &other) const;
290};
291
292struct RacyAddress {
293 uptr addr_min;
294 uptr addr_max;
295};
296
297struct FiredSuppression {
298 ReportType type;
299 uptr pc_or_addr;
300 Suppression *supp;
301};
302
303struct Context {
304 Context();
305
306 bool initialized;
307#if !SANITIZER_GO
308 bool after_multithreaded_fork;
309#endif
310
311 MetaMap metamap;
312
313 Mutex report_mtx;
314 int nreported;
315 atomic_uint64_t last_symbolize_time_ns;
316
317 void *background_thread;
318 atomic_uint32_t stop_background_thread;
319
320 ThreadRegistry thread_registry;
321
322 // This is used to prevent a very unlikely but very pathological behavior.
323 // Since memory access handling is not synchronized with DoReset,
324 // a thread running concurrently with DoReset can leave a bogus shadow value
325 // that will be later falsely detected as a race. For such false races
326 // RestoreStack will return false and we will not report it.
327 // However, consider that a thread leaves a whole lot of such bogus values
328 // and these values are later read by a whole lot of threads.
329 // This will cause massive amounts of ReportRace calls and lots of
330 // serialization. In very pathological cases the resulting slowdown
331 // can be >100x. This is very unlikely, but it was presumably observed
332 // in practice: https://github.com/google/sanitizers/issues/1552
333 // If this happens, previous access sid+epoch will be the same for all of
334 // these false races b/c if the thread will try to increment epoch, it will
335 // notice that DoReset has happened and will stop producing bogus shadow
336 // values. So, last_spurious_race is used to remember the last sid+epoch
337 // for which RestoreStack returned false. Then it is used to filter out
338 // races with the same sid+epoch very early and quickly.
339 // It is of course possible that multiple threads left multiple bogus shadow
340 // values and all of them are read by lots of threads at the same time.
341 // In such case last_spurious_race will only be able to deduplicate a few
342 // races from one thread, then few from another and so on. An alternative
343 // would be to hold an array of such sid+epoch, but we consider such scenario
344 // as even less likely.
345 // Note: this can lead to some rare false negatives as well:
346 // 1. When a legit access with the same sid+epoch participates in a race
347 // as the "previous" memory access, it will be wrongly filtered out.
348 // 2. When RestoreStack returns false for a legit memory access because it
349 // was already evicted from the thread trace, we will still remember it in
350 // last_spurious_race. Then if there is another racing memory access from
351 // the same thread that happened in the same epoch, but was stored in the
352 // next thread trace part (which is still preserved in the thread trace),
353 // we will also wrongly filter it out while RestoreStack would actually
354 // succeed for that second memory access.
355 RawShadow last_spurious_race;
356
357 Mutex racy_mtx;
358 Vector<RacyStacks> racy_stacks;
359 // Number of fired suppressions may be large enough.
360 Mutex fired_suppressions_mtx;
361 InternalMmapVector<FiredSuppression> fired_suppressions;
362 DDetector *dd;
363
364 Flags flags;
365 fd_t memprof_fd;
366
367 // The last slot index (kFreeSid) is used to denote freed memory.
368 TidSlot slots[kThreadSlotCount - 1];
369
370 // Protects global_epoch, slot_queue, trace_part_recycle.
371 Mutex slot_mtx;
372 uptr global_epoch; // guarded by slot_mtx and by all slot mutexes
373 bool resetting; // global reset is in progress
374 IList<TidSlot, &TidSlot::node> slot_queue SANITIZER_GUARDED_BY(slot_mtx);
375 IList<TraceHeader, &TraceHeader::global, TracePart> trace_part_recycle
376 SANITIZER_GUARDED_BY(slot_mtx);
377 uptr trace_part_total_allocated SANITIZER_GUARDED_BY(slot_mtx);
378 uptr trace_part_recycle_finished SANITIZER_GUARDED_BY(slot_mtx);
379 uptr trace_part_finished_excess SANITIZER_GUARDED_BY(slot_mtx);
380#if SANITIZER_GO
381 uptr mapped_shadow_begin;
382 uptr mapped_shadow_end;
383#endif
384};
385
386extern Context *ctx; // The one and the only global runtime context.
387
388ALWAYS_INLINE Flags *flags() {
389 return &ctx->flags;
390}
391
392struct ScopedIgnoreInterceptors {
393 ScopedIgnoreInterceptors() {
394#if !SANITIZER_GO
395 cur_thread()->ignore_interceptors++;
396#endif
397 }
398
399 ~ScopedIgnoreInterceptors() {
400#if !SANITIZER_GO
401 cur_thread()->ignore_interceptors--;
402#endif
403 }
404};
405
406const char *GetObjectTypeFromTag(uptr tag);
407const char *GetReportHeaderFromTag(uptr tag);
408uptr TagFromShadowStackFrame(uptr pc);
409
410class ScopedReportBase {
411 public:
412 void AddMemoryAccess(uptr addr, uptr external_tag, Shadow s, Tid tid,
413 StackTrace stack, const MutexSet *mset);
414 void AddStack(StackTrace stack, bool suppressable = false);
415 void AddThread(const ThreadContext *tctx, bool suppressable = false);
416 void AddThread(Tid tid, bool suppressable = false);
417 void AddUniqueTid(Tid unique_tid);
418 int AddMutex(uptr addr, StackID creation_stack_id);
419 void AddLocation(uptr addr, uptr size);
420 void AddSleep(StackID stack_id);
421 void SetCount(int count);
422 void SetSigNum(int sig);
423
424 const ReportDesc *GetReport() const;
425
426 protected:
427 ScopedReportBase(ReportType typ, uptr tag);
428 ~ScopedReportBase();
429
430 private:
431 ReportDesc *rep_;
432 // Symbolizer makes lots of intercepted calls. If we try to process them,
433 // at best it will cause deadlocks on internal mutexes.
434 ScopedIgnoreInterceptors ignore_interceptors_;
435
436 ScopedReportBase(const ScopedReportBase &) = delete;
437 void operator=(const ScopedReportBase &) = delete;
438};
439
440class ScopedReport : public ScopedReportBase {
441 public:
442 explicit ScopedReport(ReportType typ, uptr tag = kExternalTagNone);
443 ~ScopedReport();
444
445 private:
446 ScopedErrorReportLock lock_;
447};
448
449bool ShouldReport(ThreadState *thr, ReportType typ);
450ThreadContext *IsThreadStackOrTls(uptr addr, bool *is_stack);
451
452// The stack could look like:
453// <start> | <main> | <foo> | tag | <bar>
454// This will extract the tag and keep:
455// <start> | <main> | <foo> | <bar>
456template<typename StackTraceTy>
457void ExtractTagFromStack(StackTraceTy *stack, uptr *tag = nullptr) {
458 if (stack->size < 2) return;
459 uptr possible_tag_pc = stack->trace[stack->size - 2];
460 uptr possible_tag = TagFromShadowStackFrame(pc: possible_tag_pc);
461 if (possible_tag == kExternalTagNone) return;
462 stack->trace_buffer[stack->size - 2] = stack->trace_buffer[stack->size - 1];
463 stack->size -= 1;
464 if (tag) *tag = possible_tag;
465}
466
467template<typename StackTraceTy>
468void ObtainCurrentStack(ThreadState *thr, uptr toppc, StackTraceTy *stack,
469 uptr *tag = nullptr) {
470 uptr size = thr->shadow_stack_pos - thr->shadow_stack;
471 uptr start = 0;
472 if (size + !!toppc > kStackTraceMax) {
473 start = size + !!toppc - kStackTraceMax;
474 size = kStackTraceMax - !!toppc;
475 }
476 stack->Init(&thr->shadow_stack[start], size, toppc);
477 ExtractTagFromStack(stack, tag);
478}
479
480#define GET_STACK_TRACE_FATAL(thr, pc) \
481 VarSizeStackTrace stack; \
482 ObtainCurrentStack(thr, pc, &stack); \
483 stack.ReverseOrder();
484
485void MapShadow(uptr addr, uptr size);
486void MapThreadTrace(uptr addr, uptr size, const char *name);
487void DontNeedShadowFor(uptr addr, uptr size);
488void UnmapShadow(ThreadState *thr, uptr addr, uptr size);
489void InitializeShadowMemory();
490void DontDumpShadow(uptr addr, uptr size);
491void InitializeInterceptors();
492void InitializeLibIgnore();
493void InitializeDynamicAnnotations();
494
495void ForkBefore(ThreadState *thr, uptr pc);
496void ForkParentAfter(ThreadState *thr, uptr pc);
497void ForkChildAfter(ThreadState *thr, uptr pc, bool start_thread);
498
499void ReportRace(ThreadState *thr, RawShadow *shadow_mem, Shadow cur, Shadow old,
500 AccessType typ);
501bool OutputReport(ThreadState *thr, const ScopedReport &srep);
502bool IsFiredSuppression(Context *ctx, ReportType type, StackTrace trace);
503bool IsExpectedReport(uptr addr, uptr size);
504
505#if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 1
506# define DPrintf Printf
507#else
508# define DPrintf(...)
509#endif
510
511#if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 2
512# define DPrintf2 Printf
513#else
514# define DPrintf2(...)
515#endif
516
517StackID CurrentStackId(ThreadState *thr, uptr pc);
518ReportStack *SymbolizeStackId(StackID stack_id);
519void PrintCurrentStack(ThreadState *thr, uptr pc);
520void PrintCurrentStack(uptr pc, bool fast); // may uses libunwind
521MBlock *JavaHeapBlock(uptr addr, uptr *start);
522
523void Initialize(ThreadState *thr);
524void MaybeSpawnBackgroundThread();
525int Finalize(ThreadState *thr);
526
527void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write);
528void OnUserFree(ThreadState *thr, uptr pc, uptr p, bool write);
529
530void MemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size,
531 AccessType typ);
532void UnalignedMemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size,
533 AccessType typ);
534// This creates 2 non-inlined specialized versions of MemoryAccessRange.
535template <bool is_read>
536void MemoryAccessRangeT(ThreadState *thr, uptr pc, uptr addr, uptr size);
537
538ALWAYS_INLINE
539void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size,
540 bool is_write) {
541 if (size == 0)
542 return;
543 if (is_write)
544 MemoryAccessRangeT<false>(thr, pc, addr, size);
545 else
546 MemoryAccessRangeT<true>(thr, pc, addr, size);
547}
548
549void ShadowSet(RawShadow *p, RawShadow *end, RawShadow v);
550void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size);
551void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size);
552void MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size);
553void MemoryRangeImitateWriteOrResetRange(ThreadState *thr, uptr pc, uptr addr,
554 uptr size);
555
556void ThreadIgnoreBegin(ThreadState *thr, uptr pc);
557void ThreadIgnoreEnd(ThreadState *thr);
558void ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc);
559void ThreadIgnoreSyncEnd(ThreadState *thr);
560
561Tid ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached);
562void ThreadStart(ThreadState *thr, Tid tid, tid_t os_id,
563 ThreadType thread_type);
564void ThreadFinish(ThreadState *thr);
565Tid ThreadConsumeTid(ThreadState *thr, uptr pc, uptr uid);
566void ThreadJoin(ThreadState *thr, uptr pc, Tid tid);
567void ThreadDetach(ThreadState *thr, uptr pc, Tid tid);
568void ThreadFinalize(ThreadState *thr);
569void ThreadSetName(ThreadState *thr, const char *name);
570int ThreadCount(ThreadState *thr);
571void ProcessPendingSignalsImpl(ThreadState *thr);
572void ThreadNotJoined(ThreadState *thr, uptr pc, Tid tid, uptr uid);
573
574Processor *ProcCreate();
575void ProcDestroy(Processor *proc);
576void ProcWire(Processor *proc, ThreadState *thr);
577void ProcUnwire(Processor *proc, ThreadState *thr);
578
579// Note: the parameter is called flagz, because flags is already taken
580// by the global function that returns flags.
581void MutexCreate(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
582void MutexDestroy(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
583void MutexPreLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
584void MutexPostLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0,
585 int rec = 1);
586int MutexUnlock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
587void MutexPreReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
588void MutexPostReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0);
589void MutexReadUnlock(ThreadState *thr, uptr pc, uptr addr);
590void MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr);
591void MutexRepair(ThreadState *thr, uptr pc, uptr addr); // call on EOWNERDEAD
592void MutexInvalidAccess(ThreadState *thr, uptr pc, uptr addr);
593
594void Acquire(ThreadState *thr, uptr pc, uptr addr);
595// AcquireGlobal synchronizes the current thread with all other threads.
596// In terms of happens-before relation, it draws a HB edge from all threads
597// (where they happen to execute right now) to the current thread. We use it to
598// handle Go finalizers. Namely, finalizer goroutine executes AcquireGlobal
599// right before executing finalizers. This provides a coarse, but simple
600// approximation of the actual required synchronization.
601void AcquireGlobal(ThreadState *thr);
602void Release(ThreadState *thr, uptr pc, uptr addr);
603void ReleaseStoreAcquire(ThreadState *thr, uptr pc, uptr addr);
604void ReleaseStore(ThreadState *thr, uptr pc, uptr addr);
605void AfterSleep(ThreadState *thr, uptr pc);
606void IncrementEpoch(ThreadState *thr);
607
608#if !SANITIZER_GO
609uptr ALWAYS_INLINE HeapEnd() {
610 return HeapMemEnd() + PrimaryAllocator::AdditionalSize();
611}
612#endif
613
614void SlotAttachAndLock(ThreadState *thr) SANITIZER_ACQUIRE(thr->slot->mtx);
615void SlotDetach(ThreadState *thr);
616void SlotLock(ThreadState *thr) SANITIZER_ACQUIRE(thr->slot->mtx);
617void SlotUnlock(ThreadState *thr) SANITIZER_RELEASE(thr->slot->mtx);
618void DoReset(ThreadState *thr, uptr epoch);
619void FlushShadowMemory();
620
621ThreadState *FiberCreate(ThreadState *thr, uptr pc, unsigned flags);
622void FiberDestroy(ThreadState *thr, uptr pc, ThreadState *fiber);
623void FiberSwitch(ThreadState *thr, uptr pc, ThreadState *fiber, unsigned flags);
624
625// These need to match __tsan_switch_to_fiber_* flags defined in
626// tsan_interface.h. See documentation there as well.
627enum FiberSwitchFlags {
628 FiberSwitchFlagNoSync = 1 << 0, // __tsan_switch_to_fiber_no_sync
629};
630
631class SlotLocker {
632 public:
633 ALWAYS_INLINE
634 SlotLocker(ThreadState *thr, bool recursive = false)
635 : thr_(thr), locked_(recursive ? thr->slot_locked : false) {
636#if !SANITIZER_GO
637 // We are in trouble if we are here with in_blocking_func set.
638 // If in_blocking_func is set, all signals will be delivered synchronously,
639 // which means we can't lock slots since the signal handler will try
640 // to lock it recursively and deadlock.
641 DCHECK(!atomic_load(&thr->in_blocking_func, memory_order_relaxed));
642#endif
643 if (!locked_)
644 SlotLock(thr: thr_);
645 }
646
647 ALWAYS_INLINE
648 ~SlotLocker() {
649 if (!locked_)
650 SlotUnlock(thr: thr_);
651 }
652
653 private:
654 ThreadState *thr_;
655 bool locked_;
656};
657
658class SlotUnlocker {
659 public:
660 SlotUnlocker(ThreadState *thr) : thr_(thr), locked_(thr->slot_locked) {
661 if (locked_)
662 SlotUnlock(thr: thr_);
663 }
664
665 ~SlotUnlocker() {
666 if (locked_)
667 SlotLock(thr: thr_);
668 }
669
670 private:
671 ThreadState *thr_;
672 bool locked_;
673};
674
675ALWAYS_INLINE void ProcessPendingSignals(ThreadState *thr) {
676 if (UNLIKELY(atomic_load_relaxed(&thr->pending_signals)))
677 ProcessPendingSignalsImpl(thr);
678}
679
680extern bool is_initialized;
681
682ALWAYS_INLINE
683void LazyInitialize(ThreadState *thr) {
684 // If we can use .preinit_array, assume that __tsan_init
685 // called from .preinit_array initializes runtime before
686 // any instrumented code except when tsan is used as a
687 // shared library.
688#if (!SANITIZER_CAN_USE_PREINIT_ARRAY || defined(SANITIZER_SHARED))
689 if (UNLIKELY(!is_initialized))
690 Initialize(thr);
691#endif
692}
693
694void TraceResetForTesting();
695void TraceSwitchPart(ThreadState *thr);
696void TraceSwitchPartImpl(ThreadState *thr);
697bool RestoreStack(EventType type, Sid sid, Epoch epoch, uptr addr, uptr size,
698 AccessType typ, Tid *ptid, VarSizeStackTrace *pstk,
699 MutexSet *pmset, uptr *ptag);
700
701template <typename EventT>
702ALWAYS_INLINE WARN_UNUSED_RESULT bool TraceAcquire(ThreadState *thr,
703 EventT **ev) {
704 // TraceSwitchPart accesses shadow_stack, but it's called infrequently,
705 // so we check it here proactively.
706 DCHECK(thr->shadow_stack);
707 Event *pos = reinterpret_cast<Event *>(atomic_load_relaxed(a: &thr->trace_pos));
708#if SANITIZER_DEBUG
709 // TraceSwitch acquires these mutexes,
710 // so we lock them here to detect deadlocks more reliably.
711 { Lock lock(&ctx->slot_mtx); }
712 { Lock lock(&thr->tctx->trace.mtx); }
713 TracePart *current = thr->tctx->trace.parts.Back();
714 if (current) {
715 DCHECK_GE(pos, &current->events[0]);
716 DCHECK_LE(pos, &current->events[TracePart::kSize]);
717 } else {
718 DCHECK_EQ(pos, nullptr);
719 }
720#endif
721 // TracePart is allocated with mmap and is at least 4K aligned.
722 // So the following check is a faster way to check for part end.
723 // It may have false positives in the middle of the trace,
724 // they are filtered out in TraceSwitch.
725 if (UNLIKELY(((uptr)(pos + 1) & TracePart::kAlignment) == 0))
726 return false;
727 *ev = reinterpret_cast<EventT *>(pos);
728 return true;
729}
730
731template <typename EventT>
732ALWAYS_INLINE void TraceRelease(ThreadState *thr, EventT *evp) {
733 DCHECK_LE(evp + 1, &thr->tctx->trace.parts.Back()->events[TracePart::kSize]);
734 atomic_store_relaxed(a: &thr->trace_pos, v: (uptr)(evp + 1));
735}
736
737template <typename EventT>
738void TraceEvent(ThreadState *thr, EventT ev) {
739 EventT *evp;
740 if (!TraceAcquire(thr, &evp)) {
741 TraceSwitchPart(thr);
742 UNUSED bool res = TraceAcquire(thr, &evp);
743 DCHECK(res);
744 }
745 *evp = ev;
746 TraceRelease(thr, evp);
747}
748
749ALWAYS_INLINE WARN_UNUSED_RESULT bool TryTraceFunc(ThreadState *thr,
750 uptr pc = 0) {
751 if (!kCollectHistory)
752 return true;
753 EventFunc *ev;
754 if (UNLIKELY(!TraceAcquire(thr, &ev)))
755 return false;
756 ev->is_access = 0;
757 ev->is_func = 1;
758 ev->pc = pc;
759 TraceRelease(thr, evp: ev);
760 return true;
761}
762
763WARN_UNUSED_RESULT
764bool TryTraceMemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size,
765 AccessType typ);
766WARN_UNUSED_RESULT
767bool TryTraceMemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size,
768 AccessType typ);
769void TraceMemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size,
770 AccessType typ);
771void TraceFunc(ThreadState *thr, uptr pc = 0);
772void TraceMutexLock(ThreadState *thr, EventType type, uptr pc, uptr addr,
773 StackID stk);
774void TraceMutexUnlock(ThreadState *thr, uptr addr);
775void TraceTime(ThreadState *thr);
776
777void TraceRestartFuncExit(ThreadState *thr);
778void TraceRestartFuncEntry(ThreadState *thr, uptr pc);
779
780void GrowShadowStack(ThreadState *thr);
781
782ALWAYS_INLINE
783void FuncEntry(ThreadState *thr, uptr pc) {
784 DPrintf2("#%d: FuncEntry %p\n", (int)thr->fast_state.sid(), (void *)pc);
785 if (UNLIKELY(!TryTraceFunc(thr, pc)))
786 return TraceRestartFuncEntry(thr, pc);
787 DCHECK_GE(thr->shadow_stack_pos, thr->shadow_stack);
788#if !SANITIZER_GO
789 DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);
790#else
791 if (thr->shadow_stack_pos == thr->shadow_stack_end)
792 GrowShadowStack(thr);
793#endif
794 thr->shadow_stack_pos[0] = pc;
795 thr->shadow_stack_pos++;
796}
797
798ALWAYS_INLINE
799void FuncExit(ThreadState *thr) {
800 DPrintf2("#%d: FuncExit\n", (int)thr->fast_state.sid());
801 if (UNLIKELY(!TryTraceFunc(thr, 0)))
802 return TraceRestartFuncExit(thr);
803 DCHECK_GT(thr->shadow_stack_pos, thr->shadow_stack);
804#if !SANITIZER_GO
805 DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);
806#endif
807 thr->shadow_stack_pos--;
808}
809
810#if !SANITIZER_GO
811extern void (*on_initialize)(void);
812extern int (*on_finalize)(int);
813#endif
814} // namespace __tsan
815
816#endif // TSAN_RTL_H
817