1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// Watchpoint-based host data race detector inspired by KCSAN. The host and GPU
11/// runtimes share a configurable packed-watchpoint implementation.
12///
13//===----------------------------------------------------------------------===//
14
15#include "csan.h"
16#include "csan_watch.h"
17
18#include "sanitizer_common/sanitizer_atomic.h"
19#include "sanitizer_common/sanitizer_common.h"
20#include "sanitizer_common/sanitizer_flag_parser.h"
21#include "sanitizer_common/sanitizer_flags.h"
22#include "sanitizer_common/sanitizer_internal_defs.h"
23#include "sanitizer_common/sanitizer_libc.h"
24#include "sanitizer_common/sanitizer_mutex.h"
25#include "sanitizer_common/sanitizer_stacktrace.h"
26#include "sanitizer_common/sanitizer_symbolizer.h"
27
28using namespace __sanitizer;
29
30#define INTERFACE extern "C" SANITIZER_INTERFACE_ATTRIBUTE
31
32static constexpr u32 kHostWatchpointEntries = 64;
33static constexpr u32 kHostCheckAdjacent = 1;
34static constexpr uptr kHostSlotRange = 4096;
35static constexpr uptr kHostMaxAccessSize =
36 kHostSlotRange * (1 + kHostCheckAdjacent);
37static_assert(__atomic_always_lock_free(sizeof(u64), nullptr),
38 "host watchpoints must be lock-free");
39
40using HostWatchpointTable =
41 __csan::WatchpointTable<kHostMaxAccessSize, kHostCheckAdjacent>;
42static u64 HostWatchpoints[kHostWatchpointEntries +
43 HostWatchpointTable::OverflowEntries];
44static_assert(HostWatchpointTable::AddressBits == 48,
45 "host watchpoints reserve 48 bits for addresses");
46
47static HostWatchpointTable GetHostWatchpoints() {
48 return HostWatchpointTable(HostWatchpoints);
49}
50
51static u32 HostWatchpointSlot(uptr Address) {
52 return Address / kHostSlotRange % kHostWatchpointEntries;
53}
54
55namespace __csan {
56
57Flags flags_data;
58
59static void Initialize();
60
61static atomic_uint64_t NumDataRaces;
62static THREADLOCAL u32 DisableCount;
63static THREADLOCAL s32 Skip;
64static THREADLOCAL u32 RandState;
65
66namespace {
67struct ScopedDisable {
68 ScopedDisable() { ++DisableCount; }
69 ~ScopedDisable() { --DisableCount; }
70};
71} // namespace
72
73void RecordDataRace() {
74 atomic_fetch_add(a: &NumDataRaces, v: 1, mo: memory_order_relaxed);
75}
76
77void Flags::SetDefaults() {
78#define CSAN_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue;
79#include "csan_flags.inc"
80#undef CSAN_FLAG
81}
82
83static void RegisterCsanFlags(FlagParser *Parser, Flags *F) {
84#define CSAN_FLAG(Type, Name, DefaultValue, Description) \
85 RegisterFlag(Parser, #Name, Description, &F->Name);
86#include "csan_flags.inc"
87#undef CSAN_FLAG
88}
89
90void InitializeFlags() {
91 SetCommonFlagsDefaults();
92 {
93 CommonFlags CF;
94 CF.CopyFrom(other: *common_flags());
95 CF.external_symbolizer_path = GetEnv(name: "CSAN_SYMBOLIZER_PATH");
96 OverrideCommonFlags(cf: CF);
97 }
98
99 flags()->SetDefaults();
100
101 FlagParser Parser;
102 RegisterCommonFlags(parser: &Parser);
103 RegisterCsanFlags(Parser: &Parser, F: flags());
104 Parser.ParseString(s: __csan_default_options());
105 Parser.ParseStringFromEnv(env_name: "CSAN_OPTIONS");
106 InitializeCommonFlags();
107 if (Verbosity())
108 ReportUnrecognizedFlags();
109 if (common_flags()->help)
110 Parser.PrintFlagDescriptions();
111}
112
113static u32 Random(u32 EpRo) {
114 if (EpRo <= 1)
115 return 0;
116 u32 State = RandState;
117 if (!State)
118 State = (u32)__builtin_readcyclecounter() | 1u;
119 State = 1664525u * State + 1013904223u;
120 RandState = State;
121 return State % EpRo;
122}
123
124static void ResetSkip() {
125 s32 Count = flags()->skip_watch;
126 if (Count < 0)
127 Count = 0;
128 if (Count)
129 Count -= (s32)Random(EpRo: (u32)Count);
130 Skip = Count;
131}
132
133static bool ShouldWatch(int Type) {
134 if (Type & CSAN_ACCESS_ATOMIC)
135 return false;
136 if (--Skip >= 0)
137 return false;
138 return true;
139}
140
141static void DelayAccess(int Type) {
142 s32 Delay = flags()->udelay;
143 if (Delay < 0)
144 Delay = 0;
145 if (Delay) {
146 u32 Skew = (Type & CSAN_ACCESS_COMPOUND) ? 1u : 0u;
147 u32 Span = (u32)Delay >> Skew;
148 if (!Span)
149 Span = (u32)Delay;
150 Delay -= (s32)Random(EpRo: Span);
151 }
152 if (Delay)
153 internal_usleep(useconds: (u64)Delay);
154}
155
156static u64 ReadRange(const volatile u8 *Bytes, uptr Size) {
157 u64 Sum = 0xcbf29ce484222325ull;
158 uptr I = 0;
159 for (; I < Size && ((uptr)(Bytes + I) & 7u); ++I)
160 Sum = (Sum ^ Bytes[I]) * 0x100000001b3ull;
161 for (; I + 8 <= Size; I += 8)
162 Sum = (Sum ^ *(const volatile u64 *)(Bytes + I)) * 0x100000001b3ull;
163 for (; I < Size; ++I)
164 Sum = (Sum ^ Bytes[I]) * 0x100000001b3ull;
165 return Sum;
166}
167
168static u64 ReadInstrumented(const volatile void *Ptr, uptr Size) {
169 switch (Size) {
170 case 1:
171 return *(const volatile u8 *)Ptr;
172 case 2:
173 return *(const volatile uu16 *)Ptr;
174 case 4:
175 return *(const volatile uu32 *)Ptr;
176 case 8:
177 return *(const volatile uu64 *)Ptr;
178 default:
179 return ReadRange(Bytes: (const volatile u8 *)Ptr, Size);
180 }
181}
182
183static AccessInfo MakeAccessInfo(const volatile void *Ptr, uptr Size, int Type,
184 uptr PC, uptr BP) {
185 AccessInfo AI;
186 AI.ptr = Ptr;
187 AI.size = Size;
188 AI.access_type = Type;
189 AI.tid = (u32)GetTid();
190 AI.pc = PC;
191 AI.bp = BP;
192 return AI;
193}
194
195NOINLINE static void FoundWatchpoint(const volatile void *Ptr, uptr Size,
196 int Type, uptr PC, uptr, u64 *WP,
197 u64 Encoded) {
198 ScopedDisable Disable;
199 GetHostWatchpoints().TryConsume(Watchpoint: WP, Encoded, PC,
200 IsWrite: (Type & CSAN_ACCESS_WRITE) != 0, Size);
201}
202
203NOINLINE static void SetupWatchpoint(const volatile void *Ptr, uptr Size,
204 int Type, uptr PC, uptr BP) {
205 ScopedDisable Disable;
206 ResetSkip();
207
208 if ((uptr)Ptr < GetPageSizeCached())
209 return;
210
211 u64 *WP = GetHostWatchpoints().Insert(Key: (uptr)Ptr, Size,
212 IsWrite: (Type & CSAN_ACCESS_WRITE) != 0,
213 Slot: HostWatchpointSlot(Address: (uptr)Ptr));
214 if (!WP)
215 return;
216
217 const u64 Old = ReadInstrumented(Ptr, Size);
218 DelayAccess(Type);
219 const u64 New = ReadInstrumented(Ptr, Size);
220
221 ValueChange VC = Old != New ? kValueChangeTrue : kValueChangeMaybe;
222
223 const AccessInfo AI = MakeAccessInfo(Ptr, Size, Type, PC, BP);
224 void *Peer;
225 int PeerAccess;
226 u32 PeerSize;
227 if (!GetHostWatchpoints().Consume(Watchpoint: WP, Peer, PeerAccess, PeerSize)) {
228 ReportKnownOrigin(AI, VC, PeerPC: (uptr)Peer, PeerAccess, PeerSize, Old, New);
229 } else if (VC == kValueChangeTrue) {
230 ReportUnknownOrigin(AI, Old, New);
231 }
232
233 GetHostWatchpoints().Remove(Watchpoint: WP);
234}
235
236ALWAYS_INLINE static void CheckAccess(const volatile void *Ptr, uptr Size,
237 int Type, uptr PC, uptr BP) {
238 if (UNLIKELY(!Ptr || !Size))
239 return;
240 if (Size > kHostMaxAccessSize)
241 Size = kHostMaxAccessSize;
242 if (UNLIKELY(uptr(Ptr) & ~HostWatchpointTable::AddressMask))
243 return;
244 Initialize();
245 if (UNLIKELY(DisableCount))
246 return;
247
248 u64 Encoded;
249 u64 *WP =
250 GetHostWatchpoints().Find(Key: (uptr)Ptr, Size, ExpectWrite: !(Type & CSAN_ACCESS_WRITE),
251 Slot: HostWatchpointSlot(Address: (uptr)Ptr), Encoded);
252 if (UNLIKELY(WP != nullptr))
253 FoundWatchpoint(Ptr, Size, Type, PC, BP, WP, Encoded);
254 else if (UNLIKELY(ShouldWatch(Type)))
255 SetupWatchpoint(Ptr, Size, Type, PC, BP);
256}
257
258static StaticSpinMutex InitMutex;
259static atomic_uint8_t Initialized;
260
261void Initialize() {
262 if (LIKELY(atomic_load(&Initialized, memory_order_acquire)))
263 return;
264 SpinMutexLock L(&InitMutex);
265 if (atomic_load(a: &Initialized, mo: memory_order_relaxed))
266 return;
267 SanitizerToolName = "ConcurrencySanitizer";
268 CacheBinaryName();
269 InitializeFlags();
270 atomic_store(a: &Initialized, v: 1, mo: memory_order_release);
271 Symbolizer::LateInitialize();
272}
273
274} // namespace __csan
275
276SANITIZER_INTERFACE_WEAK_DEF(const char *, __csan_default_options, void) {
277 return "";
278}
279
280INTERFACE u64 __csan_get_num_data_races() {
281 return atomic_load(a: &__csan::NumDataRaces, mo: memory_order_relaxed);
282}
283
284INTERFACE void __csan_init() { __csan::Initialize(); }
285
286INTERFACE void __csan_func_entry(void *) {}
287INTERFACE void __csan_func_exit() {}
288INTERFACE void __csan_ignore_thread_begin() { ++__csan::DisableCount; }
289INTERFACE void __csan_ignore_thread_end() {
290 if (__csan::DisableCount)
291 --__csan::DisableCount;
292}
293
294static int AccessFlags(int Flags, bool IsWrite) {
295 return Flags | (IsWrite ? CSAN_ACCESS_WRITE : 0);
296}
297
298#define CSAN_PROBE(name, N, IsWrite) \
299 INTERFACE void name(void *Addr, int Flags) { \
300 GET_CALLER_PC_BP; \
301 __csan::CheckAccess(Addr, N, AccessFlags(Flags, IsWrite), pc, bp); \
302 }
303
304#define CSAN_ACCESS(N) \
305 CSAN_PROBE(__csan_read##N, N, false) \
306 CSAN_PROBE(__csan_unaligned_read##N, N, false) \
307 CSAN_PROBE(__csan_write##N, N, true) \
308 CSAN_PROBE(__csan_unaligned_write##N, N, true) \
309 CSAN_PROBE(__csan_read_write##N, N, true) \
310 CSAN_PROBE(__csan_unaligned_read_write##N, N, true)
311
312CSAN_ACCESS(1)
313CSAN_ACCESS(2)
314CSAN_ACCESS(4)
315CSAN_ACCESS(8)
316CSAN_ACCESS(16)
317
318INTERFACE void __csan_read_range(void *Addr, uptr Size, int Flags) {
319 GET_CALLER_PC_BP;
320 __csan::CheckAccess(Ptr: Addr, Size, Type: AccessFlags(Flags, IsWrite: false), PC: pc, BP: bp);
321}
322
323INTERFACE void __csan_write_range(void *Addr, uptr Size, int Flags) {
324 GET_CALLER_PC_BP;
325 __csan::CheckAccess(Ptr: Addr, Size, Type: AccessFlags(Flags, IsWrite: true), PC: pc, BP: bp);
326}
327
328// TODO: Handle thread reordering checks like KCSAN.
329INTERFACE void __csan_atomic_thread_fence(int) {}
330INTERFACE void __csan_atomic_signal_fence(int) {}
331