| 1 | //===----------------------------------------------------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | /// |
| 9 | /// \file |
| 10 | /// Watchpoint-based host data race detector inspired by KCSAN. The host and GPU |
| 11 | /// runtimes share a configurable packed-watchpoint implementation. |
| 12 | /// |
| 13 | //===----------------------------------------------------------------------===// |
| 14 | |
| 15 | #include "csan.h" |
| 16 | #include "csan_watch.h" |
| 17 | |
| 18 | #include "sanitizer_common/sanitizer_atomic.h" |
| 19 | #include "sanitizer_common/sanitizer_common.h" |
| 20 | #include "sanitizer_common/sanitizer_flag_parser.h" |
| 21 | #include "sanitizer_common/sanitizer_flags.h" |
| 22 | #include "sanitizer_common/sanitizer_internal_defs.h" |
| 23 | #include "sanitizer_common/sanitizer_libc.h" |
| 24 | #include "sanitizer_common/sanitizer_mutex.h" |
| 25 | #include "sanitizer_common/sanitizer_stacktrace.h" |
| 26 | #include "sanitizer_common/sanitizer_symbolizer.h" |
| 27 | |
| 28 | using namespace __sanitizer; |
| 29 | |
| 30 | #define INTERFACE extern "C" SANITIZER_INTERFACE_ATTRIBUTE |
| 31 | |
| 32 | static constexpr u32 kHostWatchpointEntries = 64; |
| 33 | static constexpr u32 kHostCheckAdjacent = 1; |
| 34 | static constexpr uptr kHostSlotRange = 4096; |
| 35 | static constexpr uptr kHostMaxAccessSize = |
| 36 | kHostSlotRange * (1 + kHostCheckAdjacent); |
| 37 | static_assert(__atomic_always_lock_free(sizeof(u64), nullptr), |
| 38 | "host watchpoints must be lock-free" ); |
| 39 | |
| 40 | using HostWatchpointTable = |
| 41 | __csan::WatchpointTable<kHostMaxAccessSize, kHostCheckAdjacent>; |
| 42 | static u64 HostWatchpoints[kHostWatchpointEntries + |
| 43 | HostWatchpointTable::OverflowEntries]; |
| 44 | static_assert(HostWatchpointTable::AddressBits == 48, |
| 45 | "host watchpoints reserve 48 bits for addresses" ); |
| 46 | |
| 47 | static HostWatchpointTable GetHostWatchpoints() { |
| 48 | return HostWatchpointTable(HostWatchpoints); |
| 49 | } |
| 50 | |
| 51 | static u32 HostWatchpointSlot(uptr Address) { |
| 52 | return Address / kHostSlotRange % kHostWatchpointEntries; |
| 53 | } |
| 54 | |
| 55 | namespace __csan { |
| 56 | |
| 57 | Flags flags_data; |
| 58 | |
| 59 | static void Initialize(); |
| 60 | |
| 61 | static atomic_uint64_t NumDataRaces; |
| 62 | static THREADLOCAL u32 DisableCount; |
| 63 | static THREADLOCAL s32 Skip; |
| 64 | static THREADLOCAL u32 RandState; |
| 65 | |
| 66 | namespace { |
| 67 | struct ScopedDisable { |
| 68 | ScopedDisable() { ++DisableCount; } |
| 69 | ~ScopedDisable() { --DisableCount; } |
| 70 | }; |
| 71 | } // namespace |
| 72 | |
| 73 | void RecordDataRace() { |
| 74 | atomic_fetch_add(a: &NumDataRaces, v: 1, mo: memory_order_relaxed); |
| 75 | } |
| 76 | |
| 77 | void Flags::SetDefaults() { |
| 78 | #define CSAN_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue; |
| 79 | #include "csan_flags.inc" |
| 80 | #undef CSAN_FLAG |
| 81 | } |
| 82 | |
| 83 | static void RegisterCsanFlags(FlagParser *Parser, Flags *F) { |
| 84 | #define CSAN_FLAG(Type, Name, DefaultValue, Description) \ |
| 85 | RegisterFlag(Parser, #Name, Description, &F->Name); |
| 86 | #include "csan_flags.inc" |
| 87 | #undef CSAN_FLAG |
| 88 | } |
| 89 | |
| 90 | void InitializeFlags() { |
| 91 | SetCommonFlagsDefaults(); |
| 92 | { |
| 93 | CommonFlags CF; |
| 94 | CF.CopyFrom(other: *common_flags()); |
| 95 | CF.external_symbolizer_path = GetEnv(name: "CSAN_SYMBOLIZER_PATH" ); |
| 96 | OverrideCommonFlags(cf: CF); |
| 97 | } |
| 98 | |
| 99 | flags()->SetDefaults(); |
| 100 | |
| 101 | FlagParser Parser; |
| 102 | RegisterCommonFlags(parser: &Parser); |
| 103 | RegisterCsanFlags(Parser: &Parser, F: flags()); |
| 104 | Parser.ParseString(s: __csan_default_options()); |
| 105 | Parser.ParseStringFromEnv(env_name: "CSAN_OPTIONS" ); |
| 106 | InitializeCommonFlags(); |
| 107 | if (Verbosity()) |
| 108 | ReportUnrecognizedFlags(); |
| 109 | if (common_flags()->help) |
| 110 | Parser.PrintFlagDescriptions(); |
| 111 | } |
| 112 | |
| 113 | static u32 Random(u32 EpRo) { |
| 114 | if (EpRo <= 1) |
| 115 | return 0; |
| 116 | u32 State = RandState; |
| 117 | if (!State) |
| 118 | State = (u32)__builtin_readcyclecounter() | 1u; |
| 119 | State = 1664525u * State + 1013904223u; |
| 120 | RandState = State; |
| 121 | return State % EpRo; |
| 122 | } |
| 123 | |
| 124 | static void ResetSkip() { |
| 125 | s32 Count = flags()->skip_watch; |
| 126 | if (Count < 0) |
| 127 | Count = 0; |
| 128 | if (Count) |
| 129 | Count -= (s32)Random(EpRo: (u32)Count); |
| 130 | Skip = Count; |
| 131 | } |
| 132 | |
| 133 | static bool ShouldWatch(int Type) { |
| 134 | if (Type & CSAN_ACCESS_ATOMIC) |
| 135 | return false; |
| 136 | if (--Skip >= 0) |
| 137 | return false; |
| 138 | return true; |
| 139 | } |
| 140 | |
| 141 | static void DelayAccess(int Type) { |
| 142 | s32 Delay = flags()->udelay; |
| 143 | if (Delay < 0) |
| 144 | Delay = 0; |
| 145 | if (Delay) { |
| 146 | u32 Skew = (Type & CSAN_ACCESS_COMPOUND) ? 1u : 0u; |
| 147 | u32 Span = (u32)Delay >> Skew; |
| 148 | if (!Span) |
| 149 | Span = (u32)Delay; |
| 150 | Delay -= (s32)Random(EpRo: Span); |
| 151 | } |
| 152 | if (Delay) |
| 153 | internal_usleep(useconds: (u64)Delay); |
| 154 | } |
| 155 | |
| 156 | static u64 ReadRange(const volatile u8 *Bytes, uptr Size) { |
| 157 | u64 Sum = 0xcbf29ce484222325ull; |
| 158 | uptr I = 0; |
| 159 | for (; I < Size && ((uptr)(Bytes + I) & 7u); ++I) |
| 160 | Sum = (Sum ^ Bytes[I]) * 0x100000001b3ull; |
| 161 | for (; I + 8 <= Size; I += 8) |
| 162 | Sum = (Sum ^ *(const volatile u64 *)(Bytes + I)) * 0x100000001b3ull; |
| 163 | for (; I < Size; ++I) |
| 164 | Sum = (Sum ^ Bytes[I]) * 0x100000001b3ull; |
| 165 | return Sum; |
| 166 | } |
| 167 | |
| 168 | static u64 ReadInstrumented(const volatile void *Ptr, uptr Size) { |
| 169 | switch (Size) { |
| 170 | case 1: |
| 171 | return *(const volatile u8 *)Ptr; |
| 172 | case 2: |
| 173 | return *(const volatile uu16 *)Ptr; |
| 174 | case 4: |
| 175 | return *(const volatile uu32 *)Ptr; |
| 176 | case 8: |
| 177 | return *(const volatile uu64 *)Ptr; |
| 178 | default: |
| 179 | return ReadRange(Bytes: (const volatile u8 *)Ptr, Size); |
| 180 | } |
| 181 | } |
| 182 | |
| 183 | static AccessInfo MakeAccessInfo(const volatile void *Ptr, uptr Size, int Type, |
| 184 | uptr PC, uptr BP) { |
| 185 | AccessInfo AI; |
| 186 | AI.ptr = Ptr; |
| 187 | AI.size = Size; |
| 188 | AI.access_type = Type; |
| 189 | AI.tid = (u32)GetTid(); |
| 190 | AI.pc = PC; |
| 191 | AI.bp = BP; |
| 192 | return AI; |
| 193 | } |
| 194 | |
| 195 | NOINLINE static void FoundWatchpoint(const volatile void *Ptr, uptr Size, |
| 196 | int Type, uptr PC, uptr, u64 *WP, |
| 197 | u64 Encoded) { |
| 198 | ScopedDisable Disable; |
| 199 | GetHostWatchpoints().TryConsume(Watchpoint: WP, Encoded, PC, |
| 200 | IsWrite: (Type & CSAN_ACCESS_WRITE) != 0, Size); |
| 201 | } |
| 202 | |
| 203 | NOINLINE static void SetupWatchpoint(const volatile void *Ptr, uptr Size, |
| 204 | int Type, uptr PC, uptr BP) { |
| 205 | ScopedDisable Disable; |
| 206 | ResetSkip(); |
| 207 | |
| 208 | if ((uptr)Ptr < GetPageSizeCached()) |
| 209 | return; |
| 210 | |
| 211 | u64 *WP = GetHostWatchpoints().Insert(Key: (uptr)Ptr, Size, |
| 212 | IsWrite: (Type & CSAN_ACCESS_WRITE) != 0, |
| 213 | Slot: HostWatchpointSlot(Address: (uptr)Ptr)); |
| 214 | if (!WP) |
| 215 | return; |
| 216 | |
| 217 | const u64 Old = ReadInstrumented(Ptr, Size); |
| 218 | DelayAccess(Type); |
| 219 | const u64 New = ReadInstrumented(Ptr, Size); |
| 220 | |
| 221 | ValueChange VC = Old != New ? kValueChangeTrue : kValueChangeMaybe; |
| 222 | |
| 223 | const AccessInfo AI = MakeAccessInfo(Ptr, Size, Type, PC, BP); |
| 224 | void *Peer; |
| 225 | int PeerAccess; |
| 226 | u32 PeerSize; |
| 227 | if (!GetHostWatchpoints().Consume(Watchpoint: WP, Peer, PeerAccess, PeerSize)) { |
| 228 | ReportKnownOrigin(AI, VC, PeerPC: (uptr)Peer, PeerAccess, PeerSize, Old, New); |
| 229 | } else if (VC == kValueChangeTrue) { |
| 230 | ReportUnknownOrigin(AI, Old, New); |
| 231 | } |
| 232 | |
| 233 | GetHostWatchpoints().Remove(Watchpoint: WP); |
| 234 | } |
| 235 | |
| 236 | ALWAYS_INLINE static void CheckAccess(const volatile void *Ptr, uptr Size, |
| 237 | int Type, uptr PC, uptr BP) { |
| 238 | if (UNLIKELY(!Ptr || !Size)) |
| 239 | return; |
| 240 | if (Size > kHostMaxAccessSize) |
| 241 | Size = kHostMaxAccessSize; |
| 242 | if (UNLIKELY(uptr(Ptr) & ~HostWatchpointTable::AddressMask)) |
| 243 | return; |
| 244 | Initialize(); |
| 245 | if (UNLIKELY(DisableCount)) |
| 246 | return; |
| 247 | |
| 248 | u64 Encoded; |
| 249 | u64 *WP = |
| 250 | GetHostWatchpoints().Find(Key: (uptr)Ptr, Size, ExpectWrite: !(Type & CSAN_ACCESS_WRITE), |
| 251 | Slot: HostWatchpointSlot(Address: (uptr)Ptr), Encoded); |
| 252 | if (UNLIKELY(WP != nullptr)) |
| 253 | FoundWatchpoint(Ptr, Size, Type, PC, BP, WP, Encoded); |
| 254 | else if (UNLIKELY(ShouldWatch(Type))) |
| 255 | SetupWatchpoint(Ptr, Size, Type, PC, BP); |
| 256 | } |
| 257 | |
| 258 | static StaticSpinMutex InitMutex; |
| 259 | static atomic_uint8_t Initialized; |
| 260 | |
| 261 | void Initialize() { |
| 262 | if (LIKELY(atomic_load(&Initialized, memory_order_acquire))) |
| 263 | return; |
| 264 | SpinMutexLock L(&InitMutex); |
| 265 | if (atomic_load(a: &Initialized, mo: memory_order_relaxed)) |
| 266 | return; |
| 267 | SanitizerToolName = "ConcurrencySanitizer" ; |
| 268 | CacheBinaryName(); |
| 269 | InitializeFlags(); |
| 270 | atomic_store(a: &Initialized, v: 1, mo: memory_order_release); |
| 271 | Symbolizer::LateInitialize(); |
| 272 | } |
| 273 | |
| 274 | } // namespace __csan |
| 275 | |
| 276 | SANITIZER_INTERFACE_WEAK_DEF(const char *, __csan_default_options, void) { |
| 277 | return "" ; |
| 278 | } |
| 279 | |
| 280 | INTERFACE u64 __csan_get_num_data_races() { |
| 281 | return atomic_load(a: &__csan::NumDataRaces, mo: memory_order_relaxed); |
| 282 | } |
| 283 | |
| 284 | INTERFACE void __csan_init() { __csan::Initialize(); } |
| 285 | |
| 286 | INTERFACE void __csan_func_entry(void *) {} |
| 287 | INTERFACE void __csan_func_exit() {} |
| 288 | INTERFACE void __csan_ignore_thread_begin() { ++__csan::DisableCount; } |
| 289 | INTERFACE void __csan_ignore_thread_end() { |
| 290 | if (__csan::DisableCount) |
| 291 | --__csan::DisableCount; |
| 292 | } |
| 293 | |
| 294 | static int AccessFlags(int Flags, bool IsWrite) { |
| 295 | return Flags | (IsWrite ? CSAN_ACCESS_WRITE : 0); |
| 296 | } |
| 297 | |
| 298 | #define CSAN_PROBE(name, N, IsWrite) \ |
| 299 | INTERFACE void name(void *Addr, int Flags) { \ |
| 300 | GET_CALLER_PC_BP; \ |
| 301 | __csan::CheckAccess(Addr, N, AccessFlags(Flags, IsWrite), pc, bp); \ |
| 302 | } |
| 303 | |
| 304 | #define CSAN_ACCESS(N) \ |
| 305 | CSAN_PROBE(__csan_read##N, N, false) \ |
| 306 | CSAN_PROBE(__csan_unaligned_read##N, N, false) \ |
| 307 | CSAN_PROBE(__csan_write##N, N, true) \ |
| 308 | CSAN_PROBE(__csan_unaligned_write##N, N, true) \ |
| 309 | CSAN_PROBE(__csan_read_write##N, N, true) \ |
| 310 | CSAN_PROBE(__csan_unaligned_read_write##N, N, true) |
| 311 | |
| 312 | CSAN_ACCESS(1) |
| 313 | CSAN_ACCESS(2) |
| 314 | CSAN_ACCESS(4) |
| 315 | CSAN_ACCESS(8) |
| 316 | CSAN_ACCESS(16) |
| 317 | |
| 318 | INTERFACE void __csan_read_range(void *Addr, uptr Size, int Flags) { |
| 319 | GET_CALLER_PC_BP; |
| 320 | __csan::CheckAccess(Ptr: Addr, Size, Type: AccessFlags(Flags, IsWrite: false), PC: pc, BP: bp); |
| 321 | } |
| 322 | |
| 323 | INTERFACE void __csan_write_range(void *Addr, uptr Size, int Flags) { |
| 324 | GET_CALLER_PC_BP; |
| 325 | __csan::CheckAccess(Ptr: Addr, Size, Type: AccessFlags(Flags, IsWrite: true), PC: pc, BP: bp); |
| 326 | } |
| 327 | |
| 328 | // TODO: Handle thread reordering checks like KCSAN. |
| 329 | INTERFACE void __csan_atomic_thread_fence(int) {} |
| 330 | INTERFACE void __csan_atomic_signal_fence(int) {} |
| 331 | |