1//===-- tsan_interface_atomic.cpp -----------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is a part of ThreadSanitizer (TSan), a race detector.
10//
11//===----------------------------------------------------------------------===//
12
13// ThreadSanitizer atomic operations are based on C++11/C1x standards.
14// For background see C++11 standard. A slightly older, publicly
15// available draft of the standard (not entirely up-to-date, but close enough
16// for casual browsing) is available here:
17// http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2011/n3242.pdf
18// The following page contains more background information:
19// http://www.hpl.hp.com/personal/Hans_Boehm/c++mm/
20
21#include "sanitizer_common/sanitizer_mutex.h"
22#include "sanitizer_common/sanitizer_placement_new.h"
23#include "sanitizer_common/sanitizer_stacktrace.h"
24#include "tsan_adaptive_delay.h"
25#include "tsan_flags.h"
26#include "tsan_interface.h"
27#include "tsan_rtl.h"
28
29using namespace __tsan;
30
31#if __TSAN_HAS_INT128
32// Protects emulation of 128-bit atomic operations.
33static StaticSpinMutex mutex128;
34#endif
35
36#if SANITIZER_DEBUG
37static bool IsLoadOrder(morder mo) {
38 return mo == mo_relaxed || mo == mo_consume || mo == mo_acquire ||
39 mo == mo_seq_cst;
40}
41
42static bool IsStoreOrder(morder mo) {
43 return mo == mo_relaxed || mo == mo_release || mo == mo_seq_cst;
44}
45#endif
46
47static bool IsReleaseOrder(morder mo) {
48 return mo == mo_release || mo == mo_acq_rel || mo == mo_seq_cst;
49}
50
51static bool IsAcquireOrder(morder mo) {
52 return mo == mo_consume || mo == mo_acquire || mo == mo_acq_rel ||
53 mo == mo_seq_cst;
54}
55
56static bool IsAcqRelOrder(morder mo) {
57 return mo == mo_acq_rel || mo == mo_seq_cst;
58}
59
60template <typename T>
61T func_xchg(volatile T *v, T op) {
62 T res = __sync_lock_test_and_set(v, op);
63 // __sync_lock_test_and_set does not contain full barrier.
64 __sync_synchronize();
65 return res;
66}
67
68template <typename T>
69T func_add(volatile T *v, T op) {
70 return __sync_fetch_and_add(v, op);
71}
72
73template <typename T>
74T func_sub(volatile T *v, T op) {
75 return __sync_fetch_and_sub(v, op);
76}
77
78template <typename T>
79T func_and(volatile T *v, T op) {
80 return __sync_fetch_and_and(v, op);
81}
82
83template <typename T>
84T func_or(volatile T *v, T op) {
85 return __sync_fetch_and_or(v, op);
86}
87
88template <typename T>
89T func_xor(volatile T *v, T op) {
90 return __sync_fetch_and_xor(v, op);
91}
92
93template <typename T>
94T func_nand(volatile T *v, T op) {
95 // clang does not support __sync_fetch_and_nand.
96 T cmp = *v;
97 for (;;) {
98 T newv = ~(cmp & op);
99 T cur = __sync_val_compare_and_swap(v, cmp, newv);
100 if (cmp == cur)
101 return cmp;
102 cmp = cur;
103 }
104}
105
106template <typename T>
107T func_cas(volatile T *v, T cmp, T xch) {
108 return __sync_val_compare_and_swap(v, cmp, xch);
109}
110
111// clang does not support 128-bit atomic ops.
112// Atomic ops are executed under tsan internal mutex,
113// here we assume that the atomic variables are not accessed
114// from non-instrumented code.
115// For SANITIZER_GO builds we always use the mutex-based path regardless of
116// __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16 to avoid a libatomic dependency.
117#if __TSAN_HAS_INT128 && \
118 (!defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16) || SANITIZER_GO)
119a128 func_xchg(volatile a128 *v, a128 op) {
120 SpinMutexLock lock(&mutex128);
121 a128 cmp = *v;
122 *v = op;
123 return cmp;
124}
125
126a128 func_add(volatile a128 *v, a128 op) {
127 SpinMutexLock lock(&mutex128);
128 a128 cmp = *v;
129 *v = cmp + op;
130 return cmp;
131}
132
133a128 func_sub(volatile a128 *v, a128 op) {
134 SpinMutexLock lock(&mutex128);
135 a128 cmp = *v;
136 *v = cmp - op;
137 return cmp;
138}
139
140a128 func_and(volatile a128 *v, a128 op) {
141 SpinMutexLock lock(&mutex128);
142 a128 cmp = *v;
143 *v = cmp & op;
144 return cmp;
145}
146
147a128 func_or(volatile a128 *v, a128 op) {
148 SpinMutexLock lock(&mutex128);
149 a128 cmp = *v;
150 *v = cmp | op;
151 return cmp;
152}
153
154a128 func_xor(volatile a128 *v, a128 op) {
155 SpinMutexLock lock(&mutex128);
156 a128 cmp = *v;
157 *v = cmp ^ op;
158 return cmp;
159}
160
161a128 func_nand(volatile a128 *v, a128 op) {
162 SpinMutexLock lock(&mutex128);
163 a128 cmp = *v;
164 *v = ~(cmp & op);
165 return cmp;
166}
167
168a128 func_cas(volatile a128 *v, a128 cmp, a128 xch) {
169 SpinMutexLock lock(&mutex128);
170 a128 cur = *v;
171 if (cur == cmp)
172 *v = xch;
173 return cur;
174}
175#endif
176
177template <typename T>
178static int AccessSize() {
179 if (sizeof(T) <= 1)
180 return 1;
181 else if (sizeof(T) <= 2)
182 return 2;
183 else if (sizeof(T) <= 4)
184 return 4;
185 else
186 return 8;
187 // For 16-byte atomics we also use 8-byte memory access,
188 // this leads to false negatives only in very obscure cases.
189}
190
191#if !SANITIZER_GO
192static atomic_uint8_t *to_atomic(const volatile a8 *a) {
193 return reinterpret_cast<atomic_uint8_t *>(const_cast<a8 *>(a));
194}
195
196static atomic_uint16_t *to_atomic(const volatile a16 *a) {
197 return reinterpret_cast<atomic_uint16_t *>(const_cast<a16 *>(a));
198}
199#endif
200
201static atomic_uint32_t *to_atomic(const volatile a32 *a) {
202 return reinterpret_cast<atomic_uint32_t *>(const_cast<a32 *>(a));
203}
204
205static atomic_uint64_t *to_atomic(const volatile a64 *a) {
206 return reinterpret_cast<atomic_uint64_t *>(const_cast<a64 *>(a));
207}
208
209static memory_order to_mo(morder mo) {
210 switch (mo) {
211 case mo_relaxed:
212 return memory_order_relaxed;
213 case mo_consume:
214 return memory_order_consume;
215 case mo_acquire:
216 return memory_order_acquire;
217 case mo_release:
218 return memory_order_release;
219 case mo_acq_rel:
220 return memory_order_acq_rel;
221 case mo_seq_cst:
222 return memory_order_seq_cst;
223 }
224 DCHECK(0);
225 return memory_order_seq_cst;
226}
227
228namespace {
229
230template <typename T, T (*F)(volatile T *v, T op)>
231static T AtomicRMW(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) {
232 MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(), kAccessWrite | kAccessAtomic);
233 if (LIKELY(mo == mo_relaxed))
234 return F(a, v);
235 SlotLocker locker(thr);
236 {
237 auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr: (uptr)a, save_stack: false);
238 RWLock lock(&s->mtx, IsReleaseOrder(mo));
239 if (IsAcqRelOrder(mo))
240 thr->clock.ReleaseAcquire(dstp: &s->clock);
241 else if (IsReleaseOrder(mo))
242 thr->clock.Release(dstp: &s->clock);
243 else if (IsAcquireOrder(mo))
244 thr->clock.Acquire(src: s->clock);
245 v = F(a, v);
246 }
247 if (IsReleaseOrder(mo))
248 IncrementEpoch(thr);
249 return v;
250}
251
252struct OpLoad {
253 template <typename T>
254 static T NoTsanAtomic(morder mo, const volatile T *a) {
255 return atomic_load(to_atomic(a), to_mo(mo));
256 }
257
258#if __TSAN_HAS_INT128
259 static a128 NoTsanAtomic(morder mo, const volatile a128 *a) {
260 SpinMutexLock lock(&mutex128);
261 return *a;
262 }
263#endif
264
265 template <typename T>
266 static T Atomic(ThreadState *thr, uptr pc, morder mo, const volatile T *a) {
267 DCHECK(IsLoadOrder(mo));
268 // This fast-path is critical for performance.
269 // Assume the access is atomic.
270 if (!IsAcquireOrder(mo)) {
271 MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(),
272 kAccessRead | kAccessAtomic);
273 return NoTsanAtomic(mo, a);
274 }
275 // Don't create sync object if it does not exist yet. For example, an atomic
276 // pointer is initialized to nullptr and then periodically acquire-loaded.
277 T v = NoTsanAtomic(mo, a);
278 SyncVar *s = ctx->metamap.GetSyncIfExists(addr: (uptr)a);
279 if (s) {
280 SlotLocker locker(thr);
281 ReadLock lock(&s->mtx);
282 thr->clock.Acquire(src: s->clock);
283 // Re-read under sync mutex because we need a consistent snapshot
284 // of the value and the clock we acquire.
285 v = NoTsanAtomic(mo, a);
286 }
287 MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(),
288 kAccessRead | kAccessAtomic);
289 return v;
290 }
291};
292
293struct OpStore {
294 template <typename T>
295 static void NoTsanAtomic(morder mo, volatile T *a, T v) {
296 atomic_store(to_atomic(a), v, to_mo(mo));
297 }
298
299#if __TSAN_HAS_INT128
300 static void NoTsanAtomic(morder mo, volatile a128 *a, a128 v) {
301 SpinMutexLock lock(&mutex128);
302 *a = v;
303 }
304#endif
305
306 template <typename T>
307 static void Atomic(ThreadState *thr, uptr pc, morder mo, volatile T *a, T v) {
308 DCHECK(IsStoreOrder(mo));
309 MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(),
310 kAccessWrite | kAccessAtomic);
311 // This fast-path is critical for performance.
312 // Assume the access is atomic.
313 // Strictly saying even relaxed store cuts off release sequence,
314 // so must reset the clock.
315 if (!IsReleaseOrder(mo)) {
316 NoTsanAtomic(mo, a, v);
317 return;
318 }
319 SlotLocker locker(thr);
320 {
321 auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr: (uptr)a, save_stack: false);
322 Lock lock(&s->mtx);
323 thr->clock.ReleaseStore(dstp: &s->clock);
324 NoTsanAtomic(mo, a, v);
325 }
326 IncrementEpoch(thr);
327 }
328};
329
330struct OpExchange {
331 template <typename T>
332 static T NoTsanAtomic(morder mo, volatile T *a, T v) {
333 return func_xchg(a, v);
334 }
335 template <typename T>
336 static T Atomic(ThreadState *thr, uptr pc, morder mo, volatile T *a, T v) {
337 return AtomicRMW<T, func_xchg>(thr, pc, a, v, mo);
338 }
339};
340
341struct OpFetchAdd {
342 template <typename T>
343 static T NoTsanAtomic(morder mo, volatile T *a, T v) {
344 return func_add(a, v);
345 }
346
347 template <typename T>
348 static T Atomic(ThreadState *thr, uptr pc, morder mo, volatile T *a, T v) {
349 return AtomicRMW<T, func_add>(thr, pc, a, v, mo);
350 }
351};
352
353struct OpFetchSub {
354 template <typename T>
355 [[maybe_unused]] static T NoTsanAtomic(morder mo, volatile T* a, T v) {
356 return func_sub(a, v);
357 }
358
359 template <typename T>
360 [[maybe_unused]] static T Atomic(ThreadState* thr, uptr pc, morder mo,
361 volatile T* a, T v) {
362 return AtomicRMW<T, func_sub>(thr, pc, a, v, mo);
363 }
364};
365
366struct OpFetchAnd {
367 template <typename T>
368 static T NoTsanAtomic(morder mo, volatile T *a, T v) {
369 return func_and(a, v);
370 }
371
372 template <typename T>
373 static T Atomic(ThreadState *thr, uptr pc, morder mo, volatile T *a, T v) {
374 return AtomicRMW<T, func_and>(thr, pc, a, v, mo);
375 }
376};
377
378struct OpFetchOr {
379 template <typename T>
380 static T NoTsanAtomic(morder mo, volatile T *a, T v) {
381 return func_or(a, v);
382 }
383
384 template <typename T>
385 static T Atomic(ThreadState *thr, uptr pc, morder mo, volatile T *a, T v) {
386 return AtomicRMW<T, func_or>(thr, pc, a, v, mo);
387 }
388};
389
390struct OpFetchXor {
391 template <typename T>
392 [[maybe_unused]] static T NoTsanAtomic(morder mo, volatile T* a, T v) {
393 return func_xor(a, v);
394 }
395
396 template <typename T>
397 [[maybe_unused]] static T Atomic(ThreadState* thr, uptr pc, morder mo,
398 volatile T* a, T v) {
399 return AtomicRMW<T, func_xor>(thr, pc, a, v, mo);
400 }
401};
402
403struct OpFetchNand {
404 template <typename T>
405 [[maybe_unused]] static T NoTsanAtomic(morder mo, volatile T* a, T v) {
406 return func_nand(a, v);
407 }
408
409 template <typename T>
410 [[maybe_unused]] static T Atomic(ThreadState* thr, uptr pc, morder mo,
411 volatile T* a, T v) {
412 return AtomicRMW<T, func_nand>(thr, pc, a, v, mo);
413 }
414};
415
416struct OpCAS {
417 template <typename T>
418 static bool NoTsanAtomic(morder mo, morder fmo, volatile T *a, T *c, T v) {
419 return atomic_compare_exchange_strong(to_atomic(a), c, v, to_mo(mo));
420 }
421
422#if __TSAN_HAS_INT128
423 static bool NoTsanAtomic(morder mo, morder fmo, volatile a128 *a, a128 *c,
424 a128 v) {
425 a128 old = *c;
426 a128 cur = func_cas(v: a, cmp: old, xch: v);
427 if (cur == old)
428 return true;
429 *c = cur;
430 return false;
431 }
432#endif
433
434 template <typename T>
435 static T NoTsanAtomic(morder mo, morder fmo, volatile T *a, T c, T v) {
436 NoTsanAtomic(mo, fmo, a, &c, v);
437 return c;
438 }
439
440 template <typename T>
441 static bool Atomic(ThreadState *thr, uptr pc, morder mo, morder fmo,
442 volatile T *a, T *c, T v) {
443 // 31.7.2.18: "The failure argument shall not be memory_order_release
444 // nor memory_order_acq_rel". LLVM (2021-05) fallbacks to Monotonic
445 // (mo_relaxed) when those are used.
446 DCHECK(IsLoadOrder(fmo));
447
448 MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(),
449 kAccessWrite | kAccessAtomic);
450 if (LIKELY(mo == mo_relaxed && fmo == mo_relaxed)) {
451 T cc = *c;
452 T pr = func_cas(a, cc, v);
453 if (pr == cc)
454 return true;
455 *c = pr;
456 return false;
457 }
458 SlotLocker locker(thr);
459 bool release = IsReleaseOrder(mo);
460 bool success;
461 {
462 auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr: (uptr)a, save_stack: false);
463 RWLock lock(&s->mtx, release);
464 T cc = *c;
465 T pr = func_cas(a, cc, v);
466 success = pr == cc;
467 if (!success) {
468 *c = pr;
469 mo = fmo;
470 }
471 if (success && IsAcqRelOrder(mo))
472 thr->clock.ReleaseAcquire(dstp: &s->clock);
473 else if (success && IsReleaseOrder(mo))
474 thr->clock.Release(dstp: &s->clock);
475 else if (IsAcquireOrder(mo))
476 thr->clock.Acquire(src: s->clock);
477 }
478 if (success && release)
479 IncrementEpoch(thr);
480 return success;
481 }
482
483 template <typename T>
484 static T Atomic(ThreadState *thr, uptr pc, morder mo, morder fmo,
485 volatile T *a, T c, T v) {
486 Atomic(thr, pc, mo, fmo, a, &c, v);
487 return c;
488 }
489};
490
491#if !SANITIZER_GO
492struct OpFence {
493 static void NoTsanAtomic(morder mo) { __sync_synchronize(); }
494
495 static void Atomic(ThreadState *thr, uptr pc, morder mo) {
496 // FIXME(dvyukov): not implemented.
497 __sync_synchronize();
498 }
499};
500#endif
501
502} // namespace
503
504// Interface functions follow.
505#if !SANITIZER_GO
506
507// C/C++
508
509static morder convert_morder(morder mo) {
510 return flags()->force_seq_cst_atomics ? mo_seq_cst : mo;
511}
512
513static morder to_morder(int mo) {
514 // Filter out additional memory order flags:
515 // MEMMODEL_SYNC = 1 << 15
516 // __ATOMIC_HLE_ACQUIRE = 1 << 16
517 // __ATOMIC_HLE_RELEASE = 1 << 17
518 //
519 // HLE is an optimization, and we pretend that elision always fails.
520 // MEMMODEL_SYNC is used when lowering __sync_ atomics,
521 // since we use __sync_ atomics for actual atomic operations,
522 // we can safely ignore it as well. It also subtly affects semantics,
523 // but we don't model the difference.
524 morder res = static_cast<morder>(static_cast<u8>(mo));
525 DCHECK_LE(res, mo_seq_cst);
526 return res;
527}
528
529template <class... Types>
530ALWAYS_INLINE auto AtomicDelayImpl(morder mo, Types... args) {
531 AdaptiveDelay::AtomicOpFence(mo);
532}
533
534template <class AddrType, class... Types>
535ALWAYS_INLINE auto AtomicDelayImpl(morder mo, AddrType addr, Types... args) {
536 AdaptiveDelay::AtomicOpAddr(addr: (uptr)addr, mo: (int)mo);
537}
538
539template <class Op, class... Types>
540ALWAYS_INLINE auto AtomicImpl(morder mo, Types... args) {
541 AtomicDelayImpl(mo, args...);
542 ThreadState *const thr = cur_thread();
543 ProcessPendingSignals(thr);
544 if (UNLIKELY(thr->ignore_sync || thr->ignore_interceptors))
545 return Op::NoTsanAtomic(mo, args...);
546 return Op::Atomic(thr, GET_CALLER_PC(), convert_morder(mo), args...);
547}
548
549extern "C" {
550SANITIZER_INTERFACE_ATTRIBUTE
551a8 __tsan_atomic8_load(const volatile a8 *a, int mo) {
552 return AtomicImpl<OpLoad>(mo: to_morder(mo), args: a);
553}
554
555SANITIZER_INTERFACE_ATTRIBUTE
556a16 __tsan_atomic16_load(const volatile a16 *a, int mo) {
557 return AtomicImpl<OpLoad>(mo: to_morder(mo), args: a);
558}
559
560SANITIZER_INTERFACE_ATTRIBUTE
561a32 __tsan_atomic32_load(const volatile a32 *a, int mo) {
562 return AtomicImpl<OpLoad>(mo: to_morder(mo), args: a);
563}
564
565SANITIZER_INTERFACE_ATTRIBUTE
566a64 __tsan_atomic64_load(const volatile a64 *a, int mo) {
567 return AtomicImpl<OpLoad>(mo: to_morder(mo), args: a);
568}
569
570# if __TSAN_HAS_INT128
571SANITIZER_INTERFACE_ATTRIBUTE
572a128 __tsan_atomic128_load(const volatile a128 *a, int mo) {
573 return AtomicImpl<OpLoad>(mo: to_morder(mo), args: a);
574}
575# endif
576
577SANITIZER_INTERFACE_ATTRIBUTE
578void __tsan_atomic8_store(volatile a8 *a, a8 v, int mo) {
579 return AtomicImpl<OpStore>(mo: to_morder(mo), args: a, args: v);
580}
581
582SANITIZER_INTERFACE_ATTRIBUTE
583void __tsan_atomic16_store(volatile a16 *a, a16 v, int mo) {
584 return AtomicImpl<OpStore>(mo: to_morder(mo), args: a, args: v);
585}
586
587SANITIZER_INTERFACE_ATTRIBUTE
588void __tsan_atomic32_store(volatile a32 *a, a32 v, int mo) {
589 return AtomicImpl<OpStore>(mo: to_morder(mo), args: a, args: v);
590}
591
592SANITIZER_INTERFACE_ATTRIBUTE
593void __tsan_atomic64_store(volatile a64 *a, a64 v, int mo) {
594 return AtomicImpl<OpStore>(mo: to_morder(mo), args: a, args: v);
595}
596
597# if __TSAN_HAS_INT128
598SANITIZER_INTERFACE_ATTRIBUTE
599void __tsan_atomic128_store(volatile a128 *a, a128 v, int mo) {
600 return AtomicImpl<OpStore>(mo: to_morder(mo), args: a, args: v);
601}
602# endif
603
604SANITIZER_INTERFACE_ATTRIBUTE
605a8 __tsan_atomic8_exchange(volatile a8 *a, a8 v, int mo) {
606 return AtomicImpl<OpExchange>(mo: to_morder(mo), args: a, args: v);
607}
608
609SANITIZER_INTERFACE_ATTRIBUTE
610a16 __tsan_atomic16_exchange(volatile a16 *a, a16 v, int mo) {
611 return AtomicImpl<OpExchange>(mo: to_morder(mo), args: a, args: v);
612}
613
614SANITIZER_INTERFACE_ATTRIBUTE
615a32 __tsan_atomic32_exchange(volatile a32 *a, a32 v, int mo) {
616 return AtomicImpl<OpExchange>(mo: to_morder(mo), args: a, args: v);
617}
618
619SANITIZER_INTERFACE_ATTRIBUTE
620a64 __tsan_atomic64_exchange(volatile a64 *a, a64 v, int mo) {
621 return AtomicImpl<OpExchange>(mo: to_morder(mo), args: a, args: v);
622}
623
624# if __TSAN_HAS_INT128
625SANITIZER_INTERFACE_ATTRIBUTE
626a128 __tsan_atomic128_exchange(volatile a128 *a, a128 v, int mo) {
627 return AtomicImpl<OpExchange>(mo: to_morder(mo), args: a, args: v);
628}
629# endif
630
631SANITIZER_INTERFACE_ATTRIBUTE
632a8 __tsan_atomic8_fetch_add(volatile a8 *a, a8 v, int mo) {
633 return AtomicImpl<OpFetchAdd>(mo: to_morder(mo), args: a, args: v);
634}
635
636SANITIZER_INTERFACE_ATTRIBUTE
637a16 __tsan_atomic16_fetch_add(volatile a16 *a, a16 v, int mo) {
638 return AtomicImpl<OpFetchAdd>(mo: to_morder(mo), args: a, args: v);
639}
640
641SANITIZER_INTERFACE_ATTRIBUTE
642a32 __tsan_atomic32_fetch_add(volatile a32 *a, a32 v, int mo) {
643 return AtomicImpl<OpFetchAdd>(mo: to_morder(mo), args: a, args: v);
644}
645
646SANITIZER_INTERFACE_ATTRIBUTE
647a64 __tsan_atomic64_fetch_add(volatile a64 *a, a64 v, int mo) {
648 return AtomicImpl<OpFetchAdd>(mo: to_morder(mo), args: a, args: v);
649}
650
651# if __TSAN_HAS_INT128
652SANITIZER_INTERFACE_ATTRIBUTE
653a128 __tsan_atomic128_fetch_add(volatile a128 *a, a128 v, int mo) {
654 return AtomicImpl<OpFetchAdd>(mo: to_morder(mo), args: a, args: v);
655}
656# endif
657
658SANITIZER_INTERFACE_ATTRIBUTE
659a8 __tsan_atomic8_fetch_sub(volatile a8 *a, a8 v, int mo) {
660 return AtomicImpl<OpFetchSub>(mo: to_morder(mo), args: a, args: v);
661}
662
663SANITIZER_INTERFACE_ATTRIBUTE
664a16 __tsan_atomic16_fetch_sub(volatile a16 *a, a16 v, int mo) {
665 return AtomicImpl<OpFetchSub>(mo: to_morder(mo), args: a, args: v);
666}
667
668SANITIZER_INTERFACE_ATTRIBUTE
669a32 __tsan_atomic32_fetch_sub(volatile a32 *a, a32 v, int mo) {
670 return AtomicImpl<OpFetchSub>(mo: to_morder(mo), args: a, args: v);
671}
672
673SANITIZER_INTERFACE_ATTRIBUTE
674a64 __tsan_atomic64_fetch_sub(volatile a64 *a, a64 v, int mo) {
675 return AtomicImpl<OpFetchSub>(mo: to_morder(mo), args: a, args: v);
676}
677
678# if __TSAN_HAS_INT128
679SANITIZER_INTERFACE_ATTRIBUTE
680a128 __tsan_atomic128_fetch_sub(volatile a128 *a, a128 v, int mo) {
681 return AtomicImpl<OpFetchSub>(mo: to_morder(mo), args: a, args: v);
682}
683# endif
684
685SANITIZER_INTERFACE_ATTRIBUTE
686a8 __tsan_atomic8_fetch_and(volatile a8 *a, a8 v, int mo) {
687 return AtomicImpl<OpFetchAnd>(mo: to_morder(mo), args: a, args: v);
688}
689
690SANITIZER_INTERFACE_ATTRIBUTE
691a16 __tsan_atomic16_fetch_and(volatile a16 *a, a16 v, int mo) {
692 return AtomicImpl<OpFetchAnd>(mo: to_morder(mo), args: a, args: v);
693}
694
695SANITIZER_INTERFACE_ATTRIBUTE
696a32 __tsan_atomic32_fetch_and(volatile a32 *a, a32 v, int mo) {
697 return AtomicImpl<OpFetchAnd>(mo: to_morder(mo), args: a, args: v);
698}
699
700SANITIZER_INTERFACE_ATTRIBUTE
701a64 __tsan_atomic64_fetch_and(volatile a64 *a, a64 v, int mo) {
702 return AtomicImpl<OpFetchAnd>(mo: to_morder(mo), args: a, args: v);
703}
704
705# if __TSAN_HAS_INT128
706SANITIZER_INTERFACE_ATTRIBUTE
707a128 __tsan_atomic128_fetch_and(volatile a128 *a, a128 v, int mo) {
708 return AtomicImpl<OpFetchAnd>(mo: to_morder(mo), args: a, args: v);
709}
710# endif
711
712SANITIZER_INTERFACE_ATTRIBUTE
713a8 __tsan_atomic8_fetch_or(volatile a8 *a, a8 v, int mo) {
714 return AtomicImpl<OpFetchOr>(mo: to_morder(mo), args: a, args: v);
715}
716
717SANITIZER_INTERFACE_ATTRIBUTE
718a16 __tsan_atomic16_fetch_or(volatile a16 *a, a16 v, int mo) {
719 return AtomicImpl<OpFetchOr>(mo: to_morder(mo), args: a, args: v);
720}
721
722SANITIZER_INTERFACE_ATTRIBUTE
723a32 __tsan_atomic32_fetch_or(volatile a32 *a, a32 v, int mo) {
724 return AtomicImpl<OpFetchOr>(mo: to_morder(mo), args: a, args: v);
725}
726
727SANITIZER_INTERFACE_ATTRIBUTE
728a64 __tsan_atomic64_fetch_or(volatile a64 *a, a64 v, int mo) {
729 return AtomicImpl<OpFetchOr>(mo: to_morder(mo), args: a, args: v);
730}
731
732# if __TSAN_HAS_INT128
733SANITIZER_INTERFACE_ATTRIBUTE
734a128 __tsan_atomic128_fetch_or(volatile a128 *a, a128 v, int mo) {
735 return AtomicImpl<OpFetchOr>(mo: to_morder(mo), args: a, args: v);
736}
737# endif
738
739SANITIZER_INTERFACE_ATTRIBUTE
740a8 __tsan_atomic8_fetch_xor(volatile a8 *a, a8 v, int mo) {
741 return AtomicImpl<OpFetchXor>(mo: to_morder(mo), args: a, args: v);
742}
743
744SANITIZER_INTERFACE_ATTRIBUTE
745a16 __tsan_atomic16_fetch_xor(volatile a16 *a, a16 v, int mo) {
746 return AtomicImpl<OpFetchXor>(mo: to_morder(mo), args: a, args: v);
747}
748
749SANITIZER_INTERFACE_ATTRIBUTE
750a32 __tsan_atomic32_fetch_xor(volatile a32 *a, a32 v, int mo) {
751 return AtomicImpl<OpFetchXor>(mo: to_morder(mo), args: a, args: v);
752}
753
754SANITIZER_INTERFACE_ATTRIBUTE
755a64 __tsan_atomic64_fetch_xor(volatile a64 *a, a64 v, int mo) {
756 return AtomicImpl<OpFetchXor>(mo: to_morder(mo), args: a, args: v);
757}
758
759# if __TSAN_HAS_INT128
760SANITIZER_INTERFACE_ATTRIBUTE
761a128 __tsan_atomic128_fetch_xor(volatile a128 *a, a128 v, int mo) {
762 return AtomicImpl<OpFetchXor>(mo: to_morder(mo), args: a, args: v);
763}
764# endif
765
766SANITIZER_INTERFACE_ATTRIBUTE
767a8 __tsan_atomic8_fetch_nand(volatile a8 *a, a8 v, int mo) {
768 return AtomicImpl<OpFetchNand>(mo: to_morder(mo), args: a, args: v);
769}
770
771SANITIZER_INTERFACE_ATTRIBUTE
772a16 __tsan_atomic16_fetch_nand(volatile a16 *a, a16 v, int mo) {
773 return AtomicImpl<OpFetchNand>(mo: to_morder(mo), args: a, args: v);
774}
775
776SANITIZER_INTERFACE_ATTRIBUTE
777a32 __tsan_atomic32_fetch_nand(volatile a32 *a, a32 v, int mo) {
778 return AtomicImpl<OpFetchNand>(mo: to_morder(mo), args: a, args: v);
779}
780
781SANITIZER_INTERFACE_ATTRIBUTE
782a64 __tsan_atomic64_fetch_nand(volatile a64 *a, a64 v, int mo) {
783 return AtomicImpl<OpFetchNand>(mo: to_morder(mo), args: a, args: v);
784}
785
786# if __TSAN_HAS_INT128
787SANITIZER_INTERFACE_ATTRIBUTE
788a128 __tsan_atomic128_fetch_nand(volatile a128 *a, a128 v, int mo) {
789 return AtomicImpl<OpFetchNand>(mo: to_morder(mo), args: a, args: v);
790}
791# endif
792
793SANITIZER_INTERFACE_ATTRIBUTE
794int __tsan_atomic8_compare_exchange_strong(volatile a8 *a, a8 *c, a8 v, int mo,
795 int fmo) {
796 return AtomicImpl<OpCAS>(mo: to_morder(mo), args: to_morder(mo: fmo), args: a, args: c, args: v);
797}
798
799SANITIZER_INTERFACE_ATTRIBUTE
800int __tsan_atomic16_compare_exchange_strong(volatile a16 *a, a16 *c, a16 v,
801 int mo, int fmo) {
802 return AtomicImpl<OpCAS>(mo: to_morder(mo), args: to_morder(mo: fmo), args: a, args: c, args: v);
803}
804
805SANITIZER_INTERFACE_ATTRIBUTE
806int __tsan_atomic32_compare_exchange_strong(volatile a32 *a, a32 *c, a32 v,
807 int mo, int fmo) {
808 return AtomicImpl<OpCAS>(mo: to_morder(mo), args: to_morder(mo: fmo), args: a, args: c, args: v);
809}
810
811SANITIZER_INTERFACE_ATTRIBUTE
812int __tsan_atomic64_compare_exchange_strong(volatile a64 *a, a64 *c, a64 v,
813 int mo, int fmo) {
814 return AtomicImpl<OpCAS>(mo: to_morder(mo), args: to_morder(mo: fmo), args: a, args: c, args: v);
815}
816
817# if __TSAN_HAS_INT128
818SANITIZER_INTERFACE_ATTRIBUTE
819int __tsan_atomic128_compare_exchange_strong(volatile a128 *a, a128 *c, a128 v,
820 int mo, int fmo) {
821 return AtomicImpl<OpCAS>(mo: to_morder(mo), args: to_morder(mo: fmo), args: a, args: c, args: v);
822}
823# endif
824
825SANITIZER_INTERFACE_ATTRIBUTE
826int __tsan_atomic8_compare_exchange_weak(volatile a8 *a, a8 *c, a8 v, int mo,
827 int fmo) {
828 return AtomicImpl<OpCAS>(mo: to_morder(mo), args: to_morder(mo: fmo), args: a, args: c, args: v);
829}
830
831SANITIZER_INTERFACE_ATTRIBUTE
832int __tsan_atomic16_compare_exchange_weak(volatile a16 *a, a16 *c, a16 v,
833 int mo, int fmo) {
834 return AtomicImpl<OpCAS>(mo: to_morder(mo), args: to_morder(mo: fmo), args: a, args: c, args: v);
835}
836
837SANITIZER_INTERFACE_ATTRIBUTE
838int __tsan_atomic32_compare_exchange_weak(volatile a32 *a, a32 *c, a32 v,
839 int mo, int fmo) {
840 return AtomicImpl<OpCAS>(mo: to_morder(mo), args: to_morder(mo: fmo), args: a, args: c, args: v);
841}
842
843SANITIZER_INTERFACE_ATTRIBUTE
844int __tsan_atomic64_compare_exchange_weak(volatile a64 *a, a64 *c, a64 v,
845 int mo, int fmo) {
846 return AtomicImpl<OpCAS>(mo: to_morder(mo), args: to_morder(mo: fmo), args: a, args: c, args: v);
847}
848
849# if __TSAN_HAS_INT128
850SANITIZER_INTERFACE_ATTRIBUTE
851int __tsan_atomic128_compare_exchange_weak(volatile a128 *a, a128 *c, a128 v,
852 int mo, int fmo) {
853 return AtomicImpl<OpCAS>(mo: to_morder(mo), args: to_morder(mo: fmo), args: a, args: c, args: v);
854}
855# endif
856
857SANITIZER_INTERFACE_ATTRIBUTE
858a8 __tsan_atomic8_compare_exchange_val(volatile a8 *a, a8 c, a8 v, int mo,
859 int fmo) {
860 return AtomicImpl<OpCAS>(mo: to_morder(mo), args: to_morder(mo: fmo), args: a, args: c, args: v);
861}
862
863SANITIZER_INTERFACE_ATTRIBUTE
864a16 __tsan_atomic16_compare_exchange_val(volatile a16 *a, a16 c, a16 v, int mo,
865 int fmo) {
866 return AtomicImpl<OpCAS>(mo: to_morder(mo), args: to_morder(mo: fmo), args: a, args: c, args: v);
867}
868
869SANITIZER_INTERFACE_ATTRIBUTE
870a32 __tsan_atomic32_compare_exchange_val(volatile a32 *a, a32 c, a32 v, int mo,
871 int fmo) {
872 return AtomicImpl<OpCAS>(mo: to_morder(mo), args: to_morder(mo: fmo), args: a, args: c, args: v);
873}
874
875SANITIZER_INTERFACE_ATTRIBUTE
876a64 __tsan_atomic64_compare_exchange_val(volatile a64 *a, a64 c, a64 v, int mo,
877 int fmo) {
878 return AtomicImpl<OpCAS>(mo: to_morder(mo), args: to_morder(mo: fmo), args: a, args: c, args: v);
879}
880
881# if __TSAN_HAS_INT128
882SANITIZER_INTERFACE_ATTRIBUTE
883a128 __tsan_atomic128_compare_exchange_val(volatile a128 *a, a128 c, a128 v,
884 int mo, int fmo) {
885 return AtomicImpl<OpCAS>(mo: to_morder(mo), args: to_morder(mo: fmo), args: a, args: c, args: v);
886}
887# endif
888
889SANITIZER_INTERFACE_ATTRIBUTE
890void __tsan_atomic_thread_fence(int mo) {
891 return AtomicImpl<OpFence>(mo: to_morder(mo));
892}
893
894SANITIZER_INTERFACE_ATTRIBUTE
895void __tsan_atomic_signal_fence(int mo) {}
896} // extern "C"
897
898#else // #if !SANITIZER_GO
899
900// Go
901
902template <class Op, class... Types>
903void AtomicGo(ThreadState *thr, uptr cpc, uptr pc, Types... args) {
904 if (thr->ignore_sync) {
905 (void)Op::NoTsanAtomic(args...);
906 } else {
907 FuncEntry(thr, cpc);
908 (void)Op::Atomic(thr, pc, args...);
909 FuncExit(thr);
910 }
911}
912
913template <class Op, class... Types>
914auto AtomicGoRet(ThreadState *thr, uptr cpc, uptr pc, Types... args) {
915 if (thr->ignore_sync) {
916 return Op::NoTsanAtomic(args...);
917 } else {
918 FuncEntry(thr, cpc);
919 auto ret = Op::Atomic(thr, pc, args...);
920 FuncExit(thr);
921 return ret;
922 }
923}
924
925extern "C" {
926SANITIZER_INTERFACE_ATTRIBUTE
927void __tsan_go_atomic32_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
928 *(a32 *)(a + 8) = AtomicGoRet<OpLoad>(thr, cpc, pc, mo_acquire, *(a32 **)a);
929}
930
931SANITIZER_INTERFACE_ATTRIBUTE
932void __tsan_go_atomic64_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
933 *(a64 *)(a + 8) = AtomicGoRet<OpLoad>(thr, cpc, pc, mo_acquire, *(a64 **)a);
934}
935
936# if __TSAN_HAS_INT128
937// Go's args buffer is only 8-byte aligned; ALIGNED(8) relaxes the alignment
938// of accesses through this type.
939using a128_u64 ALIGNED(8) = a128;
940
941SANITIZER_INTERFACE_ATTRIBUTE
942void __tsan_go_atomic128_load(ThreadState* thr, uptr cpc, uptr pc, u8* a) {
943 *(a128_u64*)(a + 8) =
944 AtomicGoRet<OpLoad>(thr, cpc, pc, mo_acquire, *(a128**)a);
945}
946# endif
947
948SANITIZER_INTERFACE_ATTRIBUTE
949void __tsan_go_atomic32_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
950 AtomicGo<OpStore>(thr, cpc, pc, mo_release, *(a32 **)a, *(a32 *)(a + 8));
951}
952
953SANITIZER_INTERFACE_ATTRIBUTE
954void __tsan_go_atomic64_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
955 AtomicGo<OpStore>(thr, cpc, pc, mo_release, *(a64 **)a, *(a64 *)(a + 8));
956}
957
958# if __TSAN_HAS_INT128
959SANITIZER_INTERFACE_ATTRIBUTE
960void __tsan_go_atomic128_store(ThreadState* thr, uptr cpc, uptr pc, u8* a) {
961 AtomicGo<OpStore>(thr, cpc, pc, mo_release, *(a128**)a, *(a128_u64*)(a + 8));
962}
963# endif
964
965SANITIZER_INTERFACE_ATTRIBUTE
966void __tsan_go_atomic32_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
967 *(a32 *)(a + 16) = AtomicGoRet<OpFetchAdd>(thr, cpc, pc, mo_acq_rel,
968 *(a32 **)a, *(a32 *)(a + 8));
969}
970
971SANITIZER_INTERFACE_ATTRIBUTE
972void __tsan_go_atomic64_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
973 *(a64 *)(a + 16) = AtomicGoRet<OpFetchAdd>(thr, cpc, pc, mo_acq_rel,
974 *(a64 **)a, *(a64 *)(a + 8));
975}
976
977SANITIZER_INTERFACE_ATTRIBUTE
978void __tsan_go_atomic32_fetch_and(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
979 *(a32 *)(a + 16) = AtomicGoRet<OpFetchAnd>(thr, cpc, pc, mo_acq_rel,
980 *(a32 **)a, *(a32 *)(a + 8));
981}
982
983SANITIZER_INTERFACE_ATTRIBUTE
984void __tsan_go_atomic64_fetch_and(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
985 *(a64 *)(a + 16) = AtomicGoRet<OpFetchAnd>(thr, cpc, pc, mo_acq_rel,
986 *(a64 **)a, *(a64 *)(a + 8));
987}
988
989SANITIZER_INTERFACE_ATTRIBUTE
990void __tsan_go_atomic32_fetch_or(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
991 *(a32 *)(a + 16) = AtomicGoRet<OpFetchOr>(thr, cpc, pc, mo_acq_rel,
992 *(a32 **)a, *(a32 *)(a + 8));
993}
994
995SANITIZER_INTERFACE_ATTRIBUTE
996void __tsan_go_atomic64_fetch_or(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
997 *(a64 *)(a + 16) = AtomicGoRet<OpFetchOr>(thr, cpc, pc, mo_acq_rel,
998 *(a64 **)a, *(a64 *)(a + 8));
999}
1000
1001SANITIZER_INTERFACE_ATTRIBUTE
1002void __tsan_go_atomic32_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
1003 *(a32 *)(a + 16) = AtomicGoRet<OpExchange>(thr, cpc, pc, mo_acq_rel,
1004 *(a32 **)a, *(a32 *)(a + 8));
1005}
1006
1007SANITIZER_INTERFACE_ATTRIBUTE
1008void __tsan_go_atomic64_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
1009 *(a64 *)(a + 16) = AtomicGoRet<OpExchange>(thr, cpc, pc, mo_acq_rel,
1010 *(a64 **)a, *(a64 *)(a + 8));
1011}
1012
1013SANITIZER_INTERFACE_ATTRIBUTE
1014void __tsan_go_atomic32_compare_exchange(ThreadState *thr, uptr cpc, uptr pc,
1015 u8 *a) {
1016 a32 cmp = *(a32 *)(a + 8);
1017 a32 cur = AtomicGoRet<OpCAS>(thr, cpc, pc, mo_acq_rel, mo_acquire, *(a32 **)a,
1018 cmp, *(a32 *)(a + 12));
1019 *(bool *)(a + 16) = (cur == cmp);
1020}
1021
1022SANITIZER_INTERFACE_ATTRIBUTE
1023void __tsan_go_atomic64_compare_exchange(ThreadState *thr, uptr cpc, uptr pc,
1024 u8 *a) {
1025 a64 cmp = *(a64 *)(a + 8);
1026 a64 cur = AtomicGoRet<OpCAS>(thr, cpc, pc, mo_acq_rel, mo_acquire, *(a64 **)a,
1027 cmp, *(a64 *)(a + 16));
1028 *(bool *)(a + 24) = (cur == cmp);
1029}
1030
1031# if __TSAN_HAS_INT128
1032SANITIZER_INTERFACE_ATTRIBUTE
1033void __tsan_go_atomic128_compare_exchange(ThreadState* thr, uptr cpc, uptr pc,
1034 u8* a) {
1035 a128 cmp = *(a128_u64*)(a + 8);
1036 a128 cur = AtomicGoRet<OpCAS>(thr, cpc, pc, mo_acq_rel, mo_acquire,
1037 *(a128**)a, cmp, *(a128_u64*)(a + 24));
1038 *(bool*)(a + 40) = (cur == cmp);
1039}
1040# endif
1041} // extern "C"
1042#endif // #if !SANITIZER_GO
1043