1//===-- tsan_interface_atomic.cpp -----------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is a part of ThreadSanitizer (TSan), a race detector.
10//
11//===----------------------------------------------------------------------===//
12
13// ThreadSanitizer atomic operations are based on C++11/C1x standards.
14// For background see C++11 standard. A slightly older, publicly
15// available draft of the standard (not entirely up-to-date, but close enough
16// for casual browsing) is available here:
17// http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2011/n3242.pdf
18// The following page contains more background information:
19// http://www.hpl.hp.com/personal/Hans_Boehm/c++mm/
20
21#include "sanitizer_common/sanitizer_mutex.h"
22#include "sanitizer_common/sanitizer_placement_new.h"
23#include "sanitizer_common/sanitizer_stacktrace.h"
24#include "tsan_adaptive_delay.h"
25#include "tsan_flags.h"
26#include "tsan_interface.h"
27#include "tsan_rtl.h"
28
29using namespace __tsan;
30
31#if __TSAN_HAS_INT128
32// Protects emulation of 128-bit atomic operations.
33static StaticSpinMutex mutex128;
34#endif
35
36#if SANITIZER_DEBUG
37static bool IsLoadOrder(morder mo) {
38 return mo == mo_relaxed || mo == mo_consume || mo == mo_acquire ||
39 mo == mo_seq_cst;
40}
41
42static bool IsStoreOrder(morder mo) {
43 return mo == mo_relaxed || mo == mo_release || mo == mo_seq_cst;
44}
45#endif
46
47static bool IsReleaseOrder(morder mo) {
48 return mo == mo_release || mo == mo_acq_rel || mo == mo_seq_cst;
49}
50
51static bool IsAcquireOrder(morder mo) {
52 return mo == mo_consume || mo == mo_acquire || mo == mo_acq_rel ||
53 mo == mo_seq_cst;
54}
55
56static bool IsAcqRelOrder(morder mo) {
57 return mo == mo_acq_rel || mo == mo_seq_cst;
58}
59
60template <typename T>
61T func_xchg(volatile T *v, T op) {
62 T res = __sync_lock_test_and_set(v, op);
63 // __sync_lock_test_and_set does not contain full barrier.
64 __sync_synchronize();
65 return res;
66}
67
68template <typename T>
69T func_add(volatile T *v, T op) {
70 return __sync_fetch_and_add(v, op);
71}
72
73template <typename T>
74T func_sub(volatile T *v, T op) {
75 return __sync_fetch_and_sub(v, op);
76}
77
78template <typename T>
79T func_and(volatile T *v, T op) {
80 return __sync_fetch_and_and(v, op);
81}
82
83template <typename T>
84T func_or(volatile T *v, T op) {
85 return __sync_fetch_and_or(v, op);
86}
87
88template <typename T>
89T func_xor(volatile T *v, T op) {
90 return __sync_fetch_and_xor(v, op);
91}
92
93template <typename T>
94T func_nand(volatile T *v, T op) {
95 // clang does not support __sync_fetch_and_nand.
96 T cmp = *v;
97 for (;;) {
98 T newv = ~(cmp & op);
99 T cur = __sync_val_compare_and_swap(v, cmp, newv);
100 if (cmp == cur)
101 return cmp;
102 cmp = cur;
103 }
104}
105
106template <typename T>
107T func_cas(volatile T *v, T cmp, T xch) {
108 return __sync_val_compare_and_swap(v, cmp, xch);
109}
110
111// clang does not support 128-bit atomic ops.
112// Atomic ops are executed under tsan internal mutex,
113// here we assume that the atomic variables are not accessed
114// from non-instrumented code.
115#if !defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16) && __TSAN_HAS_INT128
116a128 func_xchg(volatile a128 *v, a128 op) {
117 SpinMutexLock lock(&mutex128);
118 a128 cmp = *v;
119 *v = op;
120 return cmp;
121}
122
123a128 func_add(volatile a128 *v, a128 op) {
124 SpinMutexLock lock(&mutex128);
125 a128 cmp = *v;
126 *v = cmp + op;
127 return cmp;
128}
129
130a128 func_sub(volatile a128 *v, a128 op) {
131 SpinMutexLock lock(&mutex128);
132 a128 cmp = *v;
133 *v = cmp - op;
134 return cmp;
135}
136
137a128 func_and(volatile a128 *v, a128 op) {
138 SpinMutexLock lock(&mutex128);
139 a128 cmp = *v;
140 *v = cmp & op;
141 return cmp;
142}
143
144a128 func_or(volatile a128 *v, a128 op) {
145 SpinMutexLock lock(&mutex128);
146 a128 cmp = *v;
147 *v = cmp | op;
148 return cmp;
149}
150
151a128 func_xor(volatile a128 *v, a128 op) {
152 SpinMutexLock lock(&mutex128);
153 a128 cmp = *v;
154 *v = cmp ^ op;
155 return cmp;
156}
157
158a128 func_nand(volatile a128 *v, a128 op) {
159 SpinMutexLock lock(&mutex128);
160 a128 cmp = *v;
161 *v = ~(cmp & op);
162 return cmp;
163}
164
165a128 func_cas(volatile a128 *v, a128 cmp, a128 xch) {
166 SpinMutexLock lock(&mutex128);
167 a128 cur = *v;
168 if (cur == cmp)
169 *v = xch;
170 return cur;
171}
172#endif
173
174template <typename T>
175static int AccessSize() {
176 if (sizeof(T) <= 1)
177 return 1;
178 else if (sizeof(T) <= 2)
179 return 2;
180 else if (sizeof(T) <= 4)
181 return 4;
182 else
183 return 8;
184 // For 16-byte atomics we also use 8-byte memory access,
185 // this leads to false negatives only in very obscure cases.
186}
187
188#if !SANITIZER_GO
189static atomic_uint8_t *to_atomic(const volatile a8 *a) {
190 return reinterpret_cast<atomic_uint8_t *>(const_cast<a8 *>(a));
191}
192
193static atomic_uint16_t *to_atomic(const volatile a16 *a) {
194 return reinterpret_cast<atomic_uint16_t *>(const_cast<a16 *>(a));
195}
196#endif
197
198static atomic_uint32_t *to_atomic(const volatile a32 *a) {
199 return reinterpret_cast<atomic_uint32_t *>(const_cast<a32 *>(a));
200}
201
202static atomic_uint64_t *to_atomic(const volatile a64 *a) {
203 return reinterpret_cast<atomic_uint64_t *>(const_cast<a64 *>(a));
204}
205
206static memory_order to_mo(morder mo) {
207 switch (mo) {
208 case mo_relaxed:
209 return memory_order_relaxed;
210 case mo_consume:
211 return memory_order_consume;
212 case mo_acquire:
213 return memory_order_acquire;
214 case mo_release:
215 return memory_order_release;
216 case mo_acq_rel:
217 return memory_order_acq_rel;
218 case mo_seq_cst:
219 return memory_order_seq_cst;
220 }
221 DCHECK(0);
222 return memory_order_seq_cst;
223}
224
225namespace {
226
227template <typename T, T (*F)(volatile T *v, T op)>
228static T AtomicRMW(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) {
229 MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(), kAccessWrite | kAccessAtomic);
230 if (LIKELY(mo == mo_relaxed))
231 return F(a, v);
232 SlotLocker locker(thr);
233 {
234 auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr: (uptr)a, save_stack: false);
235 RWLock lock(&s->mtx, IsReleaseOrder(mo));
236 if (IsAcqRelOrder(mo))
237 thr->clock.ReleaseAcquire(dstp: &s->clock);
238 else if (IsReleaseOrder(mo))
239 thr->clock.Release(dstp: &s->clock);
240 else if (IsAcquireOrder(mo))
241 thr->clock.Acquire(src: s->clock);
242 v = F(a, v);
243 }
244 if (IsReleaseOrder(mo))
245 IncrementEpoch(thr);
246 return v;
247}
248
249struct OpLoad {
250 template <typename T>
251 static T NoTsanAtomic(morder mo, const volatile T *a) {
252 return atomic_load(to_atomic(a), to_mo(mo));
253 }
254
255#if __TSAN_HAS_INT128
256 static a128 NoTsanAtomic(morder mo, const volatile a128 *a) {
257 SpinMutexLock lock(&mutex128);
258 return *a;
259 }
260#endif
261
262 template <typename T>
263 static T Atomic(ThreadState *thr, uptr pc, morder mo, const volatile T *a) {
264 DCHECK(IsLoadOrder(mo));
265 // This fast-path is critical for performance.
266 // Assume the access is atomic.
267 if (!IsAcquireOrder(mo)) {
268 MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(),
269 kAccessRead | kAccessAtomic);
270 return NoTsanAtomic(mo, a);
271 }
272 // Don't create sync object if it does not exist yet. For example, an atomic
273 // pointer is initialized to nullptr and then periodically acquire-loaded.
274 T v = NoTsanAtomic(mo, a);
275 SyncVar *s = ctx->metamap.GetSyncIfExists(addr: (uptr)a);
276 if (s) {
277 SlotLocker locker(thr);
278 ReadLock lock(&s->mtx);
279 thr->clock.Acquire(src: s->clock);
280 // Re-read under sync mutex because we need a consistent snapshot
281 // of the value and the clock we acquire.
282 v = NoTsanAtomic(mo, a);
283 }
284 MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(),
285 kAccessRead | kAccessAtomic);
286 return v;
287 }
288};
289
290struct OpStore {
291 template <typename T>
292 static void NoTsanAtomic(morder mo, volatile T *a, T v) {
293 atomic_store(to_atomic(a), v, to_mo(mo));
294 }
295
296#if __TSAN_HAS_INT128
297 static void NoTsanAtomic(morder mo, volatile a128 *a, a128 v) {
298 SpinMutexLock lock(&mutex128);
299 *a = v;
300 }
301#endif
302
303 template <typename T>
304 static void Atomic(ThreadState *thr, uptr pc, morder mo, volatile T *a, T v) {
305 DCHECK(IsStoreOrder(mo));
306 MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(),
307 kAccessWrite | kAccessAtomic);
308 // This fast-path is critical for performance.
309 // Assume the access is atomic.
310 // Strictly saying even relaxed store cuts off release sequence,
311 // so must reset the clock.
312 if (!IsReleaseOrder(mo)) {
313 NoTsanAtomic(mo, a, v);
314 return;
315 }
316 SlotLocker locker(thr);
317 {
318 auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr: (uptr)a, save_stack: false);
319 Lock lock(&s->mtx);
320 thr->clock.ReleaseStore(dstp: &s->clock);
321 NoTsanAtomic(mo, a, v);
322 }
323 IncrementEpoch(thr);
324 }
325};
326
327struct OpExchange {
328 template <typename T>
329 static T NoTsanAtomic(morder mo, volatile T *a, T v) {
330 return func_xchg(a, v);
331 }
332 template <typename T>
333 static T Atomic(ThreadState *thr, uptr pc, morder mo, volatile T *a, T v) {
334 return AtomicRMW<T, func_xchg>(thr, pc, a, v, mo);
335 }
336};
337
338struct OpFetchAdd {
339 template <typename T>
340 static T NoTsanAtomic(morder mo, volatile T *a, T v) {
341 return func_add(a, v);
342 }
343
344 template <typename T>
345 static T Atomic(ThreadState *thr, uptr pc, morder mo, volatile T *a, T v) {
346 return AtomicRMW<T, func_add>(thr, pc, a, v, mo);
347 }
348};
349
350struct OpFetchSub {
351 template <typename T>
352 [[maybe_unused]] static T NoTsanAtomic(morder mo, volatile T* a, T v) {
353 return func_sub(a, v);
354 }
355
356 template <typename T>
357 [[maybe_unused]] static T Atomic(ThreadState* thr, uptr pc, morder mo,
358 volatile T* a, T v) {
359 return AtomicRMW<T, func_sub>(thr, pc, a, v, mo);
360 }
361};
362
363struct OpFetchAnd {
364 template <typename T>
365 static T NoTsanAtomic(morder mo, volatile T *a, T v) {
366 return func_and(a, v);
367 }
368
369 template <typename T>
370 static T Atomic(ThreadState *thr, uptr pc, morder mo, volatile T *a, T v) {
371 return AtomicRMW<T, func_and>(thr, pc, a, v, mo);
372 }
373};
374
375struct OpFetchOr {
376 template <typename T>
377 static T NoTsanAtomic(morder mo, volatile T *a, T v) {
378 return func_or(a, v);
379 }
380
381 template <typename T>
382 static T Atomic(ThreadState *thr, uptr pc, morder mo, volatile T *a, T v) {
383 return AtomicRMW<T, func_or>(thr, pc, a, v, mo);
384 }
385};
386
387struct OpFetchXor {
388 template <typename T>
389 [[maybe_unused]] static T NoTsanAtomic(morder mo, volatile T* a, T v) {
390 return func_xor(a, v);
391 }
392
393 template <typename T>
394 [[maybe_unused]] static T Atomic(ThreadState* thr, uptr pc, morder mo,
395 volatile T* a, T v) {
396 return AtomicRMW<T, func_xor>(thr, pc, a, v, mo);
397 }
398};
399
400struct OpFetchNand {
401 template <typename T>
402 [[maybe_unused]] static T NoTsanAtomic(morder mo, volatile T* a, T v) {
403 return func_nand(a, v);
404 }
405
406 template <typename T>
407 [[maybe_unused]] static T Atomic(ThreadState* thr, uptr pc, morder mo,
408 volatile T* a, T v) {
409 return AtomicRMW<T, func_nand>(thr, pc, a, v, mo);
410 }
411};
412
413struct OpCAS {
414 template <typename T>
415 static bool NoTsanAtomic(morder mo, morder fmo, volatile T *a, T *c, T v) {
416 return atomic_compare_exchange_strong(to_atomic(a), c, v, to_mo(mo));
417 }
418
419#if __TSAN_HAS_INT128
420 static bool NoTsanAtomic(morder mo, morder fmo, volatile a128 *a, a128 *c,
421 a128 v) {
422 a128 old = *c;
423 a128 cur = func_cas(v: a, cmp: old, xch: v);
424 if (cur == old)
425 return true;
426 *c = cur;
427 return false;
428 }
429#endif
430
431 template <typename T>
432 static T NoTsanAtomic(morder mo, morder fmo, volatile T *a, T c, T v) {
433 NoTsanAtomic(mo, fmo, a, &c, v);
434 return c;
435 }
436
437 template <typename T>
438 static bool Atomic(ThreadState *thr, uptr pc, morder mo, morder fmo,
439 volatile T *a, T *c, T v) {
440 // 31.7.2.18: "The failure argument shall not be memory_order_release
441 // nor memory_order_acq_rel". LLVM (2021-05) fallbacks to Monotonic
442 // (mo_relaxed) when those are used.
443 DCHECK(IsLoadOrder(fmo));
444
445 MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(),
446 kAccessWrite | kAccessAtomic);
447 if (LIKELY(mo == mo_relaxed && fmo == mo_relaxed)) {
448 T cc = *c;
449 T pr = func_cas(a, cc, v);
450 if (pr == cc)
451 return true;
452 *c = pr;
453 return false;
454 }
455 SlotLocker locker(thr);
456 bool release = IsReleaseOrder(mo);
457 bool success;
458 {
459 auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr: (uptr)a, save_stack: false);
460 RWLock lock(&s->mtx, release);
461 T cc = *c;
462 T pr = func_cas(a, cc, v);
463 success = pr == cc;
464 if (!success) {
465 *c = pr;
466 mo = fmo;
467 }
468 if (success && IsAcqRelOrder(mo))
469 thr->clock.ReleaseAcquire(dstp: &s->clock);
470 else if (success && IsReleaseOrder(mo))
471 thr->clock.Release(dstp: &s->clock);
472 else if (IsAcquireOrder(mo))
473 thr->clock.Acquire(src: s->clock);
474 }
475 if (success && release)
476 IncrementEpoch(thr);
477 return success;
478 }
479
480 template <typename T>
481 static T Atomic(ThreadState *thr, uptr pc, morder mo, morder fmo,
482 volatile T *a, T c, T v) {
483 Atomic(thr, pc, mo, fmo, a, &c, v);
484 return c;
485 }
486};
487
488#if !SANITIZER_GO
489struct OpFence {
490 static void NoTsanAtomic(morder mo) { __sync_synchronize(); }
491
492 static void Atomic(ThreadState *thr, uptr pc, morder mo) {
493 // FIXME(dvyukov): not implemented.
494 __sync_synchronize();
495 }
496};
497#endif
498
499} // namespace
500
501// Interface functions follow.
502#if !SANITIZER_GO
503
504// C/C++
505
506static morder convert_morder(morder mo) {
507 return flags()->force_seq_cst_atomics ? mo_seq_cst : mo;
508}
509
510static morder to_morder(int mo) {
511 // Filter out additional memory order flags:
512 // MEMMODEL_SYNC = 1 << 15
513 // __ATOMIC_HLE_ACQUIRE = 1 << 16
514 // __ATOMIC_HLE_RELEASE = 1 << 17
515 //
516 // HLE is an optimization, and we pretend that elision always fails.
517 // MEMMODEL_SYNC is used when lowering __sync_ atomics,
518 // since we use __sync_ atomics for actual atomic operations,
519 // we can safely ignore it as well. It also subtly affects semantics,
520 // but we don't model the difference.
521 morder res = static_cast<morder>(static_cast<u8>(mo));
522 DCHECK_LE(res, mo_seq_cst);
523 return res;
524}
525
526template <class... Types>
527ALWAYS_INLINE auto AtomicDelayImpl(morder mo, Types... args) {
528 AdaptiveDelay::AtomicOpFence(mo);
529}
530
531template <class AddrType, class... Types>
532ALWAYS_INLINE auto AtomicDelayImpl(morder mo, AddrType addr, Types... args) {
533 AdaptiveDelay::AtomicOpAddr(addr: (uptr)addr, mo: (int)mo);
534}
535
536template <class Op, class... Types>
537ALWAYS_INLINE auto AtomicImpl(morder mo, Types... args) {
538 AtomicDelayImpl(mo, args...);
539 ThreadState *const thr = cur_thread();
540 ProcessPendingSignals(thr);
541 if (UNLIKELY(thr->ignore_sync || thr->ignore_interceptors))
542 return Op::NoTsanAtomic(mo, args...);
543 return Op::Atomic(thr, GET_CALLER_PC(), convert_morder(mo), args...);
544}
545
546extern "C" {
547SANITIZER_INTERFACE_ATTRIBUTE
548a8 __tsan_atomic8_load(const volatile a8 *a, int mo) {
549 return AtomicImpl<OpLoad>(mo: to_morder(mo), args: a);
550}
551
552SANITIZER_INTERFACE_ATTRIBUTE
553a16 __tsan_atomic16_load(const volatile a16 *a, int mo) {
554 return AtomicImpl<OpLoad>(mo: to_morder(mo), args: a);
555}
556
557SANITIZER_INTERFACE_ATTRIBUTE
558a32 __tsan_atomic32_load(const volatile a32 *a, int mo) {
559 return AtomicImpl<OpLoad>(mo: to_morder(mo), args: a);
560}
561
562SANITIZER_INTERFACE_ATTRIBUTE
563a64 __tsan_atomic64_load(const volatile a64 *a, int mo) {
564 return AtomicImpl<OpLoad>(mo: to_morder(mo), args: a);
565}
566
567# if __TSAN_HAS_INT128
568SANITIZER_INTERFACE_ATTRIBUTE
569a128 __tsan_atomic128_load(const volatile a128 *a, int mo) {
570 return AtomicImpl<OpLoad>(mo: to_morder(mo), args: a);
571}
572# endif
573
574SANITIZER_INTERFACE_ATTRIBUTE
575void __tsan_atomic8_store(volatile a8 *a, a8 v, int mo) {
576 return AtomicImpl<OpStore>(mo: to_morder(mo), args: a, args: v);
577}
578
579SANITIZER_INTERFACE_ATTRIBUTE
580void __tsan_atomic16_store(volatile a16 *a, a16 v, int mo) {
581 return AtomicImpl<OpStore>(mo: to_morder(mo), args: a, args: v);
582}
583
584SANITIZER_INTERFACE_ATTRIBUTE
585void __tsan_atomic32_store(volatile a32 *a, a32 v, int mo) {
586 return AtomicImpl<OpStore>(mo: to_morder(mo), args: a, args: v);
587}
588
589SANITIZER_INTERFACE_ATTRIBUTE
590void __tsan_atomic64_store(volatile a64 *a, a64 v, int mo) {
591 return AtomicImpl<OpStore>(mo: to_morder(mo), args: a, args: v);
592}
593
594# if __TSAN_HAS_INT128
595SANITIZER_INTERFACE_ATTRIBUTE
596void __tsan_atomic128_store(volatile a128 *a, a128 v, int mo) {
597 return AtomicImpl<OpStore>(mo: to_morder(mo), args: a, args: v);
598}
599# endif
600
601SANITIZER_INTERFACE_ATTRIBUTE
602a8 __tsan_atomic8_exchange(volatile a8 *a, a8 v, int mo) {
603 return AtomicImpl<OpExchange>(mo: to_morder(mo), args: a, args: v);
604}
605
606SANITIZER_INTERFACE_ATTRIBUTE
607a16 __tsan_atomic16_exchange(volatile a16 *a, a16 v, int mo) {
608 return AtomicImpl<OpExchange>(mo: to_morder(mo), args: a, args: v);
609}
610
611SANITIZER_INTERFACE_ATTRIBUTE
612a32 __tsan_atomic32_exchange(volatile a32 *a, a32 v, int mo) {
613 return AtomicImpl<OpExchange>(mo: to_morder(mo), args: a, args: v);
614}
615
616SANITIZER_INTERFACE_ATTRIBUTE
617a64 __tsan_atomic64_exchange(volatile a64 *a, a64 v, int mo) {
618 return AtomicImpl<OpExchange>(mo: to_morder(mo), args: a, args: v);
619}
620
621# if __TSAN_HAS_INT128
622SANITIZER_INTERFACE_ATTRIBUTE
623a128 __tsan_atomic128_exchange(volatile a128 *a, a128 v, int mo) {
624 return AtomicImpl<OpExchange>(mo: to_morder(mo), args: a, args: v);
625}
626# endif
627
628SANITIZER_INTERFACE_ATTRIBUTE
629a8 __tsan_atomic8_fetch_add(volatile a8 *a, a8 v, int mo) {
630 return AtomicImpl<OpFetchAdd>(mo: to_morder(mo), args: a, args: v);
631}
632
633SANITIZER_INTERFACE_ATTRIBUTE
634a16 __tsan_atomic16_fetch_add(volatile a16 *a, a16 v, int mo) {
635 return AtomicImpl<OpFetchAdd>(mo: to_morder(mo), args: a, args: v);
636}
637
638SANITIZER_INTERFACE_ATTRIBUTE
639a32 __tsan_atomic32_fetch_add(volatile a32 *a, a32 v, int mo) {
640 return AtomicImpl<OpFetchAdd>(mo: to_morder(mo), args: a, args: v);
641}
642
643SANITIZER_INTERFACE_ATTRIBUTE
644a64 __tsan_atomic64_fetch_add(volatile a64 *a, a64 v, int mo) {
645 return AtomicImpl<OpFetchAdd>(mo: to_morder(mo), args: a, args: v);
646}
647
648# if __TSAN_HAS_INT128
649SANITIZER_INTERFACE_ATTRIBUTE
650a128 __tsan_atomic128_fetch_add(volatile a128 *a, a128 v, int mo) {
651 return AtomicImpl<OpFetchAdd>(mo: to_morder(mo), args: a, args: v);
652}
653# endif
654
655SANITIZER_INTERFACE_ATTRIBUTE
656a8 __tsan_atomic8_fetch_sub(volatile a8 *a, a8 v, int mo) {
657 return AtomicImpl<OpFetchSub>(mo: to_morder(mo), args: a, args: v);
658}
659
660SANITIZER_INTERFACE_ATTRIBUTE
661a16 __tsan_atomic16_fetch_sub(volatile a16 *a, a16 v, int mo) {
662 return AtomicImpl<OpFetchSub>(mo: to_morder(mo), args: a, args: v);
663}
664
665SANITIZER_INTERFACE_ATTRIBUTE
666a32 __tsan_atomic32_fetch_sub(volatile a32 *a, a32 v, int mo) {
667 return AtomicImpl<OpFetchSub>(mo: to_morder(mo), args: a, args: v);
668}
669
670SANITIZER_INTERFACE_ATTRIBUTE
671a64 __tsan_atomic64_fetch_sub(volatile a64 *a, a64 v, int mo) {
672 return AtomicImpl<OpFetchSub>(mo: to_morder(mo), args: a, args: v);
673}
674
675# if __TSAN_HAS_INT128
676SANITIZER_INTERFACE_ATTRIBUTE
677a128 __tsan_atomic128_fetch_sub(volatile a128 *a, a128 v, int mo) {
678 return AtomicImpl<OpFetchSub>(mo: to_morder(mo), args: a, args: v);
679}
680# endif
681
682SANITIZER_INTERFACE_ATTRIBUTE
683a8 __tsan_atomic8_fetch_and(volatile a8 *a, a8 v, int mo) {
684 return AtomicImpl<OpFetchAnd>(mo: to_morder(mo), args: a, args: v);
685}
686
687SANITIZER_INTERFACE_ATTRIBUTE
688a16 __tsan_atomic16_fetch_and(volatile a16 *a, a16 v, int mo) {
689 return AtomicImpl<OpFetchAnd>(mo: to_morder(mo), args: a, args: v);
690}
691
692SANITIZER_INTERFACE_ATTRIBUTE
693a32 __tsan_atomic32_fetch_and(volatile a32 *a, a32 v, int mo) {
694 return AtomicImpl<OpFetchAnd>(mo: to_morder(mo), args: a, args: v);
695}
696
697SANITIZER_INTERFACE_ATTRIBUTE
698a64 __tsan_atomic64_fetch_and(volatile a64 *a, a64 v, int mo) {
699 return AtomicImpl<OpFetchAnd>(mo: to_morder(mo), args: a, args: v);
700}
701
702# if __TSAN_HAS_INT128
703SANITIZER_INTERFACE_ATTRIBUTE
704a128 __tsan_atomic128_fetch_and(volatile a128 *a, a128 v, int mo) {
705 return AtomicImpl<OpFetchAnd>(mo: to_morder(mo), args: a, args: v);
706}
707# endif
708
709SANITIZER_INTERFACE_ATTRIBUTE
710a8 __tsan_atomic8_fetch_or(volatile a8 *a, a8 v, int mo) {
711 return AtomicImpl<OpFetchOr>(mo: to_morder(mo), args: a, args: v);
712}
713
714SANITIZER_INTERFACE_ATTRIBUTE
715a16 __tsan_atomic16_fetch_or(volatile a16 *a, a16 v, int mo) {
716 return AtomicImpl<OpFetchOr>(mo: to_morder(mo), args: a, args: v);
717}
718
719SANITIZER_INTERFACE_ATTRIBUTE
720a32 __tsan_atomic32_fetch_or(volatile a32 *a, a32 v, int mo) {
721 return AtomicImpl<OpFetchOr>(mo: to_morder(mo), args: a, args: v);
722}
723
724SANITIZER_INTERFACE_ATTRIBUTE
725a64 __tsan_atomic64_fetch_or(volatile a64 *a, a64 v, int mo) {
726 return AtomicImpl<OpFetchOr>(mo: to_morder(mo), args: a, args: v);
727}
728
729# if __TSAN_HAS_INT128
730SANITIZER_INTERFACE_ATTRIBUTE
731a128 __tsan_atomic128_fetch_or(volatile a128 *a, a128 v, int mo) {
732 return AtomicImpl<OpFetchOr>(mo: to_morder(mo), args: a, args: v);
733}
734# endif
735
736SANITIZER_INTERFACE_ATTRIBUTE
737a8 __tsan_atomic8_fetch_xor(volatile a8 *a, a8 v, int mo) {
738 return AtomicImpl<OpFetchXor>(mo: to_morder(mo), args: a, args: v);
739}
740
741SANITIZER_INTERFACE_ATTRIBUTE
742a16 __tsan_atomic16_fetch_xor(volatile a16 *a, a16 v, int mo) {
743 return AtomicImpl<OpFetchXor>(mo: to_morder(mo), args: a, args: v);
744}
745
746SANITIZER_INTERFACE_ATTRIBUTE
747a32 __tsan_atomic32_fetch_xor(volatile a32 *a, a32 v, int mo) {
748 return AtomicImpl<OpFetchXor>(mo: to_morder(mo), args: a, args: v);
749}
750
751SANITIZER_INTERFACE_ATTRIBUTE
752a64 __tsan_atomic64_fetch_xor(volatile a64 *a, a64 v, int mo) {
753 return AtomicImpl<OpFetchXor>(mo: to_morder(mo), args: a, args: v);
754}
755
756# if __TSAN_HAS_INT128
757SANITIZER_INTERFACE_ATTRIBUTE
758a128 __tsan_atomic128_fetch_xor(volatile a128 *a, a128 v, int mo) {
759 return AtomicImpl<OpFetchXor>(mo: to_morder(mo), args: a, args: v);
760}
761# endif
762
763SANITIZER_INTERFACE_ATTRIBUTE
764a8 __tsan_atomic8_fetch_nand(volatile a8 *a, a8 v, int mo) {
765 return AtomicImpl<OpFetchNand>(mo: to_morder(mo), args: a, args: v);
766}
767
768SANITIZER_INTERFACE_ATTRIBUTE
769a16 __tsan_atomic16_fetch_nand(volatile a16 *a, a16 v, int mo) {
770 return AtomicImpl<OpFetchNand>(mo: to_morder(mo), args: a, args: v);
771}
772
773SANITIZER_INTERFACE_ATTRIBUTE
774a32 __tsan_atomic32_fetch_nand(volatile a32 *a, a32 v, int mo) {
775 return AtomicImpl<OpFetchNand>(mo: to_morder(mo), args: a, args: v);
776}
777
778SANITIZER_INTERFACE_ATTRIBUTE
779a64 __tsan_atomic64_fetch_nand(volatile a64 *a, a64 v, int mo) {
780 return AtomicImpl<OpFetchNand>(mo: to_morder(mo), args: a, args: v);
781}
782
783# if __TSAN_HAS_INT128
784SANITIZER_INTERFACE_ATTRIBUTE
785a128 __tsan_atomic128_fetch_nand(volatile a128 *a, a128 v, int mo) {
786 return AtomicImpl<OpFetchNand>(mo: to_morder(mo), args: a, args: v);
787}
788# endif
789
790SANITIZER_INTERFACE_ATTRIBUTE
791int __tsan_atomic8_compare_exchange_strong(volatile a8 *a, a8 *c, a8 v, int mo,
792 int fmo) {
793 return AtomicImpl<OpCAS>(mo: to_morder(mo), args: to_morder(mo: fmo), args: a, args: c, args: v);
794}
795
796SANITIZER_INTERFACE_ATTRIBUTE
797int __tsan_atomic16_compare_exchange_strong(volatile a16 *a, a16 *c, a16 v,
798 int mo, int fmo) {
799 return AtomicImpl<OpCAS>(mo: to_morder(mo), args: to_morder(mo: fmo), args: a, args: c, args: v);
800}
801
802SANITIZER_INTERFACE_ATTRIBUTE
803int __tsan_atomic32_compare_exchange_strong(volatile a32 *a, a32 *c, a32 v,
804 int mo, int fmo) {
805 return AtomicImpl<OpCAS>(mo: to_morder(mo), args: to_morder(mo: fmo), args: a, args: c, args: v);
806}
807
808SANITIZER_INTERFACE_ATTRIBUTE
809int __tsan_atomic64_compare_exchange_strong(volatile a64 *a, a64 *c, a64 v,
810 int mo, int fmo) {
811 return AtomicImpl<OpCAS>(mo: to_morder(mo), args: to_morder(mo: fmo), args: a, args: c, args: v);
812}
813
814# if __TSAN_HAS_INT128
815SANITIZER_INTERFACE_ATTRIBUTE
816int __tsan_atomic128_compare_exchange_strong(volatile a128 *a, a128 *c, a128 v,
817 int mo, int fmo) {
818 return AtomicImpl<OpCAS>(mo: to_morder(mo), args: to_morder(mo: fmo), args: a, args: c, args: v);
819}
820# endif
821
822SANITIZER_INTERFACE_ATTRIBUTE
823int __tsan_atomic8_compare_exchange_weak(volatile a8 *a, a8 *c, a8 v, int mo,
824 int fmo) {
825 return AtomicImpl<OpCAS>(mo: to_morder(mo), args: to_morder(mo: fmo), args: a, args: c, args: v);
826}
827
828SANITIZER_INTERFACE_ATTRIBUTE
829int __tsan_atomic16_compare_exchange_weak(volatile a16 *a, a16 *c, a16 v,
830 int mo, int fmo) {
831 return AtomicImpl<OpCAS>(mo: to_morder(mo), args: to_morder(mo: fmo), args: a, args: c, args: v);
832}
833
834SANITIZER_INTERFACE_ATTRIBUTE
835int __tsan_atomic32_compare_exchange_weak(volatile a32 *a, a32 *c, a32 v,
836 int mo, int fmo) {
837 return AtomicImpl<OpCAS>(mo: to_morder(mo), args: to_morder(mo: fmo), args: a, args: c, args: v);
838}
839
840SANITIZER_INTERFACE_ATTRIBUTE
841int __tsan_atomic64_compare_exchange_weak(volatile a64 *a, a64 *c, a64 v,
842 int mo, int fmo) {
843 return AtomicImpl<OpCAS>(mo: to_morder(mo), args: to_morder(mo: fmo), args: a, args: c, args: v);
844}
845
846# if __TSAN_HAS_INT128
847SANITIZER_INTERFACE_ATTRIBUTE
848int __tsan_atomic128_compare_exchange_weak(volatile a128 *a, a128 *c, a128 v,
849 int mo, int fmo) {
850 return AtomicImpl<OpCAS>(mo: to_morder(mo), args: to_morder(mo: fmo), args: a, args: c, args: v);
851}
852# endif
853
854SANITIZER_INTERFACE_ATTRIBUTE
855a8 __tsan_atomic8_compare_exchange_val(volatile a8 *a, a8 c, a8 v, int mo,
856 int fmo) {
857 return AtomicImpl<OpCAS>(mo: to_morder(mo), args: to_morder(mo: fmo), args: a, args: c, args: v);
858}
859
860SANITIZER_INTERFACE_ATTRIBUTE
861a16 __tsan_atomic16_compare_exchange_val(volatile a16 *a, a16 c, a16 v, int mo,
862 int fmo) {
863 return AtomicImpl<OpCAS>(mo: to_morder(mo), args: to_morder(mo: fmo), args: a, args: c, args: v);
864}
865
866SANITIZER_INTERFACE_ATTRIBUTE
867a32 __tsan_atomic32_compare_exchange_val(volatile a32 *a, a32 c, a32 v, int mo,
868 int fmo) {
869 return AtomicImpl<OpCAS>(mo: to_morder(mo), args: to_morder(mo: fmo), args: a, args: c, args: v);
870}
871
872SANITIZER_INTERFACE_ATTRIBUTE
873a64 __tsan_atomic64_compare_exchange_val(volatile a64 *a, a64 c, a64 v, int mo,
874 int fmo) {
875 return AtomicImpl<OpCAS>(mo: to_morder(mo), args: to_morder(mo: fmo), args: a, args: c, args: v);
876}
877
878# if __TSAN_HAS_INT128
879SANITIZER_INTERFACE_ATTRIBUTE
880a128 __tsan_atomic128_compare_exchange_val(volatile a128 *a, a128 c, a128 v,
881 int mo, int fmo) {
882 return AtomicImpl<OpCAS>(mo: to_morder(mo), args: to_morder(mo: fmo), args: a, args: c, args: v);
883}
884# endif
885
886SANITIZER_INTERFACE_ATTRIBUTE
887void __tsan_atomic_thread_fence(int mo) {
888 return AtomicImpl<OpFence>(mo: to_morder(mo));
889}
890
891SANITIZER_INTERFACE_ATTRIBUTE
892void __tsan_atomic_signal_fence(int mo) {}
893} // extern "C"
894
895#else // #if !SANITIZER_GO
896
897// Go
898
899template <class Op, class... Types>
900void AtomicGo(ThreadState *thr, uptr cpc, uptr pc, Types... args) {
901 if (thr->ignore_sync) {
902 (void)Op::NoTsanAtomic(args...);
903 } else {
904 FuncEntry(thr, cpc);
905 (void)Op::Atomic(thr, pc, args...);
906 FuncExit(thr);
907 }
908}
909
910template <class Op, class... Types>
911auto AtomicGoRet(ThreadState *thr, uptr cpc, uptr pc, Types... args) {
912 if (thr->ignore_sync) {
913 return Op::NoTsanAtomic(args...);
914 } else {
915 FuncEntry(thr, cpc);
916 auto ret = Op::Atomic(thr, pc, args...);
917 FuncExit(thr);
918 return ret;
919 }
920}
921
922extern "C" {
923SANITIZER_INTERFACE_ATTRIBUTE
924void __tsan_go_atomic32_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
925 *(a32 *)(a + 8) = AtomicGoRet<OpLoad>(thr, cpc, pc, mo_acquire, *(a32 **)a);
926}
927
928SANITIZER_INTERFACE_ATTRIBUTE
929void __tsan_go_atomic64_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
930 *(a64 *)(a + 8) = AtomicGoRet<OpLoad>(thr, cpc, pc, mo_acquire, *(a64 **)a);
931}
932
933# if __TSAN_HAS_INT128
934SANITIZER_INTERFACE_ATTRIBUTE
935void __tsan_go_atomic128_load(ThreadState* thr, uptr cpc, uptr pc, u8* a) {
936 *(a128*)(a + 8) = AtomicGoRet<OpLoad>(thr, cpc, pc, mo_acquire, *(a128**)a);
937}
938# endif
939
940SANITIZER_INTERFACE_ATTRIBUTE
941void __tsan_go_atomic32_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
942 AtomicGo<OpStore>(thr, cpc, pc, mo_release, *(a32 **)a, *(a32 *)(a + 8));
943}
944
945SANITIZER_INTERFACE_ATTRIBUTE
946void __tsan_go_atomic64_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
947 AtomicGo<OpStore>(thr, cpc, pc, mo_release, *(a64 **)a, *(a64 *)(a + 8));
948}
949
950# if __TSAN_HAS_INT128
951SANITIZER_INTERFACE_ATTRIBUTE
952void __tsan_go_atomic128_store(ThreadState* thr, uptr cpc, uptr pc, u8* a) {
953 AtomicGo<OpStore>(thr, cpc, pc, mo_release, *(a128**)a, *(a128*)(a + 8));
954}
955# endif
956
957SANITIZER_INTERFACE_ATTRIBUTE
958void __tsan_go_atomic32_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
959 *(a32 *)(a + 16) = AtomicGoRet<OpFetchAdd>(thr, cpc, pc, mo_acq_rel,
960 *(a32 **)a, *(a32 *)(a + 8));
961}
962
963SANITIZER_INTERFACE_ATTRIBUTE
964void __tsan_go_atomic64_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
965 *(a64 *)(a + 16) = AtomicGoRet<OpFetchAdd>(thr, cpc, pc, mo_acq_rel,
966 *(a64 **)a, *(a64 *)(a + 8));
967}
968
969SANITIZER_INTERFACE_ATTRIBUTE
970void __tsan_go_atomic32_fetch_and(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
971 *(a32 *)(a + 16) = AtomicGoRet<OpFetchAnd>(thr, cpc, pc, mo_acq_rel,
972 *(a32 **)a, *(a32 *)(a + 8));
973}
974
975SANITIZER_INTERFACE_ATTRIBUTE
976void __tsan_go_atomic64_fetch_and(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
977 *(a64 *)(a + 16) = AtomicGoRet<OpFetchAnd>(thr, cpc, pc, mo_acq_rel,
978 *(a64 **)a, *(a64 *)(a + 8));
979}
980
981SANITIZER_INTERFACE_ATTRIBUTE
982void __tsan_go_atomic32_fetch_or(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
983 *(a32 *)(a + 16) = AtomicGoRet<OpFetchOr>(thr, cpc, pc, mo_acq_rel,
984 *(a32 **)a, *(a32 *)(a + 8));
985}
986
987SANITIZER_INTERFACE_ATTRIBUTE
988void __tsan_go_atomic64_fetch_or(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
989 *(a64 *)(a + 16) = AtomicGoRet<OpFetchOr>(thr, cpc, pc, mo_acq_rel,
990 *(a64 **)a, *(a64 *)(a + 8));
991}
992
993SANITIZER_INTERFACE_ATTRIBUTE
994void __tsan_go_atomic32_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
995 *(a32 *)(a + 16) = AtomicGoRet<OpExchange>(thr, cpc, pc, mo_acq_rel,
996 *(a32 **)a, *(a32 *)(a + 8));
997}
998
999SANITIZER_INTERFACE_ATTRIBUTE
1000void __tsan_go_atomic64_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
1001 *(a64 *)(a + 16) = AtomicGoRet<OpExchange>(thr, cpc, pc, mo_acq_rel,
1002 *(a64 **)a, *(a64 *)(a + 8));
1003}
1004
1005SANITIZER_INTERFACE_ATTRIBUTE
1006void __tsan_go_atomic32_compare_exchange(ThreadState *thr, uptr cpc, uptr pc,
1007 u8 *a) {
1008 a32 cmp = *(a32 *)(a + 8);
1009 a32 cur = AtomicGoRet<OpCAS>(thr, cpc, pc, mo_acq_rel, mo_acquire, *(a32 **)a,
1010 cmp, *(a32 *)(a + 12));
1011 *(bool *)(a + 16) = (cur == cmp);
1012}
1013
1014SANITIZER_INTERFACE_ATTRIBUTE
1015void __tsan_go_atomic64_compare_exchange(ThreadState *thr, uptr cpc, uptr pc,
1016 u8 *a) {
1017 a64 cmp = *(a64 *)(a + 8);
1018 a64 cur = AtomicGoRet<OpCAS>(thr, cpc, pc, mo_acq_rel, mo_acquire, *(a64 **)a,
1019 cmp, *(a64 *)(a + 16));
1020 *(bool *)(a + 24) = (cur == cmp);
1021}
1022
1023# if __TSAN_HAS_INT128
1024SANITIZER_INTERFACE_ATTRIBUTE
1025void __tsan_go_atomic128_compare_exchange(ThreadState* thr, uptr cpc, uptr pc,
1026 u8* a) {
1027 a128 cmp = *(a128*)(a + 8);
1028 a128 cur = AtomicGoRet<OpCAS>(thr, cpc, pc, mo_acq_rel, mo_acquire,
1029 *(a128**)a, cmp, *(a128*)(a + 24));
1030 *(bool*)(a + 40) = (cur == cmp);
1031}
1032# endif
1033} // extern "C"
1034#endif // #if !SANITIZER_GO
1035