1//===- ConcurrencySanitizer.cpp - watchpoint race detector ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements instrumentation for ConcurrencySanitizer, a sampling
10// data-race detector. Memory accesses are preceded by runtime probes while the
11// original operations remain unchanged.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/Transforms/Instrumentation/ConcurrencySanitizer.h"
16#include "llvm/ADT/BitmaskEnum.h"
17#include "llvm/ADT/DenseMap.h"
18#include "llvm/ADT/SmallVector.h"
19#include "llvm/ADT/Statistic.h"
20#include "llvm/ADT/bit.h"
21#include "llvm/Analysis/CaptureTracking.h"
22#include "llvm/Analysis/TargetLibraryInfo.h"
23#include "llvm/Analysis/ValueTracking.h"
24#include "llvm/IR/DataLayout.h"
25#include "llvm/IR/IRBuilder.h"
26#include "llvm/IR/Instructions.h"
27#include "llvm/IR/IntrinsicInst.h"
28#include "llvm/IR/Intrinsics.h"
29#include "llvm/IR/Module.h"
30#include "llvm/ProfileData/InstrProf.h"
31#include "llvm/Support/AMDGPUAddrSpace.h"
32#include "llvm/Support/AtomicOrdering.h"
33#include "llvm/Support/CommandLine.h"
34#include "llvm/Support/NVPTXAddrSpace.h"
35#include "llvm/TargetParser/Triple.h"
36#include "llvm/Transforms/Utils/EscapeEnumerator.h"
37#include "llvm/Transforms/Utils/Instrumentation.h"
38#include "llvm/Transforms/Utils/Local.h"
39#include "llvm/Transforms/Utils/ModuleUtils.h"
40#include <optional>
41
42using namespace llvm;
43
44#define DEBUG_TYPE "csan"
45
46static cl::opt<bool> ClInstrumentMemoryAccesses(
47 "csan-instrument-memory-accesses", cl::init(Val: true),
48 cl::desc("Instrument memory accesses"), cl::Hidden);
49static cl::opt<bool>
50 ClInstrumentFuncEntryExit("csan-instrument-func-entry-exit", cl::init(Val: true),
51 cl::desc("Instrument function entry and exit"),
52 cl::Hidden);
53static cl::opt<bool> ClHandleCxxExceptions(
54 "csan-handle-cxx-exceptions", cl::init(Val: true),
55 cl::desc("Handle C++ exceptions (insert cleanup blocks for unwinding)"),
56 cl::Hidden);
57static cl::opt<bool> ClInstrumentAtomics("csan-instrument-atomics",
58 cl::init(Val: true),
59 cl::desc("Instrument atomics"),
60 cl::Hidden);
61static cl::opt<bool> ClInstrumentMemIntrinsics(
62 "csan-instrument-memintrinsics", cl::init(Val: true),
63 cl::desc("Instrument memintrinsics (memset/memcpy/memmove)"), cl::Hidden);
64static cl::opt<bool> ClDistinguishVolatile(
65 "csan-distinguish-volatile", cl::init(Val: false),
66 cl::desc("Treat aligned volatile accesses up to 8 bytes as atomic"),
67 cl::Hidden);
68STATISTIC(NumInstrumentedReads, "Number of instrumented reads");
69STATISTIC(NumInstrumentedWrites, "Number of instrumented writes");
70STATISTIC(NumOmittedReadsBeforeWrite,
71 "Number of reads ignored due to following writes");
72
73static constexpr char kCsanModuleCtorName[] = "csan.module_ctor";
74static constexpr char kCsanInitName[] = "__csan_init";
75
76// Accesses sizes are powers of two: 1, 2, 4, 8, 16.
77static constexpr size_t kNumAccessSizes = 5;
78static constexpr StringLiteral kCsanReadNames[kNumAccessSizes] = {
79 "__csan_read1", "__csan_read2", "__csan_read4", "__csan_read8",
80 "__csan_read16"};
81static constexpr StringLiteral kCsanWriteNames[kNumAccessSizes] = {
82 "__csan_write1", "__csan_write2", "__csan_write4", "__csan_write8",
83 "__csan_write16"};
84static constexpr StringLiteral kCsanUnalignedReadNames[kNumAccessSizes] = {
85 "__csan_unaligned_read1", "__csan_unaligned_read2",
86 "__csan_unaligned_read4", "__csan_unaligned_read8",
87 "__csan_unaligned_read16"};
88static constexpr StringLiteral kCsanUnalignedWriteNames[kNumAccessSizes] = {
89 "__csan_unaligned_write1", "__csan_unaligned_write2",
90 "__csan_unaligned_write4", "__csan_unaligned_write8",
91 "__csan_unaligned_write16"};
92static constexpr StringLiteral kCsanCompoundRWNames[kNumAccessSizes] = {
93 "__csan_read_write1", "__csan_read_write2", "__csan_read_write4",
94 "__csan_read_write8", "__csan_read_write16"};
95static constexpr StringLiteral kCsanUnalignedCompoundRWNames[kNumAccessSizes] =
96 {"__csan_unaligned_read_write1", "__csan_unaligned_read_write2",
97 "__csan_unaligned_read_write4", "__csan_unaligned_read_write8",
98 "__csan_unaligned_read_write16"};
99
100namespace {
101
102/// Must match CSAN_ACCESS_* in compiler-rt/lib/csan/csan_defs.h.
103enum class AccessFlags : unsigned {
104 None = 0,
105 Atomic = 1u << 0,
106 Compound = 1u << 1,
107 LLVM_MARK_AS_BITMASK_ENUM(Compound),
108};
109
110static bool isAtomicMemoryAccess(const Instruction *I) {
111 std::optional<SyncScope::ID> SSID = getAtomicSyncScopeID(I);
112 if (!SSID)
113 return false;
114 if (isa<LoadInst>(Val: I) || isa<StoreInst>(Val: I))
115 return *SSID != SyncScope::SingleThread;
116 return true;
117}
118
119static Value *getCallbackAddress(IRBuilderBase &IRB, Value *Addr) {
120 return IRB.CreateAddrSpaceCast(V: Addr, DestTy: IRB.getPtrTy());
121}
122
123static ConstantInt *createOrdering(IRBuilderBase &IRB, AtomicOrdering Ord) {
124 return IRB.getInt32(C: static_cast<uint32_t>(toCABI(AO: Ord)));
125}
126
127static int getAccessSizeIndex(Type *Ty, const DataLayout &DL) {
128 assert(Ty->isSized());
129 if (Ty->isScalableTy())
130 return -1;
131 uint32_t TypeSize = DL.getTypeStoreSizeInBits(Ty);
132 if (TypeSize != 8 && TypeSize != 16 && TypeSize != 32 && TypeSize != 64 &&
133 TypeSize != 128)
134 return -1;
135 unsigned Idx = llvm::countr_zero(Val: TypeSize / 8);
136 return static_cast<int>(Idx);
137}
138
139// A later write of the same pointer drops an earlier read in this window.
140static void selectMemoryAccesses(SmallVectorImpl<Instruction *> &Local,
141 SmallVectorImpl<Instruction *> &Out) {
142 DenseMap<Value *, StoreInst *> WriteTargets;
143 for (Instruction *Inst : reverse(C&: Local)) {
144 Value *Addr = getLoadStorePointerOperand(V: Inst);
145 if (StoreInst *Store = dyn_cast<StoreInst>(Val: Inst)) {
146 WriteTargets[Addr] = Store;
147 Out.push_back(Elt: Inst);
148 continue;
149 }
150
151 // A volatile write is reported as atomic when distinguished, so it does
152 // not cover a plain read. Keep the read if either access is volatile.
153 StoreInst *LaterWrite = WriteTargets.lookup(Val: Addr);
154 bool AnyVolatile =
155 LaterWrite && ClDistinguishVolatile &&
156 (cast<LoadInst>(Val: Inst)->isVolatile() || LaterWrite->isVolatile());
157 if (LaterWrite && !AnyVolatile) {
158 ++NumOmittedReadsBeforeWrite;
159 continue;
160 }
161
162 Out.push_back(Elt: Inst);
163 }
164 Local.clear();
165}
166
167static bool addressSpaceMayRace(const Triple &T, unsigned AS) {
168 if (T.isAMDGPU()) {
169 // GDS and buffer fat pointers cannot form a generic watchpoint key.
170 return AS == AMDGPUAS::FLAT_ADDRESS || AS == AMDGPUAS::GLOBAL_ADDRESS ||
171 AS == AMDGPUAS::LOCAL_ADDRESS;
172 }
173 if (T.isNVPTX()) {
174 return AS == NVPTXAS::ADDRESS_SPACE_GENERIC ||
175 AS == NVPTXAS::ADDRESS_SPACE_GLOBAL ||
176 AS == NVPTXAS::ADDRESS_SPACE_SHARED ||
177 AS == NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER;
178 }
179 if (T.isSPIRV()) {
180 // FIXME: No exposed address spaces for SPIR-V.
181 return false;
182 }
183 return AS == 0;
184}
185
186struct ConcurrencySanitizer {
187 bool sanitizeFunction(Function &F, const TargetLibraryInfo &TLI);
188
189private:
190 struct MemoryAccessLists {
191 SmallVector<Instruction *, 8> LoadsAndStores;
192 SmallVector<Instruction *, 8> AtomicAccesses;
193 SmallVector<IntrinsicInst *, 8> MemIntrinCalls;
194 bool HasCalls = false;
195 };
196
197 void initialize(Module &M, const TargetLibraryInfo &TLI);
198 void collectMemoryAccesses(Function &F, MemoryAccessLists &Out);
199 bool instrumentLoadOrStore(Instruction *I, const DataLayout &DL);
200 bool instrumentAtomic(Instruction *I, const DataLayout &DL);
201 bool instrumentMemIntrinsic(IntrinsicInst *II, const DataLayout &DL);
202 bool insertAccessProbe(Instruction *I, Value *Addr, Type *AccessTy,
203 const DataLayout &DL, bool IsWrite, bool IsCompound,
204 bool IsAtomic);
205 bool shouldInstrumentAddress(Value *Addr) const;
206 bool shouldInstrumentAccess(Instruction *I) const;
207 void insertFuncEntryExit(Function &F);
208 void insertRuntimeIgnores(Function &F);
209 Constant *getFlags(AccessFlags Flags) const {
210 return ConstantInt::get(Ty: FlagsTy, V: to_underlying(E: Flags));
211 }
212
213 Module *Mod = nullptr;
214 Type *IntptrTy = nullptr;
215 IntegerType *FlagsTy = nullptr;
216
217 // void __csan_readN(ptr, i32);
218 FunctionCallee CsanRead[kNumAccessSizes];
219 // void __csan_writeN(ptr, i32);
220 FunctionCallee CsanWrite[kNumAccessSizes];
221 // void __csan_unaligned_readN(ptr, i32);
222 FunctionCallee CsanUnalignedRead[kNumAccessSizes];
223 // void __csan_unaligned_writeN(ptr, i32);
224 FunctionCallee CsanUnalignedWrite[kNumAccessSizes];
225 // void __csan_read_writeN(ptr, i32);
226 FunctionCallee CsanCompoundRW[kNumAccessSizes];
227 // void __csan_unaligned_read_writeN(ptr, i32);
228 FunctionCallee CsanUnalignedCompoundRW[kNumAccessSizes];
229 // void __csan_func_entry(ptr);
230 FunctionCallee CsanFuncEntry;
231 // void __csan_func_exit();
232 FunctionCallee CsanFuncExit;
233 // void __csan_ignore_thread_begin();
234 FunctionCallee CsanIgnoreBegin;
235 // void __csan_ignore_thread_end();
236 FunctionCallee CsanIgnoreEnd;
237 // void __csan_read_range(ptr, intptr_t, i32);
238 FunctionCallee CsanReadRange;
239 // void __csan_write_range(ptr, intptr_t, i32);
240 FunctionCallee CsanWriteRange;
241 // void __csan_atomic_thread_fence(i32);
242 FunctionCallee CsanAtomicThreadFence;
243 // void __csan_atomic_signal_fence(i32);
244 FunctionCallee CsanAtomicSignalFence;
245};
246
247void insertModuleCtor(Module &M) {
248 getOrCreateSanitizerCtorAndInitFunctions(
249 M, CtorName: kCsanModuleCtorName, InitName: kCsanInitName, /*InitArgTypes=*/{},
250 /*InitArgs=*/{},
251 FunctionsCreatedCallback: [&](Function *Ctor, FunctionCallee) { appendToGlobalCtors(M, F: Ctor, Priority: 0); });
252}
253
254} // namespace
255
256PreservedAnalyses ConcurrencySanitizerPass::run(Function &F,
257 FunctionAnalysisManager &FAM) {
258 ConcurrencySanitizer CSan;
259 if (CSan.sanitizeFunction(F, TLI: FAM.getResult<TargetLibraryAnalysis>(IR&: F)))
260 return PreservedAnalyses::none();
261 return PreservedAnalyses::all();
262}
263
264PreservedAnalyses ModuleConcurrencySanitizerPass::run(Module &M,
265 ModuleAnalysisManager &) {
266 if (checkIfAlreadyInstrumented(M, Flag: "nosanitize_concurrency"))
267 return PreservedAnalyses::all();
268 insertModuleCtor(M);
269 return PreservedAnalyses::none();
270}
271
272void ConcurrencySanitizer::initialize(Module &M, const TargetLibraryInfo &TLI) {
273 LLVMContext &Ctx = M.getContext();
274 Mod = &M;
275 IntptrTy = M.getDataLayout().getIntPtrType(C&: Ctx);
276 FlagsTy = Type::getInt32Ty(C&: Ctx);
277
278 AttributeList Attr = AttributeList().addFnAttribute(C&: Ctx, Kind: Attribute::NoUnwind);
279 IRBuilder<> IRB(M);
280 Type *VoidTy = IRB.getVoidTy();
281 Type *PtrTy = IRB.getPtrTy();
282 CsanFuncEntry =
283 M.getOrInsertFunction(Name: "__csan_func_entry", AttributeList: Attr, RetTy: VoidTy, Args: PtrTy);
284 CsanFuncExit = M.getOrInsertFunction(Name: "__csan_func_exit", AttributeList: Attr, RetTy: VoidTy);
285 CsanIgnoreBegin =
286 M.getOrInsertFunction(Name: "__csan_ignore_thread_begin", AttributeList: Attr, RetTy: VoidTy);
287 CsanIgnoreEnd =
288 M.getOrInsertFunction(Name: "__csan_ignore_thread_end", AttributeList: Attr, RetTy: VoidTy);
289 auto AccessFn = [&](StringRef Name) {
290 return M.getOrInsertFunction(Name, AttributeList: Attr, RetTy: VoidTy, Args: PtrTy, Args: FlagsTy);
291 };
292 for (unsigned I = 0; I < kNumAccessSizes; ++I) {
293 CsanRead[I] = AccessFn(kCsanReadNames[I]);
294 CsanWrite[I] = AccessFn(kCsanWriteNames[I]);
295 CsanUnalignedRead[I] = AccessFn(kCsanUnalignedReadNames[I]);
296 CsanUnalignedWrite[I] = AccessFn(kCsanUnalignedWriteNames[I]);
297 CsanCompoundRW[I] = AccessFn(kCsanCompoundRWNames[I]);
298 CsanUnalignedCompoundRW[I] = AccessFn(kCsanUnalignedCompoundRWNames[I]);
299 }
300 IntegerType *OrdTy = IRB.getInt32Ty();
301 CsanReadRange = M.getOrInsertFunction(Name: "__csan_read_range", AttributeList: Attr, RetTy: VoidTy,
302 Args: PtrTy, Args: IntptrTy, Args: FlagsTy);
303 CsanWriteRange = M.getOrInsertFunction(Name: "__csan_write_range", AttributeList: Attr, RetTy: VoidTy,
304 Args: PtrTy, Args: IntptrTy, Args: FlagsTy);
305 CsanAtomicThreadFence = M.getOrInsertFunction(
306 Name: "__csan_atomic_thread_fence",
307 AttributeList: TLI.getAttrList(C: &Ctx, ArgNos: {0}, /*Signed=*/true, /*Ret=*/false, AL: Attr), RetTy: VoidTy,
308 Args: OrdTy);
309 CsanAtomicSignalFence = M.getOrInsertFunction(
310 Name: "__csan_atomic_signal_fence",
311 AttributeList: TLI.getAttrList(C: &Ctx, ArgNos: {0}, /*Signed=*/true, /*Ret=*/false, AL: Attr), RetTy: VoidTy,
312 Args: OrdTy);
313}
314
315bool ConcurrencySanitizer::sanitizeFunction(Function &F,
316 const TargetLibraryInfo &TLI) {
317 // This is required to prevent instrumenting call to __csan_init from within
318 // the module constructor.
319 if (F.getName() == kCsanModuleCtorName)
320 return false;
321 // Naked functions can not have prologue/epilogue
322 // (__csan_func_entry/__csan_func_exit) generated, so don't instrument them at
323 // all.
324 if (F.hasFnAttribute(Kind: Attribute::Naked))
325 return false;
326
327 // __attribute__(disable_sanitizer_instrumentation) prevents all kinds of
328 // instrumentation.
329 if (F.hasFnAttribute(Kind: Attribute::DisableSanitizerInstrumentation))
330 return false;
331
332 Mod = F.getParent();
333 MemoryAccessLists Acc;
334 collectMemoryAccesses(F, Out&: Acc);
335
336 const bool SuppressChecking =
337 F.hasFnAttribute(Kind: "sanitize_concurrency_no_checking_at_run_time");
338 const bool SanitizeFunction =
339 F.hasFnAttribute(Kind: Attribute::SanitizeConcurrency) && !SuppressChecking;
340 const bool MayInstrument =
341 SanitizeFunction &&
342 ((ClInstrumentMemoryAccesses && !Acc.LoadsAndStores.empty()) ||
343 (ClInstrumentAtomics && !Acc.AtomicAccesses.empty()) ||
344 (ClInstrumentMemIntrinsics && !Acc.MemIntrinCalls.empty()));
345 const bool NeedsRuntimeIgnores = SuppressChecking && Acc.HasCalls;
346 const bool NeedsFuncEntryExit = ClInstrumentFuncEntryExit && Acc.HasCalls;
347 if (!MayInstrument && !NeedsRuntimeIgnores && !NeedsFuncEntryExit)
348 return false;
349
350 initialize(M&: *Mod, TLI);
351 bool Res = false;
352 const DataLayout &DL = F.getDataLayout();
353
354 if (ClInstrumentMemoryAccesses && SanitizeFunction)
355 for (Instruction *I : Acc.LoadsAndStores)
356 Res |= instrumentLoadOrStore(I, DL);
357
358 if (ClInstrumentAtomics && SanitizeFunction)
359 for (Instruction *I : Acc.AtomicAccesses)
360 Res |= instrumentAtomic(I, DL);
361
362 if (ClInstrumentMemIntrinsics && SanitizeFunction)
363 for (IntrinsicInst *II : Acc.MemIntrinCalls)
364 Res |= instrumentMemIntrinsic(II, DL);
365
366 if (NeedsRuntimeIgnores) {
367 insertRuntimeIgnores(F);
368 Res = true;
369 }
370
371 if ((Res || Acc.HasCalls) && ClInstrumentFuncEntryExit) {
372 insertFuncEntryExit(F);
373 Res = true;
374 }
375 return Res;
376}
377
378bool ConcurrencySanitizer::shouldInstrumentAddress(Value *Addr) const {
379 Value *BaseAddr = Addr->stripInBoundsOffsets();
380 if (auto *GV = dyn_cast<GlobalVariable>(Val: BaseAddr)) {
381 if (GV->hasSection()) {
382 StringRef SectionName = GV->getSection();
383 Triple::ObjectFormatType OF = Mod->getTargetTriple().getObjectFormat();
384 if (SectionName.ends_with(
385 Suffix: getInstrProfSectionName(IPSK: IPSK_cnts, OF, /*AddSegmentInfo=*/false)))
386 return false;
387 }
388 }
389
390 Type *PtrTy = cast<PointerType>(Val: Addr->getType()->getScalarType());
391 unsigned AS = PtrTy->getPointerAddressSpace();
392 if (Mod->getDataLayout().getPointerSizeInBits(AS) > 64)
393 return false;
394 return addressSpaceMayRace(T: Mod->getTargetTriple(), AS);
395}
396
397bool ConcurrencySanitizer::shouldInstrumentAccess(Instruction *I) const {
398 const bool IsWrite = isa<StoreInst>(Val: I);
399 Value *Addr = getLoadStorePointerOperand(V: I);
400 if (!shouldInstrumentAddress(Addr))
401 return false;
402
403 if (!IsWrite)
404 if (auto *GV = dyn_cast<GlobalVariable>(Val: getUnderlyingObject(V: Addr)))
405 if (GV->isConstant())
406 return false;
407
408 const AllocaInst *AI = findAllocaForValue(V: Addr);
409 if (!AI || PointerMayBeCaptured(V: AI, /*ReturnCaptures=*/true))
410 return true;
411
412 const Triple &T = AI->getModule()->getTargetTriple();
413 return T.isGPU() && addressSpaceMayRace(T, AS: AI->getAddressSpace());
414}
415
416void ConcurrencySanitizer::collectMemoryAccesses(Function &F,
417 MemoryAccessLists &Out) {
418 SmallVector<Instruction *, 8> LocalLoadsAndStores;
419
420 for (BasicBlock &BB : F) {
421 for (Instruction &Inst : BB) {
422 // Skip instructions inserted by another instrumentation.
423 if (Inst.hasMetadata(KindID: LLVMContext::MD_nosanitize))
424 continue;
425 if (isAtomicMemoryAccess(I: &Inst))
426 Out.AtomicAccesses.push_back(Elt: &Inst);
427 else if ((isa<LoadInst>(Val: Inst) || isa<StoreInst>(Val: Inst)) &&
428 shouldInstrumentAccess(I: &Inst))
429 LocalLoadsAndStores.push_back(Elt: &Inst);
430 else if (isa<CallBase>(Val: Inst)) {
431 if (isa<AnyMemIntrinsic, MemSetPatternInst>(Val: Inst))
432 Out.MemIntrinCalls.push_back(Elt: cast<IntrinsicInst>(Val: &Inst));
433 Out.HasCalls = true;
434 selectMemoryAccesses(Local&: LocalLoadsAndStores, Out&: Out.LoadsAndStores);
435 }
436 }
437 selectMemoryAccesses(Local&: LocalLoadsAndStores, Out&: Out.LoadsAndStores);
438 }
439}
440
441bool ConcurrencySanitizer::instrumentLoadOrStore(Instruction *I,
442 const DataLayout &DL) {
443 const bool IsWrite = isa<StoreInst>(Val: I);
444 Value *Addr = getLoadStorePointerOperand(V: I);
445 if (!insertAccessProbe(I, Addr, AccessTy: getLoadStoreType(I), DL, IsWrite,
446 /*IsCompound=*/false, /*IsAtomic=*/false))
447 return false;
448 if (IsWrite)
449 ++NumInstrumentedWrites;
450 else
451 ++NumInstrumentedReads;
452 return true;
453}
454
455bool ConcurrencySanitizer::instrumentAtomic(Instruction *I,
456 const DataLayout &DL) {
457 if (auto *FI = dyn_cast<FenceInst>(Val: I)) {
458 InstrumentationIRBuilder IRB(I);
459 FunctionCallee Fn = FI->getSyncScopeID() == SyncScope::SingleThread
460 ? CsanAtomicSignalFence
461 : CsanAtomicThreadFence;
462 IRB.CreateCall(Callee: Fn, Args: createOrdering(IRB, Ord: FI->getOrdering()));
463 return true;
464 }
465
466 Value *Addr = nullptr;
467 Type *AccessTy = nullptr;
468 bool IsWrite = true;
469 bool IsCompound = false;
470 if (auto *LI = dyn_cast<LoadInst>(Val: I)) {
471 Addr = LI->getPointerOperand();
472 AccessTy = LI->getType();
473 IsWrite = false;
474 } else if (auto *SI = dyn_cast<StoreInst>(Val: I)) {
475 Addr = SI->getPointerOperand();
476 AccessTy = SI->getValueOperand()->getType();
477 } else if (auto *RMW = dyn_cast<AtomicRMWInst>(Val: I)) {
478 Addr = RMW->getPointerOperand();
479 AccessTy = RMW->getValOperand()->getType();
480 IsCompound = true;
481 } else if (auto *CAS = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
482 // A failed CAS is still considered a write as the output is unkonwn.
483 Addr = CAS->getPointerOperand();
484 AccessTy = CAS->getNewValOperand()->getType();
485 IsCompound = true;
486 } else {
487 return false;
488 }
489
490 if (!shouldInstrumentAddress(Addr))
491 return false;
492 if (!insertAccessProbe(I, Addr, AccessTy, DL, IsWrite, IsCompound,
493 /*IsAtomic=*/true))
494 return false;
495 if (IsCompound || IsWrite)
496 ++NumInstrumentedWrites;
497 if (IsCompound || !IsWrite)
498 ++NumInstrumentedReads;
499 return true;
500}
501
502bool ConcurrencySanitizer::instrumentMemIntrinsic(IntrinsicInst *II,
503 const DataLayout &DL) {
504 auto *MI = dyn_cast<AnyMemIntrinsic>(Val: II);
505 auto *MSP = dyn_cast<MemSetPatternInst>(Val: II);
506 Value *Dst = MI ? MI->getRawDest() : MSP->getRawDest();
507 Value *Src = nullptr;
508 if (auto *MT = dyn_cast<AnyMemTransferInst>(Val: II))
509 Src = MT->getRawSource();
510
511 bool InstrumentRead = Src && shouldInstrumentAddress(Addr: Src);
512 bool InstrumentWrite = shouldInstrumentAddress(Addr: Dst);
513 if (!InstrumentRead && !InstrumentWrite)
514 return false;
515
516 InstrumentationIRBuilder IRB(II);
517 Value *Len = IRB.CreateIntCast(V: MI ? MI->getLength() : MSP->getLength(),
518 DestTy: IntptrTy, isSigned: false);
519 // The length of a pattern memset counts patterns, not bytes.
520 if (MSP) {
521 TypeSize Size = DL.getTypeAllocSize(Ty: MSP->getValue()->getType());
522 Len = IRB.CreateMul(LHS: Len, RHS: IRB.CreateTypeSize(Ty: IntptrTy, Size));
523 }
524 Constant *Flags =
525 getFlags(Flags: MI && MI->isAtomic() ? AccessFlags::Atomic : AccessFlags::None);
526 if (InstrumentRead) {
527 IRB.CreateCall(Callee: CsanReadRange, Args: {getCallbackAddress(IRB, Addr: Src), Len, Flags});
528 ++NumInstrumentedReads;
529 }
530 if (InstrumentWrite) {
531 IRB.CreateCall(Callee: CsanWriteRange, Args: {getCallbackAddress(IRB, Addr: Dst), Len, Flags});
532 ++NumInstrumentedWrites;
533 }
534 return true;
535}
536
537bool ConcurrencySanitizer::insertAccessProbe(Instruction *I, Value *Addr,
538 Type *AccessTy,
539 const DataLayout &DL, bool IsWrite,
540 bool IsCompound, bool IsAtomic) {
541 if (Addr->isSwiftError())
542 return false;
543 AccessFlags Flags = AccessFlags::None;
544 if (IsAtomic)
545 Flags |= AccessFlags::Atomic;
546 if (IsCompound)
547 Flags |= AccessFlags::Compound;
548 int Idx = getAccessSizeIndex(Ty: AccessTy, DL);
549 if (Idx < 0) {
550 if (IsCompound)
551 return false;
552 InstrumentationIRBuilder IRB(I);
553 Value *Len = IRB.CreateTypeSize(Ty: IntptrTy, Size: DL.getTypeStoreSize(Ty: AccessTy));
554 IRB.CreateCall(Callee: IsWrite ? CsanWriteRange : CsanReadRange,
555 Args: {getCallbackAddress(IRB, Addr), Len, getFlags(Flags)});
556 return true;
557 }
558
559 Align Alignment = Align(1);
560 bool IsVolatile = false;
561 if (auto *LI = dyn_cast<LoadInst>(Val: I)) {
562 Alignment = LI->getAlign();
563 IsVolatile = LI->isVolatile();
564 } else if (auto *SI = dyn_cast<StoreInst>(Val: I)) {
565 Alignment = SI->getAlign();
566 IsVolatile = SI->isVolatile();
567 } else if (auto *RMW = dyn_cast<AtomicRMWInst>(Val: I)) {
568 Alignment = RMW->getAlign();
569 IsVolatile = RMW->isVolatile();
570 } else if (auto *CAS = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
571 Alignment = CAS->getAlign();
572 IsVolatile = CAS->isVolatile();
573 }
574
575 uint32_t TypeSize = DL.getTypeStoreSizeInBits(Ty: AccessTy);
576 bool Unaligned =
577 Alignment < Align(8) && Alignment.value() % (TypeSize / 8) != 0;
578 FunctionCallee Callback;
579 if (IsCompound)
580 Callback = Unaligned ? CsanUnalignedCompoundRW[Idx] : CsanCompoundRW[Idx];
581 else if (Unaligned)
582 Callback = IsWrite ? CsanUnalignedWrite[Idx] : CsanUnalignedRead[Idx];
583 else
584 Callback = IsWrite ? CsanWrite[Idx] : CsanRead[Idx];
585
586 InstrumentationIRBuilder IRB(I);
587 Value *CallbackAddr = getCallbackAddress(IRB, Addr);
588 Value *FlagsVal = getFlags(Flags);
589 if (ClDistinguishVolatile && IsVolatile && TypeSize <= 64) {
590 // The declared alignment may understate the pointer's real alignment.
591 Value *IsAligned = IRB.getTrue();
592 if (Unaligned)
593 IsAligned = IRB.CreateIsNull(Arg: IRB.CreateAnd(
594 LHS: IRB.CreatePtrToInt(V: CallbackAddr, DestTy: IntptrTy), RHS: TypeSize / 8 - 1));
595 FlagsVal = IRB.CreateSelect(
596 C: IsAligned, True: getFlags(Flags: Flags | AccessFlags::Atomic), False: FlagsVal);
597 }
598 IRB.CreateCall(Callee: Callback, Args: {CallbackAddr, FlagsVal});
599 return true;
600}
601
602void ConcurrencySanitizer::insertFuncEntryExit(Function &F) {
603 const DataLayout &DL = F.getDataLayout();
604 InstrumentationIRBuilder IRB(&*F.getEntryBlock().getFirstNonPHIIt());
605 Type *ProgramAsPtrTy =
606 PointerType::get(C&: F.getContext(), AddressSpace: DL.getProgramAddressSpace());
607 Value *ReturnAddress = IRB.CreateIntrinsic(ID: Intrinsic::returnaddress,
608 OverloadTypes: {ProgramAsPtrTy}, Args: IRB.getInt32(C: 0));
609 IRB.CreateCall(Callee: CsanFuncEntry, Args: ReturnAddress);
610
611 EscapeEnumerator EE(F, "csan_cleanup", ClHandleCxxExceptions);
612 while (IRBuilder<> *AtExit = EE.Next()) {
613 InstrumentationIRBuilder::ensureDebugInfo(IRB&: *AtExit, F);
614 AtExit->CreateCall(Callee: CsanFuncExit);
615 }
616}
617
618void ConcurrencySanitizer::insertRuntimeIgnores(Function &F) {
619 InstrumentationIRBuilder IRB(&*F.getEntryBlock().getFirstNonPHIIt());
620 IRB.CreateCall(Callee: CsanIgnoreBegin);
621 EscapeEnumerator EE(F, "csan_ignore_cleanup", ClHandleCxxExceptions);
622 while (IRBuilder<> *AtExit = EE.Next()) {
623 InstrumentationIRBuilder::ensureDebugInfo(IRB&: *AtExit, F);
624 AtExit->CreateCall(Callee: CsanIgnoreEnd);
625 }
626}
627