1//===-- ThreadSanitizer.cpp - race detector -------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is a part of ThreadSanitizer, a race detector.
10//
11// The tool is under development, for the details about previous versions see
12// http://code.google.com/p/data-race-test
13//
14// The instrumentation phase is quite simple:
15// - Insert calls to run-time library before every memory access.
16// - Optimizations may apply to avoid instrumenting some of the accesses.
17// - Insert calls at function entry/exit.
18// The rest is handled by the run-time library.
19//===----------------------------------------------------------------------===//
20
21#include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
22#include "InstrumentationOptions.h"
23#include "llvm/ADT/DenseMap.h"
24#include "llvm/ADT/SmallString.h"
25#include "llvm/ADT/SmallVector.h"
26#include "llvm/ADT/Statistic.h"
27#include "llvm/ADT/StringExtras.h"
28#include "llvm/Analysis/CaptureTracking.h"
29#include "llvm/Analysis/TargetLibraryInfo.h"
30#include "llvm/Analysis/ValueTracking.h"
31#include "llvm/IR/DataLayout.h"
32#include "llvm/IR/Function.h"
33#include "llvm/IR/IRBuilder.h"
34#include "llvm/IR/Instructions.h"
35#include "llvm/IR/IntrinsicInst.h"
36#include "llvm/IR/Intrinsics.h"
37#include "llvm/IR/LLVMContext.h"
38#include "llvm/IR/Metadata.h"
39#include "llvm/IR/Module.h"
40#include "llvm/IR/Type.h"
41#include "llvm/ProfileData/InstrProf.h"
42#include "llvm/Support/Debug.h"
43#include "llvm/Support/raw_ostream.h"
44#include "llvm/Transforms/Utils/EscapeEnumerator.h"
45#include "llvm/Transforms/Utils/Instrumentation.h"
46#include "llvm/Transforms/Utils/Local.h"
47#include "llvm/Transforms/Utils/ModuleUtils.h"
48
49using namespace llvm;
50
51#define DEBUG_TYPE "tsan"
52
53STATISTIC(NumInstrumentedReads, "Number of instrumented reads");
54STATISTIC(NumInstrumentedWrites, "Number of instrumented writes");
55STATISTIC(NumOmittedReadsBeforeWrite,
56 "Number of reads ignored due to following writes");
57STATISTIC(NumAccessesWithBadSize, "Number of accesses with bad size");
58STATISTIC(NumInstrumentedVtableWrites, "Number of vtable ptr writes");
59STATISTIC(NumInstrumentedVtableReads, "Number of vtable ptr reads");
60STATISTIC(NumOmittedReadsFromConstantGlobals,
61 "Number of reads from constant globals");
62STATISTIC(NumOmittedReadsFromVtable, "Number of vtable reads");
63STATISTIC(NumOmittedNonCaptured, "Number of accesses ignored due to capturing");
64
65const char kTsanModuleCtorName[] = "tsan.module_ctor";
66const char kTsanInitName[] = "__tsan_init";
67
68namespace {
69
70/// ThreadSanitizer: instrument the code in module to find races.
71///
72/// Instantiating ThreadSanitizer inserts the tsan runtime library API function
73/// declarations into the module if they don't exist already. Instantiating
74/// ensures the __tsan_init function is in the list of global constructors for
75/// the module.
76struct ThreadSanitizer {
77 ThreadSanitizer(const InstrumentationOptions &Opts) : Opts(Opts) {
78 // Check options and warn user.
79 if (Opts.tsan_instrument_read_before_write &&
80 Opts.tsan_compound_read_before_write) {
81 errs()
82 << "warning: Option -tsan-compound-read-before-write has no effect "
83 "when -tsan-instrument-read-before-write is set.\n";
84 }
85 }
86
87 bool sanitizeFunction(Function &F, const TargetLibraryInfo &TLI);
88
89private:
90 // Internal Instruction wrapper that contains more information about the
91 // Instruction from prior analysis.
92 struct InstructionInfo {
93 // Instrumentation emitted for this instruction is for a compounded set of
94 // read and write operations in the same basic block.
95 static constexpr unsigned kCompoundRW = (1U << 0);
96
97 explicit InstructionInfo(Instruction *Inst) : Inst(Inst) {}
98
99 Instruction *Inst;
100 unsigned Flags = 0;
101 };
102
103 void initialize(Module &M, const TargetLibraryInfo &TLI);
104 bool instrumentLoadOrStore(const InstructionInfo &II, const DataLayout &DL);
105 bool instrumentAtomic(Instruction *I, const DataLayout &DL);
106 bool instrumentMemIntrinsic(Instruction *I);
107 void chooseInstructionsToInstrument(SmallVectorImpl<Instruction *> &Local,
108 SmallVectorImpl<InstructionInfo> &All,
109 const DataLayout &DL);
110 bool addrPointsToConstantData(Value *Addr);
111 int getMemoryAccessFuncIndex(Type *OrigTy, Value *Addr, const DataLayout &DL);
112 void InsertRuntimeIgnores(Function &F);
113
114 const InstrumentationOptions &Opts;
115 Type *IntptrTy;
116 FunctionCallee TsanFuncEntry;
117 FunctionCallee TsanFuncExit;
118 FunctionCallee TsanIgnoreBegin;
119 FunctionCallee TsanIgnoreEnd;
120 // Accesses sizes are powers of two: 1, 2, 4, 8, 16.
121 static const size_t kNumberOfAccessSizes = 5;
122 FunctionCallee TsanRead[kNumberOfAccessSizes];
123 FunctionCallee TsanWrite[kNumberOfAccessSizes];
124 FunctionCallee TsanUnalignedRead[kNumberOfAccessSizes];
125 FunctionCallee TsanUnalignedWrite[kNumberOfAccessSizes];
126 FunctionCallee TsanVolatileRead[kNumberOfAccessSizes];
127 FunctionCallee TsanVolatileWrite[kNumberOfAccessSizes];
128 FunctionCallee TsanUnalignedVolatileRead[kNumberOfAccessSizes];
129 FunctionCallee TsanUnalignedVolatileWrite[kNumberOfAccessSizes];
130 FunctionCallee TsanCompoundRW[kNumberOfAccessSizes];
131 FunctionCallee TsanUnalignedCompoundRW[kNumberOfAccessSizes];
132 FunctionCallee TsanAtomicLoad[kNumberOfAccessSizes];
133 FunctionCallee TsanAtomicStore[kNumberOfAccessSizes];
134 FunctionCallee TsanAtomicRMW[AtomicRMWInst::LAST_BINOP + 1]
135 [kNumberOfAccessSizes];
136 FunctionCallee TsanAtomicCAS[kNumberOfAccessSizes];
137 FunctionCallee TsanAtomicThreadFence;
138 FunctionCallee TsanAtomicSignalFence;
139 FunctionCallee TsanVptrUpdate;
140 FunctionCallee TsanVptrLoad;
141 FunctionCallee MemmoveFn, MemcpyFn, MemsetFn;
142};
143
144void insertModuleCtor(Module &M) {
145 getOrCreateSanitizerCtorAndInitFunctions(
146 M, CtorName: kTsanModuleCtorName, InitName: kTsanInitName, /*InitArgTypes=*/{},
147 /*InitArgs=*/{},
148 // This callback is invoked when the functions are created the first
149 // time. Hook them into the global ctors list in that case:
150 FunctionsCreatedCallback: [&](Function *Ctor, FunctionCallee) { appendToGlobalCtors(M, F: Ctor, Priority: 0); });
151}
152} // namespace
153
154PreservedAnalyses ThreadSanitizerPass::run(Function &F,
155 FunctionAnalysisManager &FAM) {
156 ThreadSanitizer TSan(getInstrumentationOptions(F));
157 if (TSan.sanitizeFunction(F, TLI: FAM.getResult<TargetLibraryAnalysis>(IR&: F)))
158 return PreservedAnalyses::none();
159 return PreservedAnalyses::all();
160}
161
162PreservedAnalyses ModuleThreadSanitizerPass::run(Module &M,
163 ModuleAnalysisManager &MAM) {
164 // Return early if nosanitize_thread module flag is present for the module.
165 if (checkIfAlreadyInstrumented(M, Flag: "nosanitize_thread"))
166 return PreservedAnalyses::all();
167 insertModuleCtor(M);
168 return PreservedAnalyses::none();
169}
170void ThreadSanitizer::initialize(Module &M, const TargetLibraryInfo &TLI) {
171 const DataLayout &DL = M.getDataLayout();
172 LLVMContext &Ctx = M.getContext();
173 IntptrTy = DL.getIntPtrType(C&: Ctx);
174
175 IRBuilder<> IRB(M);
176 AttributeList Attr;
177 Attr = Attr.addFnAttribute(C&: Ctx, Kind: Attribute::NoUnwind);
178 // Initialize the callbacks.
179 TsanFuncEntry = M.getOrInsertFunction(Name: "__tsan_func_entry", AttributeList: Attr,
180 RetTy: IRB.getVoidTy(), Args: IRB.getPtrTy());
181 TsanFuncExit =
182 M.getOrInsertFunction(Name: "__tsan_func_exit", AttributeList: Attr, RetTy: IRB.getVoidTy());
183 TsanIgnoreBegin = M.getOrInsertFunction(Name: "__tsan_ignore_thread_begin", AttributeList: Attr,
184 RetTy: IRB.getVoidTy());
185 TsanIgnoreEnd =
186 M.getOrInsertFunction(Name: "__tsan_ignore_thread_end", AttributeList: Attr, RetTy: IRB.getVoidTy());
187 IntegerType *OrdTy = IRB.getInt32Ty();
188 for (size_t i = 0; i < kNumberOfAccessSizes; ++i) {
189 const unsigned ByteSize = 1U << i;
190 const unsigned BitSize = ByteSize * 8;
191 std::string ByteSizeStr = utostr(X: ByteSize);
192 std::string BitSizeStr = utostr(X: BitSize);
193 SmallString<32> ReadName("__tsan_read" + ByteSizeStr);
194 TsanRead[i] = M.getOrInsertFunction(Name: ReadName, AttributeList: Attr, RetTy: IRB.getVoidTy(),
195 Args: IRB.getPtrTy());
196
197 SmallString<32> WriteName("__tsan_write" + ByteSizeStr);
198 TsanWrite[i] = M.getOrInsertFunction(Name: WriteName, AttributeList: Attr, RetTy: IRB.getVoidTy(),
199 Args: IRB.getPtrTy());
200
201 SmallString<64> UnalignedReadName("__tsan_unaligned_read" + ByteSizeStr);
202 TsanUnalignedRead[i] = M.getOrInsertFunction(
203 Name: UnalignedReadName, AttributeList: Attr, RetTy: IRB.getVoidTy(), Args: IRB.getPtrTy());
204
205 SmallString<64> UnalignedWriteName("__tsan_unaligned_write" + ByteSizeStr);
206 TsanUnalignedWrite[i] = M.getOrInsertFunction(
207 Name: UnalignedWriteName, AttributeList: Attr, RetTy: IRB.getVoidTy(), Args: IRB.getPtrTy());
208
209 SmallString<64> VolatileReadName("__tsan_volatile_read" + ByteSizeStr);
210 TsanVolatileRead[i] = M.getOrInsertFunction(
211 Name: VolatileReadName, AttributeList: Attr, RetTy: IRB.getVoidTy(), Args: IRB.getPtrTy());
212
213 SmallString<64> VolatileWriteName("__tsan_volatile_write" + ByteSizeStr);
214 TsanVolatileWrite[i] = M.getOrInsertFunction(
215 Name: VolatileWriteName, AttributeList: Attr, RetTy: IRB.getVoidTy(), Args: IRB.getPtrTy());
216
217 SmallString<64> UnalignedVolatileReadName("__tsan_unaligned_volatile_read" +
218 ByteSizeStr);
219 TsanUnalignedVolatileRead[i] = M.getOrInsertFunction(
220 Name: UnalignedVolatileReadName, AttributeList: Attr, RetTy: IRB.getVoidTy(), Args: IRB.getPtrTy());
221
222 SmallString<64> UnalignedVolatileWriteName(
223 "__tsan_unaligned_volatile_write" + ByteSizeStr);
224 TsanUnalignedVolatileWrite[i] = M.getOrInsertFunction(
225 Name: UnalignedVolatileWriteName, AttributeList: Attr, RetTy: IRB.getVoidTy(), Args: IRB.getPtrTy());
226
227 SmallString<64> CompoundRWName("__tsan_read_write" + ByteSizeStr);
228 TsanCompoundRW[i] = M.getOrInsertFunction(
229 Name: CompoundRWName, AttributeList: Attr, RetTy: IRB.getVoidTy(), Args: IRB.getPtrTy());
230
231 SmallString<64> UnalignedCompoundRWName("__tsan_unaligned_read_write" +
232 ByteSizeStr);
233 TsanUnalignedCompoundRW[i] = M.getOrInsertFunction(
234 Name: UnalignedCompoundRWName, AttributeList: Attr, RetTy: IRB.getVoidTy(), Args: IRB.getPtrTy());
235
236 Type *Ty = Type::getIntNTy(C&: Ctx, N: BitSize);
237 Type *PtrTy = PointerType::get(C&: Ctx, AddressSpace: 0);
238 SmallString<32> AtomicLoadName("__tsan_atomic" + BitSizeStr + "_load");
239 TsanAtomicLoad[i] =
240 M.getOrInsertFunction(Name: AtomicLoadName,
241 AttributeList: TLI.getAttrList(C: &Ctx, ArgNos: {1}, /*Signed=*/true,
242 /*Ret=*/BitSize <= 32, AL: Attr),
243 RetTy: Ty, Args: PtrTy, Args: OrdTy);
244
245 // Args of type Ty need extension only when BitSize is 32 or less.
246 using Idxs = std::vector<unsigned>;
247 Idxs Idxs2Or12 ((BitSize <= 32) ? Idxs({1, 2}) : Idxs({2}));
248 Idxs Idxs34Or1234((BitSize <= 32) ? Idxs({1, 2, 3, 4}) : Idxs({3, 4}));
249 SmallString<32> AtomicStoreName("__tsan_atomic" + BitSizeStr + "_store");
250 TsanAtomicStore[i] = M.getOrInsertFunction(
251 Name: AtomicStoreName,
252 AttributeList: TLI.getAttrList(C: &Ctx, ArgNos: Idxs2Or12, /*Signed=*/true, /*Ret=*/false, AL: Attr),
253 RetTy: IRB.getVoidTy(), Args: PtrTy, Args: Ty, Args: OrdTy);
254
255 for (unsigned Op = AtomicRMWInst::FIRST_BINOP;
256 Op <= AtomicRMWInst::LAST_BINOP; ++Op) {
257 TsanAtomicRMW[Op][i] = nullptr;
258 const char *NamePart = nullptr;
259 if (Op == AtomicRMWInst::Xchg)
260 NamePart = "_exchange";
261 else if (Op == AtomicRMWInst::Add)
262 NamePart = "_fetch_add";
263 else if (Op == AtomicRMWInst::Sub)
264 NamePart = "_fetch_sub";
265 else if (Op == AtomicRMWInst::And)
266 NamePart = "_fetch_and";
267 else if (Op == AtomicRMWInst::Or)
268 NamePart = "_fetch_or";
269 else if (Op == AtomicRMWInst::Xor)
270 NamePart = "_fetch_xor";
271 else if (Op == AtomicRMWInst::Nand)
272 NamePart = "_fetch_nand";
273 else
274 continue;
275 SmallString<32> RMWName("__tsan_atomic" + itostr(X: BitSize) + NamePart);
276 TsanAtomicRMW[Op][i] = M.getOrInsertFunction(
277 Name: RMWName,
278 AttributeList: TLI.getAttrList(C: &Ctx, ArgNos: Idxs2Or12, /*Signed=*/true,
279 /*Ret=*/BitSize <= 32, AL: Attr),
280 RetTy: Ty, Args: PtrTy, Args: Ty, Args: OrdTy);
281 }
282
283 SmallString<32> AtomicCASName("__tsan_atomic" + BitSizeStr +
284 "_compare_exchange_val");
285 TsanAtomicCAS[i] = M.getOrInsertFunction(
286 Name: AtomicCASName,
287 AttributeList: TLI.getAttrList(C: &Ctx, ArgNos: Idxs34Or1234, /*Signed=*/true,
288 /*Ret=*/BitSize <= 32, AL: Attr),
289 RetTy: Ty, Args: PtrTy, Args: Ty, Args: Ty, Args: OrdTy, Args: OrdTy);
290 }
291 TsanVptrUpdate =
292 M.getOrInsertFunction(Name: "__tsan_vptr_update", AttributeList: Attr, RetTy: IRB.getVoidTy(),
293 Args: IRB.getPtrTy(), Args: IRB.getPtrTy());
294 TsanVptrLoad = M.getOrInsertFunction(Name: "__tsan_vptr_read", AttributeList: Attr,
295 RetTy: IRB.getVoidTy(), Args: IRB.getPtrTy());
296 TsanAtomicThreadFence = M.getOrInsertFunction(
297 Name: "__tsan_atomic_thread_fence",
298 AttributeList: TLI.getAttrList(C: &Ctx, ArgNos: {0}, /*Signed=*/true, /*Ret=*/false, AL: Attr),
299 RetTy: IRB.getVoidTy(), Args: OrdTy);
300
301 TsanAtomicSignalFence = M.getOrInsertFunction(
302 Name: "__tsan_atomic_signal_fence",
303 AttributeList: TLI.getAttrList(C: &Ctx, ArgNos: {0}, /*Signed=*/true, /*Ret=*/false, AL: Attr),
304 RetTy: IRB.getVoidTy(), Args: OrdTy);
305
306 MemmoveFn =
307 M.getOrInsertFunction(Name: "__tsan_memmove", AttributeList: Attr, RetTy: IRB.getPtrTy(),
308 Args: IRB.getPtrTy(), Args: IRB.getPtrTy(), Args: IntptrTy);
309 MemcpyFn =
310 M.getOrInsertFunction(Name: "__tsan_memcpy", AttributeList: Attr, RetTy: IRB.getPtrTy(),
311 Args: IRB.getPtrTy(), Args: IRB.getPtrTy(), Args: IntptrTy);
312 MemsetFn = M.getOrInsertFunction(
313 Name: "__tsan_memset",
314 AttributeList: TLI.getAttrList(C: &Ctx, ArgNos: {1}, /*Signed=*/true, /*Ret=*/false, AL: Attr),
315 RetTy: IRB.getPtrTy(), Args: IRB.getPtrTy(), Args: IRB.getInt32Ty(), Args: IntptrTy);
316}
317
318static bool isVtableAccess(Instruction *I) {
319 if (MDNode *Tag = I->getMetadata(KindID: LLVMContext::MD_tbaa))
320 return Tag->isTBAAVtableAccess();
321 return false;
322}
323
324// Do not instrument known races/"benign races" that come from compiler
325// instrumentation. The user has no way of suppressing them.
326static bool shouldInstrumentReadWriteFromAddress(const Module *M, Value *Addr) {
327 // Peel off GEPs and BitCasts.
328 // Note: This also peels AddrspaceCasts, so this should not be used when
329 // checking the address space below.
330 Value *PeeledAddr = Addr->stripInBoundsOffsets();
331
332 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Val: PeeledAddr)) {
333 if (GV->hasSection()) {
334 StringRef SectionName = GV->getSection();
335 // Check if the global is in the PGO counters section.
336 auto OF = M->getTargetTriple().getObjectFormat();
337 if (SectionName.ends_with(
338 Suffix: getInstrProfSectionName(IPSK: IPSK_cnts, OF, /*AddSegmentInfo=*/false)))
339 return false;
340 }
341 }
342
343 // Do not instrument accesses from different address spaces; we cannot deal
344 // with them.
345 Type *PtrTy = cast<PointerType>(Val: Addr->getType()->getScalarType());
346 if (PtrTy->getPointerAddressSpace() != 0)
347 return false;
348
349 return true;
350}
351
352bool ThreadSanitizer::addrPointsToConstantData(Value *Addr) {
353 // If this is a GEP, just analyze its pointer operand.
354 if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Val: Addr))
355 Addr = GEP->getPointerOperand();
356
357 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Val: Addr)) {
358 if (GV->isConstant()) {
359 // Reads from constant globals can not race with any writes.
360 NumOmittedReadsFromConstantGlobals++;
361 return true;
362 }
363 } else if (LoadInst *L = dyn_cast<LoadInst>(Val: Addr)) {
364 if (isVtableAccess(I: L)) {
365 // Reads from a vtable pointer can not race with any writes.
366 NumOmittedReadsFromVtable++;
367 return true;
368 }
369 }
370 return false;
371}
372
373// Instrumenting some of the accesses may be proven redundant.
374// Currently handled:
375// - read-before-write (within same BB, no calls between)
376// - not captured variables
377//
378// We do not handle some of the patterns that should not survive
379// after the classic compiler optimizations.
380// E.g. two reads from the same temp should be eliminated by CSE,
381// two writes should be eliminated by DSE, etc.
382//
383// 'Local' is a vector of insns within the same BB (no calls between).
384// 'All' is a vector of insns that will be instrumented.
385void ThreadSanitizer::chooseInstructionsToInstrument(
386 SmallVectorImpl<Instruction *> &Local,
387 SmallVectorImpl<InstructionInfo> &All, const DataLayout &DL) {
388 DenseMap<Value *, size_t> WriteTargets; // Map of addresses to index in All
389 // Iterate from the end.
390 for (Instruction *I : reverse(C&: Local)) {
391 const bool IsWrite = isa<StoreInst>(Val: *I);
392 Value *Addr = IsWrite ? cast<StoreInst>(Val: I)->getPointerOperand()
393 : cast<LoadInst>(Val: I)->getPointerOperand();
394
395 if (!shouldInstrumentReadWriteFromAddress(M: I->getModule(), Addr))
396 continue;
397
398 if (!IsWrite) {
399 const auto WriteEntry = WriteTargets.find(Val: Addr);
400 if (!Opts.tsan_instrument_read_before_write &&
401 WriteEntry != WriteTargets.end()) {
402 auto &WI = All[WriteEntry->second];
403 // If we distinguish volatile accesses and if either the read or write
404 // is volatile, do not omit any instrumentation.
405 const bool AnyVolatile = Opts.tsan_distinguish_volatile &&
406 (cast<LoadInst>(Val: I)->isVolatile() ||
407 cast<StoreInst>(Val: WI.Inst)->isVolatile());
408 if (!AnyVolatile) {
409 // We will write to this temp, so no reason to analyze the read.
410 // Mark the write instruction as compound.
411 WI.Flags |= InstructionInfo::kCompoundRW;
412 NumOmittedReadsBeforeWrite++;
413 continue;
414 }
415 }
416
417 if (addrPointsToConstantData(Addr)) {
418 // Addr points to some constant data -- it can not race with any writes.
419 continue;
420 }
421 }
422
423 const AllocaInst *AI = findAllocaForValue(V: Addr);
424 // Instead of Addr, we should check whether its base pointer is captured.
425 if (AI && !PointerMayBeCaptured(V: AI, /*ReturnCaptures=*/true) &&
426 Opts.tsan_omit_by_pointer_capturing) {
427 // The variable is addressable but not captured, so it cannot be
428 // referenced from a different thread and participate in a data race
429 // (see llvm/Analysis/CaptureTracking.h for details).
430 NumOmittedNonCaptured++;
431 continue;
432 }
433
434 // Instrument this instruction.
435 All.emplace_back(Args&: I);
436 if (IsWrite) {
437 // For read-before-write and compound instrumentation we only need one
438 // write target, and we can override any previous entry if it exists.
439 WriteTargets[Addr] = All.size() - 1;
440 }
441 }
442 Local.clear();
443}
444
445static bool isTsanAtomic(const Instruction *I) {
446 // TODO: Ask TTI whether synchronization scope is between threads.
447 auto SSID = getAtomicSyncScopeID(I);
448 if (!SSID)
449 return false;
450 if (isa<LoadInst>(Val: I) || isa<StoreInst>(Val: I))
451 return *SSID != SyncScope::SingleThread;
452 return true;
453}
454
455void ThreadSanitizer::InsertRuntimeIgnores(Function &F) {
456 InstrumentationIRBuilder IRB(F.getEntryBlock().getFirstNonPHIIt());
457 IRB.CreateCall(Callee: TsanIgnoreBegin);
458 EscapeEnumerator EE(F, "tsan_ignore_cleanup",
459 Opts.tsan_handle_cxx_exceptions);
460 while (IRBuilder<> *AtExit = EE.Next()) {
461 InstrumentationIRBuilder::ensureDebugInfo(IRB&: *AtExit, F);
462 AtExit->CreateCall(Callee: TsanIgnoreEnd);
463 }
464}
465
466bool ThreadSanitizer::sanitizeFunction(Function &F,
467 const TargetLibraryInfo &TLI) {
468 // This is required to prevent instrumenting call to __tsan_init from within
469 // the module constructor.
470 if (F.getName() == kTsanModuleCtorName)
471 return false;
472 // Naked functions can not have prologue/epilogue
473 // (__tsan_func_entry/__tsan_func_exit) generated, so don't instrument them at
474 // all.
475 if (F.hasFnAttribute(Kind: Attribute::Naked))
476 return false;
477
478 // __attribute__(disable_sanitizer_instrumentation) prevents all kinds of
479 // instrumentation.
480 if (F.hasFnAttribute(Kind: Attribute::DisableSanitizerInstrumentation))
481 return false;
482
483 initialize(M&: *F.getParent(), TLI);
484 SmallVector<InstructionInfo, 8> AllLoadsAndStores;
485 SmallVector<Instruction*, 8> LocalLoadsAndStores;
486 SmallVector<Instruction*, 8> AtomicAccesses;
487 SmallVector<Instruction*, 8> MemIntrinCalls;
488 bool Res = false;
489 bool HasCalls = false;
490 bool SanitizeFunction = F.hasFnAttribute(Kind: Attribute::SanitizeThread);
491 const DataLayout &DL = F.getDataLayout();
492
493 // Traverse all instructions, collect loads/stores/returns, check for calls.
494 for (auto &BB : F) {
495 for (auto &Inst : BB) {
496 // Skip instructions inserted by another instrumentation.
497 if (Inst.hasMetadata(KindID: LLVMContext::MD_nosanitize))
498 continue;
499 if (isTsanAtomic(I: &Inst))
500 AtomicAccesses.push_back(Elt: &Inst);
501 else if (isa<LoadInst>(Val: Inst) || isa<StoreInst>(Val: Inst))
502 LocalLoadsAndStores.push_back(Elt: &Inst);
503 else if (isa<CallInst>(Val: Inst) || isa<InvokeInst>(Val: Inst)) {
504 if (CallInst *CI = dyn_cast<CallInst>(Val: &Inst))
505 maybeMarkSanitizerLibraryCallNoBuiltin(CI, TLI: &TLI);
506 if (isa<MemIntrinsic>(Val: Inst))
507 MemIntrinCalls.push_back(Elt: &Inst);
508 HasCalls = true;
509 chooseInstructionsToInstrument(Local&: LocalLoadsAndStores, All&: AllLoadsAndStores,
510 DL);
511 }
512 }
513 chooseInstructionsToInstrument(Local&: LocalLoadsAndStores, All&: AllLoadsAndStores, DL);
514 }
515
516 // We have collected all loads and stores.
517 // FIXME: many of these accesses do not need to be checked for races
518 // (e.g. variables that do not escape, etc).
519
520 // Instrument memory accesses only if we want to report bugs in the function.
521 if (Opts.tsan_instrument_memory_accesses && SanitizeFunction)
522 for (const auto &II : AllLoadsAndStores) {
523 Res |= instrumentLoadOrStore(II, DL);
524 }
525
526 // Instrument atomic memory accesses in any case (they can be used to
527 // implement synchronization).
528 if (Opts.tsan_instrument_atomics)
529 for (auto *Inst : AtomicAccesses) {
530 Res |= instrumentAtomic(I: Inst, DL);
531 }
532
533 if (Opts.tsan_instrument_memintrinsics && SanitizeFunction)
534 for (auto *Inst : MemIntrinCalls) {
535 Res |= instrumentMemIntrinsic(I: Inst);
536 }
537
538 if (F.hasFnAttribute(Kind: "sanitize_thread_no_checking_at_run_time")) {
539 assert(!F.hasFnAttribute(Attribute::SanitizeThread));
540 if (HasCalls)
541 InsertRuntimeIgnores(F);
542 }
543
544 // Instrument function entry/exit points if there were instrumented accesses.
545 if ((Res || HasCalls) && Opts.tsan_instrument_func_entry_exit) {
546 InstrumentationIRBuilder IRB(F.getEntryBlock().getFirstNonPHIIt());
547 auto ProgramAsPtrTy = PointerType::get(C&: F.getParent()->getContext(),
548 AddressSpace: DL.getProgramAddressSpace());
549 Value *ReturnAddress = IRB.CreateIntrinsic(
550 ID: Intrinsic::returnaddress, OverloadTypes: {ProgramAsPtrTy}, Args: IRB.getInt32(C: 0));
551 IRB.CreateCall(Callee: TsanFuncEntry, Args: ReturnAddress);
552
553 EscapeEnumerator EE(F, "tsan_cleanup", Opts.tsan_handle_cxx_exceptions);
554 while (IRBuilder<> *AtExit = EE.Next()) {
555 InstrumentationIRBuilder::ensureDebugInfo(IRB&: *AtExit, F);
556 AtExit->CreateCall(Callee: TsanFuncExit, Args: {});
557 }
558 Res = true;
559 }
560 return Res;
561}
562
563bool ThreadSanitizer::instrumentLoadOrStore(const InstructionInfo &II,
564 const DataLayout &DL) {
565 InstrumentationIRBuilder IRB(II.Inst);
566 const bool IsWrite = isa<StoreInst>(Val: *II.Inst);
567 Value *Addr = IsWrite ? cast<StoreInst>(Val: II.Inst)->getPointerOperand()
568 : cast<LoadInst>(Val: II.Inst)->getPointerOperand();
569 Type *OrigTy = getLoadStoreType(I: II.Inst);
570
571 // swifterror memory addresses are mem2reg promoted by instruction selection.
572 // As such they cannot have regular uses like an instrumentation function and
573 // it makes no sense to track them as memory.
574 if (Addr->isSwiftError())
575 return false;
576
577 int Idx = getMemoryAccessFuncIndex(OrigTy, Addr, DL);
578 if (Idx < 0)
579 return false;
580 if (IsWrite && isVtableAccess(I: II.Inst)) {
581 LLVM_DEBUG(dbgs() << " VPTR : " << *II.Inst << "\n");
582 Value *StoredValue = cast<StoreInst>(Val: II.Inst)->getValueOperand();
583 // StoredValue may be a vector type if we are storing several vptrs at once.
584 // In this case, just take the first element of the vector since this is
585 // enough to find vptr races.
586 if (isa<VectorType>(Val: StoredValue->getType()))
587 StoredValue = IRB.CreateExtractElement(
588 Vec: StoredValue, Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: 0));
589 if (StoredValue->getType()->isIntegerTy())
590 StoredValue = IRB.CreateIntToPtr(V: StoredValue, DestTy: IRB.getPtrTy());
591 // Call TsanVptrUpdate.
592 IRB.CreateCall(Callee: TsanVptrUpdate, Args: {Addr, StoredValue});
593 NumInstrumentedVtableWrites++;
594 return true;
595 }
596 if (!IsWrite && isVtableAccess(I: II.Inst)) {
597 IRB.CreateCall(Callee: TsanVptrLoad, Args: Addr);
598 NumInstrumentedVtableReads++;
599 return true;
600 }
601
602 const Align Alignment = IsWrite ? cast<StoreInst>(Val: II.Inst)->getAlign()
603 : cast<LoadInst>(Val: II.Inst)->getAlign();
604 const bool IsCompoundRW = Opts.tsan_compound_read_before_write &&
605 (II.Flags & InstructionInfo::kCompoundRW);
606 const bool IsVolatile = Opts.tsan_distinguish_volatile &&
607 (IsWrite ? cast<StoreInst>(Val: II.Inst)->isVolatile()
608 : cast<LoadInst>(Val: II.Inst)->isVolatile());
609 assert((!IsVolatile || !IsCompoundRW) && "Compound volatile invalid!");
610
611 const uint32_t TypeSize = DL.getTypeStoreSizeInBits(Ty: OrigTy);
612 FunctionCallee OnAccessFunc = nullptr;
613 if (Alignment >= Align(8) || (Alignment.value() % (TypeSize / 8)) == 0) {
614 if (IsCompoundRW)
615 OnAccessFunc = TsanCompoundRW[Idx];
616 else if (IsVolatile)
617 OnAccessFunc = IsWrite ? TsanVolatileWrite[Idx] : TsanVolatileRead[Idx];
618 else
619 OnAccessFunc = IsWrite ? TsanWrite[Idx] : TsanRead[Idx];
620 } else {
621 if (IsCompoundRW)
622 OnAccessFunc = TsanUnalignedCompoundRW[Idx];
623 else if (IsVolatile)
624 OnAccessFunc = IsWrite ? TsanUnalignedVolatileWrite[Idx]
625 : TsanUnalignedVolatileRead[Idx];
626 else
627 OnAccessFunc = IsWrite ? TsanUnalignedWrite[Idx] : TsanUnalignedRead[Idx];
628 }
629 IRB.CreateCall(Callee: OnAccessFunc, Args: Addr);
630 if (IsCompoundRW || IsWrite)
631 NumInstrumentedWrites++;
632 if (IsCompoundRW || !IsWrite)
633 NumInstrumentedReads++;
634 return true;
635}
636
637static ConstantInt *createOrdering(IRBuilder<> *IRB, AtomicOrdering ord) {
638 uint32_t v = 0;
639 switch (ord) {
640 case AtomicOrdering::NotAtomic:
641 llvm_unreachable("unexpected atomic ordering!");
642 case AtomicOrdering::Unordered: [[fallthrough]];
643 case AtomicOrdering::Monotonic: v = 0; break;
644 // Not specified yet:
645 // case AtomicOrdering::Consume: v = 1; break;
646 case AtomicOrdering::Acquire: v = 2; break;
647 case AtomicOrdering::Release: v = 3; break;
648 case AtomicOrdering::AcquireRelease: v = 4; break;
649 case AtomicOrdering::SequentiallyConsistent: v = 5; break;
650 }
651 return IRB->getInt32(C: v);
652}
653
654// If a memset intrinsic gets inlined by the code gen, we will miss races on it.
655// So, we either need to ensure the intrinsic is not inlined, or instrument it.
656// We do not instrument memset/memmove/memcpy intrinsics (too complicated),
657// instead we simply replace them with regular function calls, which are then
658// intercepted by the run-time.
659// Since tsan is running after everyone else, the calls should not be
660// replaced back with intrinsics. If that becomes wrong at some point,
661// we will need to call e.g. __tsan_memset to avoid the intrinsics.
662bool ThreadSanitizer::instrumentMemIntrinsic(Instruction *I) {
663 InstrumentationIRBuilder IRB(I);
664 if (MemSetInst *M = dyn_cast<MemSetInst>(Val: I)) {
665 Value *Cast1 = IRB.CreateIntCast(V: M->getArgOperand(i: 1), DestTy: IRB.getInt32Ty(), isSigned: false);
666 Value *Cast2 = IRB.CreateIntCast(V: M->getArgOperand(i: 2), DestTy: IntptrTy, isSigned: false);
667 IRB.CreateCall(
668 Callee: MemsetFn,
669 Args: {M->getArgOperand(i: 0),
670 Cast1,
671 Cast2});
672 I->eraseFromParent();
673 } else if (MemTransferInst *M = dyn_cast<MemTransferInst>(Val: I)) {
674 IRB.CreateCall(
675 Callee: isa<MemCpyInst>(Val: M) ? MemcpyFn : MemmoveFn,
676 Args: {M->getArgOperand(i: 0),
677 M->getArgOperand(i: 1),
678 IRB.CreateIntCast(V: M->getArgOperand(i: 2), DestTy: IntptrTy, isSigned: false)});
679 I->eraseFromParent();
680 }
681 return false;
682}
683
684// Both llvm and ThreadSanitizer atomic operations are based on C++11/C1x
685// standards. For background see C++11 standard. A slightly older, publicly
686// available draft of the standard (not entirely up-to-date, but close enough
687// for casual browsing) is available here:
688// http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2011/n3242.pdf
689// The following page contains more background information:
690// http://www.hpl.hp.com/personal/Hans_Boehm/c++mm/
691
692bool ThreadSanitizer::instrumentAtomic(Instruction *I, const DataLayout &DL) {
693 InstrumentationIRBuilder IRB(I);
694 if (LoadInst *LI = dyn_cast<LoadInst>(Val: I)) {
695 Value *Addr = LI->getPointerOperand();
696 Type *OrigTy = LI->getType();
697 int Idx = getMemoryAccessFuncIndex(OrigTy, Addr, DL);
698 if (Idx < 0)
699 return false;
700 Value *Args[] = {Addr,
701 createOrdering(IRB: &IRB, ord: LI->getOrdering())};
702 Value *C = IRB.CreateCall(Callee: TsanAtomicLoad[Idx], Args);
703 Value *Cast = IRB.CreateBitOrPointerCast(V: C, DestTy: OrigTy);
704 I->replaceAllUsesWith(V: Cast);
705 I->eraseFromParent();
706 } else if (StoreInst *SI = dyn_cast<StoreInst>(Val: I)) {
707 Value *Addr = SI->getPointerOperand();
708 int Idx =
709 getMemoryAccessFuncIndex(OrigTy: SI->getValueOperand()->getType(), Addr, DL);
710 if (Idx < 0)
711 return false;
712 const unsigned ByteSize = 1U << Idx;
713 const unsigned BitSize = ByteSize * 8;
714 Type *Ty = Type::getIntNTy(C&: IRB.getContext(), N: BitSize);
715 Value *Args[] = {Addr,
716 IRB.CreateBitOrPointerCast(V: SI->getValueOperand(), DestTy: Ty),
717 createOrdering(IRB: &IRB, ord: SI->getOrdering())};
718 IRB.CreateCall(Callee: TsanAtomicStore[Idx], Args);
719 SI->eraseFromParent();
720 } else if (AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(Val: I)) {
721 Value *Addr = RMWI->getPointerOperand();
722 int Idx =
723 getMemoryAccessFuncIndex(OrigTy: RMWI->getValOperand()->getType(), Addr, DL);
724 if (Idx < 0)
725 return false;
726 FunctionCallee F = TsanAtomicRMW[RMWI->getOperation()][Idx];
727 if (!F)
728 return false;
729 const unsigned ByteSize = 1U << Idx;
730 const unsigned BitSize = ByteSize * 8;
731 Type *Ty = Type::getIntNTy(C&: IRB.getContext(), N: BitSize);
732 Value *Val = RMWI->getValOperand();
733 Value *Args[] = {Addr, IRB.CreateBitOrPointerCast(V: Val, DestTy: Ty),
734 createOrdering(IRB: &IRB, ord: RMWI->getOrdering())};
735 Value *C = IRB.CreateCall(Callee: F, Args);
736 I->replaceAllUsesWith(V: IRB.CreateBitOrPointerCast(V: C, DestTy: Val->getType()));
737 I->eraseFromParent();
738 } else if (AtomicCmpXchgInst *CASI = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
739 Value *Addr = CASI->getPointerOperand();
740 Type *OrigOldValTy = CASI->getNewValOperand()->getType();
741 int Idx = getMemoryAccessFuncIndex(OrigTy: OrigOldValTy, Addr, DL);
742 if (Idx < 0)
743 return false;
744 const unsigned ByteSize = 1U << Idx;
745 const unsigned BitSize = ByteSize * 8;
746 Type *Ty = Type::getIntNTy(C&: IRB.getContext(), N: BitSize);
747 Value *CmpOperand =
748 IRB.CreateBitOrPointerCast(V: CASI->getCompareOperand(), DestTy: Ty);
749 Value *NewOperand =
750 IRB.CreateBitOrPointerCast(V: CASI->getNewValOperand(), DestTy: Ty);
751 Value *Args[] = {Addr,
752 CmpOperand,
753 NewOperand,
754 createOrdering(IRB: &IRB, ord: CASI->getSuccessOrdering()),
755 createOrdering(IRB: &IRB, ord: CASI->getFailureOrdering())};
756 CallInst *C = IRB.CreateCall(Callee: TsanAtomicCAS[Idx], Args);
757 Value *Success = IRB.CreateICmpEQ(LHS: C, RHS: CmpOperand);
758 Value *OldVal = C;
759 if (Ty != OrigOldValTy) {
760 // The value is a pointer, so we need to cast the return value.
761 OldVal = IRB.CreateIntToPtr(V: C, DestTy: OrigOldValTy);
762 }
763
764 Value *Res =
765 IRB.CreateInsertValue(Agg: PoisonValue::get(T: CASI->getType()), Val: OldVal, Idxs: 0);
766 Res = IRB.CreateInsertValue(Agg: Res, Val: Success, Idxs: 1);
767
768 I->replaceAllUsesWith(V: Res);
769 I->eraseFromParent();
770 } else if (FenceInst *FI = dyn_cast<FenceInst>(Val: I)) {
771 Value *Args[] = {createOrdering(IRB: &IRB, ord: FI->getOrdering())};
772 FunctionCallee F = FI->getSyncScopeID() == SyncScope::SingleThread
773 ? TsanAtomicSignalFence
774 : TsanAtomicThreadFence;
775 IRB.CreateCall(Callee: F, Args);
776 FI->eraseFromParent();
777 }
778 return true;
779}
780
781int ThreadSanitizer::getMemoryAccessFuncIndex(Type *OrigTy, Value *Addr,
782 const DataLayout &DL) {
783 assert(OrigTy->isSized());
784 if (OrigTy->isScalableTy()) {
785 // FIXME: support vscale.
786 return -1;
787 }
788 uint32_t TypeSize = DL.getTypeStoreSizeInBits(Ty: OrigTy);
789 if (TypeSize != 8 && TypeSize != 16 &&
790 TypeSize != 32 && TypeSize != 64 && TypeSize != 128) {
791 NumAccessesWithBadSize++;
792 // Ignore all unusual sizes.
793 return -1;
794 }
795 size_t Idx = llvm::countr_zero(Val: TypeSize / 8);
796 assert(Idx < kNumberOfAccessSizes);
797 return Idx;
798}
799