| 1 | //===- ConcurrencySanitizer.cpp - watchpoint race detector ----------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file implements instrumentation for ConcurrencySanitizer, a sampling |
| 10 | // data-race detector. Memory accesses are preceded by runtime probes while the |
| 11 | // original operations remain unchanged. |
| 12 | // |
| 13 | //===----------------------------------------------------------------------===// |
| 14 | |
| 15 | #include "llvm/Transforms/Instrumentation/ConcurrencySanitizer.h" |
| 16 | #include "llvm/ADT/BitmaskEnum.h" |
| 17 | #include "llvm/ADT/DenseMap.h" |
| 18 | #include "llvm/ADT/SmallVector.h" |
| 19 | #include "llvm/ADT/Statistic.h" |
| 20 | #include "llvm/ADT/bit.h" |
| 21 | #include "llvm/Analysis/CaptureTracking.h" |
| 22 | #include "llvm/Analysis/TargetLibraryInfo.h" |
| 23 | #include "llvm/Analysis/ValueTracking.h" |
| 24 | #include "llvm/IR/DataLayout.h" |
| 25 | #include "llvm/IR/IRBuilder.h" |
| 26 | #include "llvm/IR/Instructions.h" |
| 27 | #include "llvm/IR/IntrinsicInst.h" |
| 28 | #include "llvm/IR/Intrinsics.h" |
| 29 | #include "llvm/IR/Module.h" |
| 30 | #include "llvm/ProfileData/InstrProf.h" |
| 31 | #include "llvm/Support/AMDGPUAddrSpace.h" |
| 32 | #include "llvm/Support/AtomicOrdering.h" |
| 33 | #include "llvm/Support/CommandLine.h" |
| 34 | #include "llvm/Support/NVPTXAddrSpace.h" |
| 35 | #include "llvm/TargetParser/Triple.h" |
| 36 | #include "llvm/Transforms/Utils/EscapeEnumerator.h" |
| 37 | #include "llvm/Transforms/Utils/Instrumentation.h" |
| 38 | #include "llvm/Transforms/Utils/Local.h" |
| 39 | #include "llvm/Transforms/Utils/ModuleUtils.h" |
| 40 | #include <optional> |
| 41 | |
| 42 | using namespace llvm; |
| 43 | |
| 44 | #define DEBUG_TYPE "csan" |
| 45 | |
| 46 | static cl::opt<bool> ClInstrumentMemoryAccesses( |
| 47 | "csan-instrument-memory-accesses" , cl::init(Val: true), |
| 48 | cl::desc("Instrument memory accesses" ), cl::Hidden); |
| 49 | static cl::opt<bool> |
| 50 | ClInstrumentFuncEntryExit("csan-instrument-func-entry-exit" , cl::init(Val: true), |
| 51 | cl::desc("Instrument function entry and exit" ), |
| 52 | cl::Hidden); |
| 53 | static cl::opt<bool> ClHandleCxxExceptions( |
| 54 | "csan-handle-cxx-exceptions" , cl::init(Val: true), |
| 55 | cl::desc("Handle C++ exceptions (insert cleanup blocks for unwinding)" ), |
| 56 | cl::Hidden); |
| 57 | static cl::opt<bool> ClInstrumentAtomics("csan-instrument-atomics" , |
| 58 | cl::init(Val: true), |
| 59 | cl::desc("Instrument atomics" ), |
| 60 | cl::Hidden); |
| 61 | static cl::opt<bool> ClInstrumentMemIntrinsics( |
| 62 | "csan-instrument-memintrinsics" , cl::init(Val: true), |
| 63 | cl::desc("Instrument memintrinsics (memset/memcpy/memmove)" ), cl::Hidden); |
| 64 | static cl::opt<bool> ClDistinguishVolatile( |
| 65 | "csan-distinguish-volatile" , cl::init(Val: false), |
| 66 | cl::desc("Treat aligned volatile accesses up to 8 bytes as atomic" ), |
| 67 | cl::Hidden); |
| 68 | STATISTIC(NumInstrumentedReads, "Number of instrumented reads" ); |
| 69 | STATISTIC(NumInstrumentedWrites, "Number of instrumented writes" ); |
| 70 | STATISTIC(NumOmittedReadsBeforeWrite, |
| 71 | "Number of reads ignored due to following writes" ); |
| 72 | |
| 73 | static constexpr char kCsanModuleCtorName[] = "csan.module_ctor" ; |
| 74 | static constexpr char kCsanInitName[] = "__csan_init" ; |
| 75 | |
| 76 | // Accesses sizes are powers of two: 1, 2, 4, 8, 16. |
| 77 | static constexpr size_t kNumAccessSizes = 5; |
| 78 | static constexpr StringLiteral kCsanReadNames[kNumAccessSizes] = { |
| 79 | "__csan_read1" , "__csan_read2" , "__csan_read4" , "__csan_read8" , |
| 80 | "__csan_read16" }; |
| 81 | static constexpr StringLiteral kCsanWriteNames[kNumAccessSizes] = { |
| 82 | "__csan_write1" , "__csan_write2" , "__csan_write4" , "__csan_write8" , |
| 83 | "__csan_write16" }; |
| 84 | static constexpr StringLiteral kCsanUnalignedReadNames[kNumAccessSizes] = { |
| 85 | "__csan_unaligned_read1" , "__csan_unaligned_read2" , |
| 86 | "__csan_unaligned_read4" , "__csan_unaligned_read8" , |
| 87 | "__csan_unaligned_read16" }; |
| 88 | static constexpr StringLiteral kCsanUnalignedWriteNames[kNumAccessSizes] = { |
| 89 | "__csan_unaligned_write1" , "__csan_unaligned_write2" , |
| 90 | "__csan_unaligned_write4" , "__csan_unaligned_write8" , |
| 91 | "__csan_unaligned_write16" }; |
| 92 | static constexpr StringLiteral kCsanCompoundRWNames[kNumAccessSizes] = { |
| 93 | "__csan_read_write1" , "__csan_read_write2" , "__csan_read_write4" , |
| 94 | "__csan_read_write8" , "__csan_read_write16" }; |
| 95 | static constexpr StringLiteral kCsanUnalignedCompoundRWNames[kNumAccessSizes] = |
| 96 | {"__csan_unaligned_read_write1" , "__csan_unaligned_read_write2" , |
| 97 | "__csan_unaligned_read_write4" , "__csan_unaligned_read_write8" , |
| 98 | "__csan_unaligned_read_write16" }; |
| 99 | |
| 100 | namespace { |
| 101 | |
| 102 | /// Must match CSAN_ACCESS_* in compiler-rt/lib/csan/csan_defs.h. |
| 103 | enum class AccessFlags : unsigned { |
| 104 | None = 0, |
| 105 | Atomic = 1u << 0, |
| 106 | Compound = 1u << 1, |
| 107 | LLVM_MARK_AS_BITMASK_ENUM(Compound), |
| 108 | }; |
| 109 | |
| 110 | static bool isAtomicMemoryAccess(const Instruction *I) { |
| 111 | std::optional<SyncScope::ID> SSID = getAtomicSyncScopeID(I); |
| 112 | if (!SSID) |
| 113 | return false; |
| 114 | if (isa<LoadInst>(Val: I) || isa<StoreInst>(Val: I)) |
| 115 | return *SSID != SyncScope::SingleThread; |
| 116 | return true; |
| 117 | } |
| 118 | |
| 119 | static Value *getCallbackAddress(IRBuilderBase &IRB, Value *Addr) { |
| 120 | return IRB.CreateAddrSpaceCast(V: Addr, DestTy: IRB.getPtrTy()); |
| 121 | } |
| 122 | |
| 123 | static ConstantInt *createOrdering(IRBuilderBase &IRB, AtomicOrdering Ord) { |
| 124 | return IRB.getInt32(C: static_cast<uint32_t>(toCABI(AO: Ord))); |
| 125 | } |
| 126 | |
| 127 | static int getAccessSizeIndex(Type *Ty, const DataLayout &DL) { |
| 128 | assert(Ty->isSized()); |
| 129 | if (Ty->isScalableTy()) |
| 130 | return -1; |
| 131 | uint32_t TypeSize = DL.getTypeStoreSizeInBits(Ty); |
| 132 | if (TypeSize != 8 && TypeSize != 16 && TypeSize != 32 && TypeSize != 64 && |
| 133 | TypeSize != 128) |
| 134 | return -1; |
| 135 | unsigned Idx = llvm::countr_zero(Val: TypeSize / 8); |
| 136 | return static_cast<int>(Idx); |
| 137 | } |
| 138 | |
| 139 | // A later write of the same pointer drops an earlier read in this window. |
| 140 | static void selectMemoryAccesses(SmallVectorImpl<Instruction *> &Local, |
| 141 | SmallVectorImpl<Instruction *> &Out) { |
| 142 | DenseMap<Value *, StoreInst *> WriteTargets; |
| 143 | for (Instruction *Inst : reverse(C&: Local)) { |
| 144 | Value *Addr = getLoadStorePointerOperand(V: Inst); |
| 145 | if (StoreInst *Store = dyn_cast<StoreInst>(Val: Inst)) { |
| 146 | WriteTargets[Addr] = Store; |
| 147 | Out.push_back(Elt: Inst); |
| 148 | continue; |
| 149 | } |
| 150 | |
| 151 | // A volatile write is reported as atomic when distinguished, so it does |
| 152 | // not cover a plain read. Keep the read if either access is volatile. |
| 153 | StoreInst *LaterWrite = WriteTargets.lookup(Val: Addr); |
| 154 | bool AnyVolatile = |
| 155 | LaterWrite && ClDistinguishVolatile && |
| 156 | (cast<LoadInst>(Val: Inst)->isVolatile() || LaterWrite->isVolatile()); |
| 157 | if (LaterWrite && !AnyVolatile) { |
| 158 | ++NumOmittedReadsBeforeWrite; |
| 159 | continue; |
| 160 | } |
| 161 | |
| 162 | Out.push_back(Elt: Inst); |
| 163 | } |
| 164 | Local.clear(); |
| 165 | } |
| 166 | |
| 167 | static bool addressSpaceMayRace(const Triple &T, unsigned AS) { |
| 168 | if (T.isAMDGPU()) { |
| 169 | // GDS and buffer fat pointers cannot form a generic watchpoint key. |
| 170 | return AS == AMDGPUAS::FLAT_ADDRESS || AS == AMDGPUAS::GLOBAL_ADDRESS || |
| 171 | AS == AMDGPUAS::LOCAL_ADDRESS; |
| 172 | } |
| 173 | if (T.isNVPTX()) { |
| 174 | return AS == NVPTXAS::ADDRESS_SPACE_GENERIC || |
| 175 | AS == NVPTXAS::ADDRESS_SPACE_GLOBAL || |
| 176 | AS == NVPTXAS::ADDRESS_SPACE_SHARED || |
| 177 | AS == NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER; |
| 178 | } |
| 179 | if (T.isSPIRV()) { |
| 180 | // FIXME: No exposed address spaces for SPIR-V. |
| 181 | return false; |
| 182 | } |
| 183 | return AS == 0; |
| 184 | } |
| 185 | |
| 186 | struct ConcurrencySanitizer { |
| 187 | bool sanitizeFunction(Function &F, const TargetLibraryInfo &TLI); |
| 188 | |
| 189 | private: |
| 190 | struct MemoryAccessLists { |
| 191 | SmallVector<Instruction *, 8> LoadsAndStores; |
| 192 | SmallVector<Instruction *, 8> AtomicAccesses; |
| 193 | SmallVector<IntrinsicInst *, 8> MemIntrinCalls; |
| 194 | bool HasCalls = false; |
| 195 | }; |
| 196 | |
| 197 | void initialize(Module &M, const TargetLibraryInfo &TLI); |
| 198 | void collectMemoryAccesses(Function &F, MemoryAccessLists &Out); |
| 199 | bool instrumentLoadOrStore(Instruction *I, const DataLayout &DL); |
| 200 | bool instrumentAtomic(Instruction *I, const DataLayout &DL); |
| 201 | bool instrumentMemIntrinsic(IntrinsicInst *II, const DataLayout &DL); |
| 202 | bool insertAccessProbe(Instruction *I, Value *Addr, Type *AccessTy, |
| 203 | const DataLayout &DL, bool IsWrite, bool IsCompound, |
| 204 | bool IsAtomic); |
| 205 | bool shouldInstrumentAddress(Value *Addr) const; |
| 206 | bool shouldInstrumentAccess(Instruction *I) const; |
| 207 | void insertFuncEntryExit(Function &F); |
| 208 | void insertRuntimeIgnores(Function &F); |
| 209 | Constant *getFlags(AccessFlags Flags) const { |
| 210 | return ConstantInt::get(Ty: FlagsTy, V: to_underlying(E: Flags)); |
| 211 | } |
| 212 | |
| 213 | Module *Mod = nullptr; |
| 214 | Type *IntptrTy = nullptr; |
| 215 | IntegerType *FlagsTy = nullptr; |
| 216 | |
| 217 | // void __csan_readN(ptr, i32); |
| 218 | FunctionCallee CsanRead[kNumAccessSizes]; |
| 219 | // void __csan_writeN(ptr, i32); |
| 220 | FunctionCallee CsanWrite[kNumAccessSizes]; |
| 221 | // void __csan_unaligned_readN(ptr, i32); |
| 222 | FunctionCallee CsanUnalignedRead[kNumAccessSizes]; |
| 223 | // void __csan_unaligned_writeN(ptr, i32); |
| 224 | FunctionCallee CsanUnalignedWrite[kNumAccessSizes]; |
| 225 | // void __csan_read_writeN(ptr, i32); |
| 226 | FunctionCallee CsanCompoundRW[kNumAccessSizes]; |
| 227 | // void __csan_unaligned_read_writeN(ptr, i32); |
| 228 | FunctionCallee CsanUnalignedCompoundRW[kNumAccessSizes]; |
| 229 | // void __csan_func_entry(ptr); |
| 230 | FunctionCallee CsanFuncEntry; |
| 231 | // void __csan_func_exit(); |
| 232 | FunctionCallee CsanFuncExit; |
| 233 | // void __csan_ignore_thread_begin(); |
| 234 | FunctionCallee CsanIgnoreBegin; |
| 235 | // void __csan_ignore_thread_end(); |
| 236 | FunctionCallee CsanIgnoreEnd; |
| 237 | // void __csan_read_range(ptr, intptr_t, i32); |
| 238 | FunctionCallee CsanReadRange; |
| 239 | // void __csan_write_range(ptr, intptr_t, i32); |
| 240 | FunctionCallee CsanWriteRange; |
| 241 | // void __csan_atomic_thread_fence(i32); |
| 242 | FunctionCallee CsanAtomicThreadFence; |
| 243 | // void __csan_atomic_signal_fence(i32); |
| 244 | FunctionCallee CsanAtomicSignalFence; |
| 245 | }; |
| 246 | |
| 247 | void insertModuleCtor(Module &M) { |
| 248 | getOrCreateSanitizerCtorAndInitFunctions( |
| 249 | M, CtorName: kCsanModuleCtorName, InitName: kCsanInitName, /*InitArgTypes=*/{}, |
| 250 | /*InitArgs=*/{}, |
| 251 | FunctionsCreatedCallback: [&](Function *Ctor, FunctionCallee) { appendToGlobalCtors(M, F: Ctor, Priority: 0); }); |
| 252 | } |
| 253 | |
| 254 | } // namespace |
| 255 | |
| 256 | PreservedAnalyses ConcurrencySanitizerPass::run(Function &F, |
| 257 | FunctionAnalysisManager &FAM) { |
| 258 | ConcurrencySanitizer CSan; |
| 259 | if (CSan.sanitizeFunction(F, TLI: FAM.getResult<TargetLibraryAnalysis>(IR&: F))) |
| 260 | return PreservedAnalyses::none(); |
| 261 | return PreservedAnalyses::all(); |
| 262 | } |
| 263 | |
| 264 | PreservedAnalyses ModuleConcurrencySanitizerPass::run(Module &M, |
| 265 | ModuleAnalysisManager &) { |
| 266 | if (checkIfAlreadyInstrumented(M, Flag: "nosanitize_concurrency" )) |
| 267 | return PreservedAnalyses::all(); |
| 268 | insertModuleCtor(M); |
| 269 | return PreservedAnalyses::none(); |
| 270 | } |
| 271 | |
| 272 | void ConcurrencySanitizer::initialize(Module &M, const TargetLibraryInfo &TLI) { |
| 273 | LLVMContext &Ctx = M.getContext(); |
| 274 | Mod = &M; |
| 275 | IntptrTy = M.getDataLayout().getIntPtrType(C&: Ctx); |
| 276 | FlagsTy = Type::getInt32Ty(C&: Ctx); |
| 277 | |
| 278 | AttributeList Attr = AttributeList().addFnAttribute(C&: Ctx, Kind: Attribute::NoUnwind); |
| 279 | IRBuilder<> IRB(M); |
| 280 | Type *VoidTy = IRB.getVoidTy(); |
| 281 | Type *PtrTy = IRB.getPtrTy(); |
| 282 | CsanFuncEntry = |
| 283 | M.getOrInsertFunction(Name: "__csan_func_entry" , AttributeList: Attr, RetTy: VoidTy, Args: PtrTy); |
| 284 | CsanFuncExit = M.getOrInsertFunction(Name: "__csan_func_exit" , AttributeList: Attr, RetTy: VoidTy); |
| 285 | CsanIgnoreBegin = |
| 286 | M.getOrInsertFunction(Name: "__csan_ignore_thread_begin" , AttributeList: Attr, RetTy: VoidTy); |
| 287 | CsanIgnoreEnd = |
| 288 | M.getOrInsertFunction(Name: "__csan_ignore_thread_end" , AttributeList: Attr, RetTy: VoidTy); |
| 289 | auto AccessFn = [&](StringRef Name) { |
| 290 | return M.getOrInsertFunction(Name, AttributeList: Attr, RetTy: VoidTy, Args: PtrTy, Args: FlagsTy); |
| 291 | }; |
| 292 | for (unsigned I = 0; I < kNumAccessSizes; ++I) { |
| 293 | CsanRead[I] = AccessFn(kCsanReadNames[I]); |
| 294 | CsanWrite[I] = AccessFn(kCsanWriteNames[I]); |
| 295 | CsanUnalignedRead[I] = AccessFn(kCsanUnalignedReadNames[I]); |
| 296 | CsanUnalignedWrite[I] = AccessFn(kCsanUnalignedWriteNames[I]); |
| 297 | CsanCompoundRW[I] = AccessFn(kCsanCompoundRWNames[I]); |
| 298 | CsanUnalignedCompoundRW[I] = AccessFn(kCsanUnalignedCompoundRWNames[I]); |
| 299 | } |
| 300 | IntegerType *OrdTy = IRB.getInt32Ty(); |
| 301 | CsanReadRange = M.getOrInsertFunction(Name: "__csan_read_range" , AttributeList: Attr, RetTy: VoidTy, |
| 302 | Args: PtrTy, Args: IntptrTy, Args: FlagsTy); |
| 303 | CsanWriteRange = M.getOrInsertFunction(Name: "__csan_write_range" , AttributeList: Attr, RetTy: VoidTy, |
| 304 | Args: PtrTy, Args: IntptrTy, Args: FlagsTy); |
| 305 | CsanAtomicThreadFence = M.getOrInsertFunction( |
| 306 | Name: "__csan_atomic_thread_fence" , |
| 307 | AttributeList: TLI.getAttrList(C: &Ctx, ArgNos: {0}, /*Signed=*/true, /*Ret=*/false, AL: Attr), RetTy: VoidTy, |
| 308 | Args: OrdTy); |
| 309 | CsanAtomicSignalFence = M.getOrInsertFunction( |
| 310 | Name: "__csan_atomic_signal_fence" , |
| 311 | AttributeList: TLI.getAttrList(C: &Ctx, ArgNos: {0}, /*Signed=*/true, /*Ret=*/false, AL: Attr), RetTy: VoidTy, |
| 312 | Args: OrdTy); |
| 313 | } |
| 314 | |
| 315 | bool ConcurrencySanitizer::sanitizeFunction(Function &F, |
| 316 | const TargetLibraryInfo &TLI) { |
| 317 | // This is required to prevent instrumenting call to __csan_init from within |
| 318 | // the module constructor. |
| 319 | if (F.getName() == kCsanModuleCtorName) |
| 320 | return false; |
| 321 | // Naked functions can not have prologue/epilogue |
| 322 | // (__csan_func_entry/__csan_func_exit) generated, so don't instrument them at |
| 323 | // all. |
| 324 | if (F.hasFnAttribute(Kind: Attribute::Naked)) |
| 325 | return false; |
| 326 | |
| 327 | // __attribute__(disable_sanitizer_instrumentation) prevents all kinds of |
| 328 | // instrumentation. |
| 329 | if (F.hasFnAttribute(Kind: Attribute::DisableSanitizerInstrumentation)) |
| 330 | return false; |
| 331 | |
| 332 | Mod = F.getParent(); |
| 333 | MemoryAccessLists Acc; |
| 334 | collectMemoryAccesses(F, Out&: Acc); |
| 335 | |
| 336 | const bool SuppressChecking = |
| 337 | F.hasFnAttribute(Kind: "sanitize_concurrency_no_checking_at_run_time" ); |
| 338 | const bool SanitizeFunction = |
| 339 | F.hasFnAttribute(Kind: Attribute::SanitizeConcurrency) && !SuppressChecking; |
| 340 | const bool MayInstrument = |
| 341 | SanitizeFunction && |
| 342 | ((ClInstrumentMemoryAccesses && !Acc.LoadsAndStores.empty()) || |
| 343 | (ClInstrumentAtomics && !Acc.AtomicAccesses.empty()) || |
| 344 | (ClInstrumentMemIntrinsics && !Acc.MemIntrinCalls.empty())); |
| 345 | const bool NeedsRuntimeIgnores = SuppressChecking && Acc.HasCalls; |
| 346 | const bool NeedsFuncEntryExit = ClInstrumentFuncEntryExit && Acc.HasCalls; |
| 347 | if (!MayInstrument && !NeedsRuntimeIgnores && !NeedsFuncEntryExit) |
| 348 | return false; |
| 349 | |
| 350 | initialize(M&: *Mod, TLI); |
| 351 | bool Res = false; |
| 352 | const DataLayout &DL = F.getDataLayout(); |
| 353 | |
| 354 | if (ClInstrumentMemoryAccesses && SanitizeFunction) |
| 355 | for (Instruction *I : Acc.LoadsAndStores) |
| 356 | Res |= instrumentLoadOrStore(I, DL); |
| 357 | |
| 358 | if (ClInstrumentAtomics && SanitizeFunction) |
| 359 | for (Instruction *I : Acc.AtomicAccesses) |
| 360 | Res |= instrumentAtomic(I, DL); |
| 361 | |
| 362 | if (ClInstrumentMemIntrinsics && SanitizeFunction) |
| 363 | for (IntrinsicInst *II : Acc.MemIntrinCalls) |
| 364 | Res |= instrumentMemIntrinsic(II, DL); |
| 365 | |
| 366 | if (NeedsRuntimeIgnores) { |
| 367 | insertRuntimeIgnores(F); |
| 368 | Res = true; |
| 369 | } |
| 370 | |
| 371 | if ((Res || Acc.HasCalls) && ClInstrumentFuncEntryExit) { |
| 372 | insertFuncEntryExit(F); |
| 373 | Res = true; |
| 374 | } |
| 375 | return Res; |
| 376 | } |
| 377 | |
| 378 | bool ConcurrencySanitizer::shouldInstrumentAddress(Value *Addr) const { |
| 379 | Value *BaseAddr = Addr->stripInBoundsOffsets(); |
| 380 | if (auto *GV = dyn_cast<GlobalVariable>(Val: BaseAddr)) { |
| 381 | if (GV->hasSection()) { |
| 382 | StringRef SectionName = GV->getSection(); |
| 383 | Triple::ObjectFormatType OF = Mod->getTargetTriple().getObjectFormat(); |
| 384 | if (SectionName.ends_with( |
| 385 | Suffix: getInstrProfSectionName(IPSK: IPSK_cnts, OF, /*AddSegmentInfo=*/false))) |
| 386 | return false; |
| 387 | } |
| 388 | } |
| 389 | |
| 390 | Type *PtrTy = cast<PointerType>(Val: Addr->getType()->getScalarType()); |
| 391 | unsigned AS = PtrTy->getPointerAddressSpace(); |
| 392 | if (Mod->getDataLayout().getPointerSizeInBits(AS) > 64) |
| 393 | return false; |
| 394 | return addressSpaceMayRace(T: Mod->getTargetTriple(), AS); |
| 395 | } |
| 396 | |
| 397 | bool ConcurrencySanitizer::shouldInstrumentAccess(Instruction *I) const { |
| 398 | const bool IsWrite = isa<StoreInst>(Val: I); |
| 399 | Value *Addr = getLoadStorePointerOperand(V: I); |
| 400 | if (!shouldInstrumentAddress(Addr)) |
| 401 | return false; |
| 402 | |
| 403 | if (!IsWrite) |
| 404 | if (auto *GV = dyn_cast<GlobalVariable>(Val: getUnderlyingObject(V: Addr))) |
| 405 | if (GV->isConstant()) |
| 406 | return false; |
| 407 | |
| 408 | const AllocaInst *AI = findAllocaForValue(V: Addr); |
| 409 | if (!AI || PointerMayBeCaptured(V: AI, /*ReturnCaptures=*/true)) |
| 410 | return true; |
| 411 | |
| 412 | const Triple &T = AI->getModule()->getTargetTriple(); |
| 413 | return T.isGPU() && addressSpaceMayRace(T, AS: AI->getAddressSpace()); |
| 414 | } |
| 415 | |
| 416 | void ConcurrencySanitizer::collectMemoryAccesses(Function &F, |
| 417 | MemoryAccessLists &Out) { |
| 418 | SmallVector<Instruction *, 8> LocalLoadsAndStores; |
| 419 | |
| 420 | for (BasicBlock &BB : F) { |
| 421 | for (Instruction &Inst : BB) { |
| 422 | // Skip instructions inserted by another instrumentation. |
| 423 | if (Inst.hasMetadata(KindID: LLVMContext::MD_nosanitize)) |
| 424 | continue; |
| 425 | if (isAtomicMemoryAccess(I: &Inst)) |
| 426 | Out.AtomicAccesses.push_back(Elt: &Inst); |
| 427 | else if ((isa<LoadInst>(Val: Inst) || isa<StoreInst>(Val: Inst)) && |
| 428 | shouldInstrumentAccess(I: &Inst)) |
| 429 | LocalLoadsAndStores.push_back(Elt: &Inst); |
| 430 | else if (isa<CallBase>(Val: Inst)) { |
| 431 | if (isa<AnyMemIntrinsic, MemSetPatternInst>(Val: Inst)) |
| 432 | Out.MemIntrinCalls.push_back(Elt: cast<IntrinsicInst>(Val: &Inst)); |
| 433 | Out.HasCalls = true; |
| 434 | selectMemoryAccesses(Local&: LocalLoadsAndStores, Out&: Out.LoadsAndStores); |
| 435 | } |
| 436 | } |
| 437 | selectMemoryAccesses(Local&: LocalLoadsAndStores, Out&: Out.LoadsAndStores); |
| 438 | } |
| 439 | } |
| 440 | |
| 441 | bool ConcurrencySanitizer::instrumentLoadOrStore(Instruction *I, |
| 442 | const DataLayout &DL) { |
| 443 | const bool IsWrite = isa<StoreInst>(Val: I); |
| 444 | Value *Addr = getLoadStorePointerOperand(V: I); |
| 445 | if (!insertAccessProbe(I, Addr, AccessTy: getLoadStoreType(I), DL, IsWrite, |
| 446 | /*IsCompound=*/false, /*IsAtomic=*/false)) |
| 447 | return false; |
| 448 | if (IsWrite) |
| 449 | ++NumInstrumentedWrites; |
| 450 | else |
| 451 | ++NumInstrumentedReads; |
| 452 | return true; |
| 453 | } |
| 454 | |
| 455 | bool ConcurrencySanitizer::instrumentAtomic(Instruction *I, |
| 456 | const DataLayout &DL) { |
| 457 | if (auto *FI = dyn_cast<FenceInst>(Val: I)) { |
| 458 | InstrumentationIRBuilder IRB(I); |
| 459 | FunctionCallee Fn = FI->getSyncScopeID() == SyncScope::SingleThread |
| 460 | ? CsanAtomicSignalFence |
| 461 | : CsanAtomicThreadFence; |
| 462 | IRB.CreateCall(Callee: Fn, Args: createOrdering(IRB, Ord: FI->getOrdering())); |
| 463 | return true; |
| 464 | } |
| 465 | |
| 466 | Value *Addr = nullptr; |
| 467 | Type *AccessTy = nullptr; |
| 468 | bool IsWrite = true; |
| 469 | bool IsCompound = false; |
| 470 | if (auto *LI = dyn_cast<LoadInst>(Val: I)) { |
| 471 | Addr = LI->getPointerOperand(); |
| 472 | AccessTy = LI->getType(); |
| 473 | IsWrite = false; |
| 474 | } else if (auto *SI = dyn_cast<StoreInst>(Val: I)) { |
| 475 | Addr = SI->getPointerOperand(); |
| 476 | AccessTy = SI->getValueOperand()->getType(); |
| 477 | } else if (auto *RMW = dyn_cast<AtomicRMWInst>(Val: I)) { |
| 478 | Addr = RMW->getPointerOperand(); |
| 479 | AccessTy = RMW->getValOperand()->getType(); |
| 480 | IsCompound = true; |
| 481 | } else if (auto *CAS = dyn_cast<AtomicCmpXchgInst>(Val: I)) { |
| 482 | // A failed CAS is still considered a write as the output is unkonwn. |
| 483 | Addr = CAS->getPointerOperand(); |
| 484 | AccessTy = CAS->getNewValOperand()->getType(); |
| 485 | IsCompound = true; |
| 486 | } else { |
| 487 | return false; |
| 488 | } |
| 489 | |
| 490 | if (!shouldInstrumentAddress(Addr)) |
| 491 | return false; |
| 492 | if (!insertAccessProbe(I, Addr, AccessTy, DL, IsWrite, IsCompound, |
| 493 | /*IsAtomic=*/true)) |
| 494 | return false; |
| 495 | if (IsCompound || IsWrite) |
| 496 | ++NumInstrumentedWrites; |
| 497 | if (IsCompound || !IsWrite) |
| 498 | ++NumInstrumentedReads; |
| 499 | return true; |
| 500 | } |
| 501 | |
| 502 | bool ConcurrencySanitizer::instrumentMemIntrinsic(IntrinsicInst *II, |
| 503 | const DataLayout &DL) { |
| 504 | auto *MI = dyn_cast<AnyMemIntrinsic>(Val: II); |
| 505 | auto *MSP = dyn_cast<MemSetPatternInst>(Val: II); |
| 506 | Value *Dst = MI ? MI->getRawDest() : MSP->getRawDest(); |
| 507 | Value *Src = nullptr; |
| 508 | if (auto *MT = dyn_cast<AnyMemTransferInst>(Val: II)) |
| 509 | Src = MT->getRawSource(); |
| 510 | |
| 511 | bool InstrumentRead = Src && shouldInstrumentAddress(Addr: Src); |
| 512 | bool InstrumentWrite = shouldInstrumentAddress(Addr: Dst); |
| 513 | if (!InstrumentRead && !InstrumentWrite) |
| 514 | return false; |
| 515 | |
| 516 | InstrumentationIRBuilder IRB(II); |
| 517 | Value *Len = IRB.CreateIntCast(V: MI ? MI->getLength() : MSP->getLength(), |
| 518 | DestTy: IntptrTy, isSigned: false); |
| 519 | // The length of a pattern memset counts patterns, not bytes. |
| 520 | if (MSP) { |
| 521 | TypeSize Size = DL.getTypeAllocSize(Ty: MSP->getValue()->getType()); |
| 522 | Len = IRB.CreateMul(LHS: Len, RHS: IRB.CreateTypeSize(Ty: IntptrTy, Size)); |
| 523 | } |
| 524 | Constant *Flags = |
| 525 | getFlags(Flags: MI && MI->isAtomic() ? AccessFlags::Atomic : AccessFlags::None); |
| 526 | if (InstrumentRead) { |
| 527 | IRB.CreateCall(Callee: CsanReadRange, Args: {getCallbackAddress(IRB, Addr: Src), Len, Flags}); |
| 528 | ++NumInstrumentedReads; |
| 529 | } |
| 530 | if (InstrumentWrite) { |
| 531 | IRB.CreateCall(Callee: CsanWriteRange, Args: {getCallbackAddress(IRB, Addr: Dst), Len, Flags}); |
| 532 | ++NumInstrumentedWrites; |
| 533 | } |
| 534 | return true; |
| 535 | } |
| 536 | |
| 537 | bool ConcurrencySanitizer::insertAccessProbe(Instruction *I, Value *Addr, |
| 538 | Type *AccessTy, |
| 539 | const DataLayout &DL, bool IsWrite, |
| 540 | bool IsCompound, bool IsAtomic) { |
| 541 | if (Addr->isSwiftError()) |
| 542 | return false; |
| 543 | AccessFlags Flags = AccessFlags::None; |
| 544 | if (IsAtomic) |
| 545 | Flags |= AccessFlags::Atomic; |
| 546 | if (IsCompound) |
| 547 | Flags |= AccessFlags::Compound; |
| 548 | int Idx = getAccessSizeIndex(Ty: AccessTy, DL); |
| 549 | if (Idx < 0) { |
| 550 | if (IsCompound) |
| 551 | return false; |
| 552 | InstrumentationIRBuilder IRB(I); |
| 553 | Value *Len = IRB.CreateTypeSize(Ty: IntptrTy, Size: DL.getTypeStoreSize(Ty: AccessTy)); |
| 554 | IRB.CreateCall(Callee: IsWrite ? CsanWriteRange : CsanReadRange, |
| 555 | Args: {getCallbackAddress(IRB, Addr), Len, getFlags(Flags)}); |
| 556 | return true; |
| 557 | } |
| 558 | |
| 559 | Align Alignment = Align(1); |
| 560 | bool IsVolatile = false; |
| 561 | if (auto *LI = dyn_cast<LoadInst>(Val: I)) { |
| 562 | Alignment = LI->getAlign(); |
| 563 | IsVolatile = LI->isVolatile(); |
| 564 | } else if (auto *SI = dyn_cast<StoreInst>(Val: I)) { |
| 565 | Alignment = SI->getAlign(); |
| 566 | IsVolatile = SI->isVolatile(); |
| 567 | } else if (auto *RMW = dyn_cast<AtomicRMWInst>(Val: I)) { |
| 568 | Alignment = RMW->getAlign(); |
| 569 | IsVolatile = RMW->isVolatile(); |
| 570 | } else if (auto *CAS = dyn_cast<AtomicCmpXchgInst>(Val: I)) { |
| 571 | Alignment = CAS->getAlign(); |
| 572 | IsVolatile = CAS->isVolatile(); |
| 573 | } |
| 574 | |
| 575 | uint32_t TypeSize = DL.getTypeStoreSizeInBits(Ty: AccessTy); |
| 576 | bool Unaligned = |
| 577 | Alignment < Align(8) && Alignment.value() % (TypeSize / 8) != 0; |
| 578 | FunctionCallee Callback; |
| 579 | if (IsCompound) |
| 580 | Callback = Unaligned ? CsanUnalignedCompoundRW[Idx] : CsanCompoundRW[Idx]; |
| 581 | else if (Unaligned) |
| 582 | Callback = IsWrite ? CsanUnalignedWrite[Idx] : CsanUnalignedRead[Idx]; |
| 583 | else |
| 584 | Callback = IsWrite ? CsanWrite[Idx] : CsanRead[Idx]; |
| 585 | |
| 586 | InstrumentationIRBuilder IRB(I); |
| 587 | Value *CallbackAddr = getCallbackAddress(IRB, Addr); |
| 588 | Value *FlagsVal = getFlags(Flags); |
| 589 | if (ClDistinguishVolatile && IsVolatile && TypeSize <= 64) { |
| 590 | // The declared alignment may understate the pointer's real alignment. |
| 591 | Value *IsAligned = IRB.getTrue(); |
| 592 | if (Unaligned) |
| 593 | IsAligned = IRB.CreateIsNull(Arg: IRB.CreateAnd( |
| 594 | LHS: IRB.CreatePtrToInt(V: CallbackAddr, DestTy: IntptrTy), RHS: TypeSize / 8 - 1)); |
| 595 | FlagsVal = IRB.CreateSelect( |
| 596 | C: IsAligned, True: getFlags(Flags: Flags | AccessFlags::Atomic), False: FlagsVal); |
| 597 | } |
| 598 | IRB.CreateCall(Callee: Callback, Args: {CallbackAddr, FlagsVal}); |
| 599 | return true; |
| 600 | } |
| 601 | |
| 602 | void ConcurrencySanitizer::insertFuncEntryExit(Function &F) { |
| 603 | const DataLayout &DL = F.getDataLayout(); |
| 604 | InstrumentationIRBuilder IRB(&*F.getEntryBlock().getFirstNonPHIIt()); |
| 605 | Type *ProgramAsPtrTy = |
| 606 | PointerType::get(C&: F.getContext(), AddressSpace: DL.getProgramAddressSpace()); |
| 607 | Value *ReturnAddress = IRB.CreateIntrinsic(ID: Intrinsic::returnaddress, |
| 608 | OverloadTypes: {ProgramAsPtrTy}, Args: IRB.getInt32(C: 0)); |
| 609 | IRB.CreateCall(Callee: CsanFuncEntry, Args: ReturnAddress); |
| 610 | |
| 611 | EscapeEnumerator EE(F, "csan_cleanup" , ClHandleCxxExceptions); |
| 612 | while (IRBuilder<> *AtExit = EE.Next()) { |
| 613 | InstrumentationIRBuilder::ensureDebugInfo(IRB&: *AtExit, F); |
| 614 | AtExit->CreateCall(Callee: CsanFuncExit); |
| 615 | } |
| 616 | } |
| 617 | |
| 618 | void ConcurrencySanitizer::insertRuntimeIgnores(Function &F) { |
| 619 | InstrumentationIRBuilder IRB(&*F.getEntryBlock().getFirstNonPHIIt()); |
| 620 | IRB.CreateCall(Callee: CsanIgnoreBegin); |
| 621 | EscapeEnumerator EE(F, "csan_ignore_cleanup" , ClHandleCxxExceptions); |
| 622 | while (IRBuilder<> *AtExit = EE.Next()) { |
| 623 | InstrumentationIRBuilder::ensureDebugInfo(IRB&: *AtExit, F); |
| 624 | AtExit->CreateCall(Callee: CsanIgnoreEnd); |
| 625 | } |
| 626 | } |
| 627 | |