1//===-- SanitizerCoverage.cpp - coverage instrumentation for sanitizers ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Coverage instrumentation done on LLVM IR level, works with Sanitizers.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/Transforms/Instrumentation/SanitizerCoverage.h"
14#include "llvm/ADT/ArrayRef.h"
15#include "llvm/ADT/SmallVector.h"
16#include "llvm/Analysis/GlobalsModRef.h"
17#include "llvm/Analysis/PostDominators.h"
18#include "llvm/IR/Constant.h"
19#include "llvm/IR/Constants.h"
20#include "llvm/IR/DataLayout.h"
21#include "llvm/IR/Dominators.h"
22#include "llvm/IR/EHPersonalities.h"
23#include "llvm/IR/Function.h"
24#include "llvm/IR/GlobalVariable.h"
25#include "llvm/IR/IRBuilder.h"
26#include "llvm/IR/IntrinsicInst.h"
27#include "llvm/IR/Intrinsics.h"
28#include "llvm/IR/LLVMContext.h"
29#include "llvm/IR/MDBuilder.h"
30#include "llvm/IR/Module.h"
31#include "llvm/IR/Type.h"
32#include "llvm/IR/ValueSymbolTable.h"
33#include "llvm/Support/CommandLine.h"
34#include "llvm/Support/SpecialCaseList.h"
35#include "llvm/Support/VirtualFileSystem.h"
36#include "llvm/TargetParser/Triple.h"
37#include "llvm/Transforms/Utils/BasicBlockUtils.h"
38#include "llvm/Transforms/Utils/EscapeEnumerator.h"
39#include "llvm/Transforms/Utils/ModuleUtils.h"
40
41using namespace llvm;
42
43#define DEBUG_TYPE "sancov"
44
45const char SanCovTracePCIndirName[] = "__sanitizer_cov_trace_pc_indir";
46const char SanCovTracePCName[] = "__sanitizer_cov_trace_pc";
47const char SanCovTracePCEntryName[] = "__sanitizer_cov_trace_pc_entry";
48const char SanCovTracePCExitName[] = "__sanitizer_cov_trace_pc_exit";
49const char SanCovTraceCmp1[] = "__sanitizer_cov_trace_cmp1";
50const char SanCovTraceCmp2[] = "__sanitizer_cov_trace_cmp2";
51const char SanCovTraceCmp4[] = "__sanitizer_cov_trace_cmp4";
52const char SanCovTraceCmp8[] = "__sanitizer_cov_trace_cmp8";
53const char SanCovTraceConstCmp1[] = "__sanitizer_cov_trace_const_cmp1";
54const char SanCovTraceConstCmp2[] = "__sanitizer_cov_trace_const_cmp2";
55const char SanCovTraceConstCmp4[] = "__sanitizer_cov_trace_const_cmp4";
56const char SanCovTraceConstCmp8[] = "__sanitizer_cov_trace_const_cmp8";
57const char SanCovLoad1[] = "__sanitizer_cov_load1";
58const char SanCovLoad2[] = "__sanitizer_cov_load2";
59const char SanCovLoad4[] = "__sanitizer_cov_load4";
60const char SanCovLoad8[] = "__sanitizer_cov_load8";
61const char SanCovLoad16[] = "__sanitizer_cov_load16";
62const char SanCovStore1[] = "__sanitizer_cov_store1";
63const char SanCovStore2[] = "__sanitizer_cov_store2";
64const char SanCovStore4[] = "__sanitizer_cov_store4";
65const char SanCovStore8[] = "__sanitizer_cov_store8";
66const char SanCovStore16[] = "__sanitizer_cov_store16";
67const char SanCovTraceDiv4[] = "__sanitizer_cov_trace_div4";
68const char SanCovTraceDiv8[] = "__sanitizer_cov_trace_div8";
69const char SanCovTraceGep[] = "__sanitizer_cov_trace_gep";
70const char SanCovTraceSwitchName[] = "__sanitizer_cov_trace_switch";
71const char SanCovModuleCtorTracePcGuardName[] =
72 "sancov.module_ctor_trace_pc_guard";
73const char SanCovModuleCtor8bitCountersName[] =
74 "sancov.module_ctor_8bit_counters";
75const char SanCovModuleCtorBoolFlagName[] = "sancov.module_ctor_bool_flag";
76static const uint64_t SanCtorAndDtorPriority = 2;
77
78const char SanCovTracePCGuardName[] = "__sanitizer_cov_trace_pc_guard";
79const char SanCovTracePCGuardInitName[] = "__sanitizer_cov_trace_pc_guard_init";
80const char SanCov8bitCountersInitName[] = "__sanitizer_cov_8bit_counters_init";
81const char SanCovBoolFlagInitName[] = "__sanitizer_cov_bool_flag_init";
82const char SanCovPCsInitName[] = "__sanitizer_cov_pcs_init";
83const char SanCovCFsInitName[] = "__sanitizer_cov_cfs_init";
84
85const char SanCovGuardsSectionName[] = "sancov_guards";
86const char SanCovCountersSectionName[] = "sancov_cntrs";
87const char SanCovBoolFlagSectionName[] = "sancov_bools";
88const char SanCovPCsSectionName[] = "sancov_pcs";
89const char SanCovCFsSectionName[] = "sancov_cfs";
90const char SanCovCallbackGateSectionName[] = "sancov_gate";
91
92const char SanCovStackDepthCallbackName[] = "__sanitizer_cov_stack_depth";
93const char SanCovLowestStackName[] = "__sancov_lowest_stack";
94const char SanCovCallbackGateName[] = "__sancov_should_track";
95
96static cl::opt<int> ClCoverageLevel(
97 "sanitizer-coverage-level",
98 cl::desc("Sanitizer Coverage. 0: none, 1: entry block, 2: all blocks, "
99 "3: all blocks and critical edges"),
100 cl::Hidden);
101
102static cl::opt<bool> ClTracePC("sanitizer-coverage-trace-pc",
103 cl::desc("Experimental pc tracing"), cl::Hidden);
104
105static cl::opt<bool> ClTracePCEntryExit(
106 "sanitizer-coverage-trace-pc-entry-exit",
107 cl::desc("pc tracing with separate entry/exit callbacks"), cl::Hidden);
108
109static cl::opt<bool> ClTracePCGuard("sanitizer-coverage-trace-pc-guard",
110 cl::desc("pc tracing with a guard"),
111 cl::Hidden);
112
113// If true, we create a global variable that contains PCs of all instrumented
114// BBs, put this global into a named section, and pass this section's bounds
115// to __sanitizer_cov_pcs_init.
116// This way the coverage instrumentation does not need to acquire the PCs
117// at run-time. Works with trace-pc-guard, inline-8bit-counters, and
118// inline-bool-flag.
119static cl::opt<bool> ClCreatePCTable("sanitizer-coverage-pc-table",
120 cl::desc("create a static PC table"),
121 cl::Hidden);
122
123static cl::opt<bool>
124 ClInline8bitCounters("sanitizer-coverage-inline-8bit-counters",
125 cl::desc("increments 8-bit counter for every edge"),
126 cl::Hidden);
127
128static cl::opt<bool>
129 ClSancovDropCtors("sanitizer-coverage-drop-ctors",
130 cl::desc("do not emit module ctors for global counters"),
131 cl::Hidden);
132
133static cl::opt<bool>
134 ClInlineBoolFlag("sanitizer-coverage-inline-bool-flag",
135 cl::desc("sets a boolean flag for every edge"),
136 cl::Hidden);
137
138static cl::opt<bool>
139 ClCMPTracing("sanitizer-coverage-trace-compares",
140 cl::desc("Tracing of CMP and similar instructions"),
141 cl::Hidden);
142
143static cl::opt<bool> ClDIVTracing("sanitizer-coverage-trace-divs",
144 cl::desc("Tracing of DIV instructions"),
145 cl::Hidden);
146
147static cl::opt<bool> ClLoadTracing("sanitizer-coverage-trace-loads",
148 cl::desc("Tracing of load instructions"),
149 cl::Hidden);
150
151static cl::opt<bool> ClStoreTracing("sanitizer-coverage-trace-stores",
152 cl::desc("Tracing of store instructions"),
153 cl::Hidden);
154
155static cl::opt<bool> ClGEPTracing("sanitizer-coverage-trace-geps",
156 cl::desc("Tracing of GEP instructions"),
157 cl::Hidden);
158
159static cl::opt<bool>
160 ClPruneBlocks("sanitizer-coverage-prune-blocks",
161 cl::desc("Reduce the number of instrumented blocks"),
162 cl::Hidden, cl::init(Val: true));
163
164static cl::opt<bool> ClStackDepth("sanitizer-coverage-stack-depth",
165 cl::desc("max stack depth tracing"),
166 cl::Hidden);
167
168static cl::opt<int> ClStackDepthCallbackMin(
169 "sanitizer-coverage-stack-depth-callback-min",
170 cl::desc("max stack depth tracing should use callback and only when "
171 "stack depth more than specified"),
172 cl::Hidden);
173
174static cl::opt<bool>
175 ClCollectCF("sanitizer-coverage-control-flow",
176 cl::desc("collect control flow for each function"), cl::Hidden);
177
178static cl::opt<bool> ClGatedCallbacks(
179 "sanitizer-coverage-gated-trace-callbacks",
180 cl::desc("Gate the invocation of the tracing callbacks on a global variable"
181 ". Currently only supported for trace-pc-guard and trace-cmp."),
182 cl::Hidden, cl::init(Val: false));
183
184namespace {
185
186SanitizerCoverageOptions getOptions(int LegacyCoverageLevel) {
187 SanitizerCoverageOptions Res;
188 switch (LegacyCoverageLevel) {
189 case 0:
190 Res.CoverageType = SanitizerCoverageOptions::SCK_None;
191 break;
192 case 1:
193 Res.CoverageType = SanitizerCoverageOptions::SCK_Function;
194 break;
195 case 2:
196 Res.CoverageType = SanitizerCoverageOptions::SCK_BB;
197 break;
198 case 3:
199 Res.CoverageType = SanitizerCoverageOptions::SCK_Edge;
200 break;
201 case 4:
202 Res.CoverageType = SanitizerCoverageOptions::SCK_Edge;
203 Res.IndirectCalls = true;
204 break;
205 }
206 return Res;
207}
208
209SanitizerCoverageOptions OverrideFromCL(SanitizerCoverageOptions Options) {
210 // Sets CoverageType and IndirectCalls.
211 SanitizerCoverageOptions CLOpts = getOptions(LegacyCoverageLevel: ClCoverageLevel);
212 Options.CoverageType = std::max(a: Options.CoverageType, b: CLOpts.CoverageType);
213 Options.IndirectCalls |= CLOpts.IndirectCalls;
214 Options.TraceCmp |= ClCMPTracing;
215 Options.TraceDiv |= ClDIVTracing;
216 Options.TraceGep |= ClGEPTracing;
217 Options.TracePC |= ClTracePC;
218 Options.TracePCEntryExit |= ClTracePCEntryExit;
219 Options.TracePCGuard |= ClTracePCGuard;
220 Options.Inline8bitCounters |= ClInline8bitCounters;
221 Options.InlineBoolFlag |= ClInlineBoolFlag;
222 Options.PCTable |= ClCreatePCTable;
223 Options.NoPrune |= !ClPruneBlocks;
224 Options.StackDepth |= ClStackDepth;
225 Options.StackDepthCallbackMin = std::max(a: Options.StackDepthCallbackMin,
226 b: ClStackDepthCallbackMin.getValue());
227 Options.TraceLoads |= ClLoadTracing;
228 Options.TraceStores |= ClStoreTracing;
229 Options.GatedCallbacks |= ClGatedCallbacks;
230 if (!Options.TracePCGuard && !Options.TracePC && !Options.TracePCEntryExit &&
231 !Options.Inline8bitCounters && !Options.StackDepth &&
232 !Options.InlineBoolFlag && !Options.TraceLoads && !Options.TraceStores)
233 Options.TracePCGuard = true; // TracePCGuard is default.
234 Options.CollectControlFlow |= ClCollectCF;
235 return Options;
236}
237
238class ModuleSanitizerCoverage {
239public:
240 using DomTreeCallback = function_ref<const DominatorTree &(Function &F)>;
241 using PostDomTreeCallback =
242 function_ref<const PostDominatorTree &(Function &F)>;
243
244 ModuleSanitizerCoverage(Module &M, DomTreeCallback DTCallback,
245 PostDomTreeCallback PDTCallback,
246 const SanitizerCoverageOptions &Options,
247 const SpecialCaseList *Allowlist,
248 const SpecialCaseList *Blocklist)
249 : M(M), DTCallback(DTCallback), PDTCallback(PDTCallback),
250 Options(Options), Allowlist(Allowlist), Blocklist(Blocklist) {}
251
252 bool instrumentModule();
253
254private:
255 void createFunctionControlFlow(Function &F);
256 void instrumentFunction(Function &F);
257 void InjectCoverageForIndirectCalls(Function &F,
258 ArrayRef<Instruction *> IndirCalls);
259 void InjectTraceForCmp(Function &F, ArrayRef<Instruction *> CmpTraceTargets,
260 Value *&FunctionGateCmp);
261 void InjectTraceForDiv(Function &F,
262 ArrayRef<BinaryOperator *> DivTraceTargets);
263 void InjectTraceForGep(Function &F,
264 ArrayRef<GetElementPtrInst *> GepTraceTargets);
265 void InjectTraceForLoadsAndStores(Function &F, ArrayRef<LoadInst *> Loads,
266 ArrayRef<StoreInst *> Stores);
267 void InjectTraceForExits(Function &F);
268 void InjectTraceForSwitch(Function &F,
269 ArrayRef<Instruction *> SwitchTraceTargets,
270 Value *&FunctionGateCmp);
271 bool InjectCoverage(Function &F, ArrayRef<BasicBlock *> AllBlocks,
272 Value *&FunctionGateCmp, bool IsLeafFunc);
273 GlobalVariable *CreateFunctionLocalArrayInSection(size_t NumElements,
274 Function &F, Type *Ty,
275 const char *Section);
276 GlobalVariable *CreatePCArray(Function &F, ArrayRef<BasicBlock *> AllBlocks);
277 void CreateFunctionLocalArrays(Function &F, ArrayRef<BasicBlock *> AllBlocks);
278 Instruction *CreateGateBranch(Function &F, Value *&FunctionGateCmp,
279 Instruction *I);
280 Value *CreateFunctionLocalGateCmp(IRBuilder<> &IRB);
281 void InjectCoverageAtBlock(Function &F, BasicBlock &BB, size_t Idx,
282 Value *&FunctionGateCmp, bool IsLeafFunc);
283 Function *CreateInitCallsForSections(Module &M, const char *CtorName,
284 const char *InitFunctionName, Type *Ty,
285 const char *Section);
286 std::pair<Value *, Value *> CreateSecStartEnd(Module &M, const char *Section,
287 Type *Ty);
288
289 std::string getSectionName(const std::string &Section) const;
290 std::string getSectionStart(const std::string &Section) const;
291 std::string getSectionEnd(const std::string &Section) const;
292
293 Module &M;
294 DomTreeCallback DTCallback;
295 PostDomTreeCallback PDTCallback;
296
297 FunctionCallee SanCovStackDepthCallback;
298 FunctionCallee SanCovTracePCIndir;
299 FunctionCallee SanCovTracePC, SanCovTracePCGuard;
300 FunctionCallee SanCovTracePCEntry, SanCovTracePCExit;
301 std::array<FunctionCallee, 4> SanCovTraceCmpFunction;
302 std::array<FunctionCallee, 4> SanCovTraceConstCmpFunction;
303 std::array<FunctionCallee, 5> SanCovLoadFunction;
304 std::array<FunctionCallee, 5> SanCovStoreFunction;
305 std::array<FunctionCallee, 2> SanCovTraceDivFunction;
306 FunctionCallee SanCovTraceGepFunction;
307 FunctionCallee SanCovTraceSwitchFunction;
308 GlobalVariable *SanCovLowestStack;
309 GlobalVariable *SanCovCallbackGate;
310 Type *PtrTy, *IntptrTy, *Int64Ty, *Int32Ty, *Int16Ty, *Int8Ty, *Int1Ty;
311 Module *CurModule;
312 Triple TargetTriple;
313 LLVMContext *C;
314 const DataLayout *DL;
315
316 GlobalVariable *FunctionGuardArray; // for trace-pc-guard.
317 GlobalVariable *Function8bitCounterArray; // for inline-8bit-counters.
318 GlobalVariable *FunctionBoolArray; // for inline-bool-flag.
319 GlobalVariable *FunctionPCsArray; // for pc-table.
320 GlobalVariable *FunctionCFsArray; // for control flow table
321 SmallVector<GlobalValue *, 20> GlobalsToAppendToUsed;
322 SmallVector<GlobalValue *, 20> GlobalsToAppendToCompilerUsed;
323
324 SanitizerCoverageOptions Options;
325
326 const SpecialCaseList *Allowlist;
327 const SpecialCaseList *Blocklist;
328};
329} // namespace
330
331SanitizerCoveragePass::SanitizerCoveragePass(
332 SanitizerCoverageOptions Options, IntrusiveRefCntPtr<vfs::FileSystem> VFS,
333 const std::vector<std::string> &AllowlistFiles,
334 const std::vector<std::string> &BlocklistFiles)
335 : Options(std::move(Options)),
336 VFS(VFS ? std::move(VFS) : vfs::getRealFileSystem()) {
337 if (AllowlistFiles.size() > 0)
338 Allowlist = SpecialCaseList::createOrDie(Paths: AllowlistFiles, FS&: *this->VFS);
339 if (BlocklistFiles.size() > 0)
340 Blocklist = SpecialCaseList::createOrDie(Paths: BlocklistFiles, FS&: *this->VFS);
341}
342
343PreservedAnalyses SanitizerCoveragePass::run(Module &M,
344 ModuleAnalysisManager &MAM) {
345 auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
346 auto DTCallback = [&FAM](Function &F) -> const DominatorTree & {
347 return FAM.getResult<DominatorTreeAnalysis>(IR&: F);
348 };
349 auto PDTCallback = [&FAM](Function &F) -> const PostDominatorTree & {
350 return FAM.getResult<PostDominatorTreeAnalysis>(IR&: F);
351 };
352 ModuleSanitizerCoverage ModuleSancov(M, DTCallback, PDTCallback,
353 OverrideFromCL(Options), Allowlist.get(),
354 Blocklist.get());
355 if (!ModuleSancov.instrumentModule())
356 return PreservedAnalyses::all();
357
358 PreservedAnalyses PA = PreservedAnalyses::none();
359 // GlobalsAA is considered stateless and does not get invalidated unless
360 // explicitly invalidated; PreservedAnalyses::none() is not enough. Sanitizers
361 // make changes that require GlobalsAA to be invalidated.
362 PA.abandon<GlobalsAA>();
363 return PA;
364}
365
366std::pair<Value *, Value *>
367ModuleSanitizerCoverage::CreateSecStartEnd(Module &M, const char *Section,
368 Type *Ty) {
369 // Use ExternalWeak so that if all sections are discarded due to section
370 // garbage collection, the linker will not report undefined symbol errors.
371 // Windows defines the start/stop symbols in compiler-rt so no need for
372 // ExternalWeak.
373 GlobalValue::LinkageTypes Linkage = TargetTriple.isOSBinFormatCOFF()
374 ? GlobalVariable::ExternalLinkage
375 : GlobalVariable::ExternalWeakLinkage;
376 GlobalVariable *SecStart = new GlobalVariable(M, Ty, false, Linkage, nullptr,
377 getSectionStart(Section));
378 SecStart->setVisibility(GlobalValue::HiddenVisibility);
379 GlobalVariable *SecEnd = new GlobalVariable(M, Ty, false, Linkage, nullptr,
380 getSectionEnd(Section));
381 SecEnd->setVisibility(GlobalValue::HiddenVisibility);
382 if (!TargetTriple.isOSBinFormatCOFF())
383 return std::make_pair(x&: SecStart, y&: SecEnd);
384
385 // Account for the fact that on windows-msvc __start_* symbols actually
386 // point to a uint64_t before the start of the array.
387 auto *GEP = ConstantExpr::getPtrAdd(
388 Ptr: SecStart, Offset: ConstantInt::get(Ty: IntptrTy, V: sizeof(uint64_t)));
389 return std::make_pair(x&: GEP, y&: SecEnd);
390}
391
392Function *ModuleSanitizerCoverage::CreateInitCallsForSections(
393 Module &M, const char *CtorName, const char *InitFunctionName, Type *Ty,
394 const char *Section) {
395 if (ClSancovDropCtors)
396 return nullptr;
397 auto SecStartEnd = CreateSecStartEnd(M, Section, Ty);
398 auto SecStart = SecStartEnd.first;
399 auto SecEnd = SecStartEnd.second;
400 Function *CtorFunc;
401 std::tie(args&: CtorFunc, args: std::ignore) = createSanitizerCtorAndInitFunctions(
402 M, CtorName, InitName: InitFunctionName, InitArgTypes: {PtrTy, PtrTy}, InitArgs: {SecStart, SecEnd});
403 assert(CtorFunc->getName() == CtorName);
404
405 if (TargetTriple.supportsCOMDAT()) {
406 // Use comdat to dedup CtorFunc.
407 CtorFunc->setComdat(M.getOrInsertComdat(Name: CtorName));
408 appendToGlobalCtors(M, F: CtorFunc, Priority: SanCtorAndDtorPriority, Data: CtorFunc);
409 } else {
410 appendToGlobalCtors(M, F: CtorFunc, Priority: SanCtorAndDtorPriority);
411 }
412
413 if (TargetTriple.isOSBinFormatCOFF()) {
414 // In COFF files, if the contructors are set as COMDAT (they are because
415 // COFF supports COMDAT) and the linker flag /OPT:REF (strip unreferenced
416 // functions and data) is used, the constructors get stripped. To prevent
417 // this, give the constructors weak ODR linkage and ensure the linker knows
418 // to include the sancov constructor. This way the linker can deduplicate
419 // the constructors but always leave one copy.
420 CtorFunc->setLinkage(GlobalValue::WeakODRLinkage);
421 }
422 return CtorFunc;
423}
424
425bool ModuleSanitizerCoverage::instrumentModule() {
426 if (Options.CoverageType == SanitizerCoverageOptions::SCK_None)
427 return false;
428 if (Allowlist &&
429 !Allowlist->inSection(Section: "coverage", Prefix: "src", Query: M.getSourceFileName()))
430 return false;
431 if (Blocklist &&
432 Blocklist->inSection(Section: "coverage", Prefix: "src", Query: M.getSourceFileName()))
433 return false;
434 C = &(M.getContext());
435 DL = &M.getDataLayout();
436 CurModule = &M;
437 TargetTriple = M.getTargetTriple();
438 FunctionGuardArray = nullptr;
439 Function8bitCounterArray = nullptr;
440 FunctionBoolArray = nullptr;
441 FunctionPCsArray = nullptr;
442 FunctionCFsArray = nullptr;
443 IntptrTy = Type::getIntNTy(C&: *C, N: DL->getPointerSizeInBits());
444 PtrTy = PointerType::getUnqual(C&: *C);
445 Type *VoidTy = Type::getVoidTy(C&: *C);
446 IRBuilder<> IRB(M);
447 Int64Ty = IRB.getInt64Ty();
448 Int32Ty = IRB.getInt32Ty();
449 Int16Ty = IRB.getInt16Ty();
450 Int8Ty = IRB.getInt8Ty();
451 Int1Ty = IRB.getInt1Ty();
452
453 SanCovTracePCIndir =
454 M.getOrInsertFunction(Name: SanCovTracePCIndirName, RetTy: VoidTy, Args: IntptrTy);
455 // Make sure smaller parameters are zero-extended to i64 if required by the
456 // target ABI.
457 AttributeList SanCovTraceCmpZeroExtAL;
458 SanCovTraceCmpZeroExtAL =
459 SanCovTraceCmpZeroExtAL.addParamAttribute(C&: *C, ArgNo: 0, Kind: Attribute::ZExt);
460 SanCovTraceCmpZeroExtAL =
461 SanCovTraceCmpZeroExtAL.addParamAttribute(C&: *C, ArgNo: 1, Kind: Attribute::ZExt);
462
463 SanCovTraceCmpFunction[0] =
464 M.getOrInsertFunction(Name: SanCovTraceCmp1, AttributeList: SanCovTraceCmpZeroExtAL, RetTy: VoidTy,
465 Args: IRB.getInt8Ty(), Args: IRB.getInt8Ty());
466 SanCovTraceCmpFunction[1] =
467 M.getOrInsertFunction(Name: SanCovTraceCmp2, AttributeList: SanCovTraceCmpZeroExtAL, RetTy: VoidTy,
468 Args: IRB.getInt16Ty(), Args: IRB.getInt16Ty());
469 SanCovTraceCmpFunction[2] =
470 M.getOrInsertFunction(Name: SanCovTraceCmp4, AttributeList: SanCovTraceCmpZeroExtAL, RetTy: VoidTy,
471 Args: IRB.getInt32Ty(), Args: IRB.getInt32Ty());
472 SanCovTraceCmpFunction[3] =
473 M.getOrInsertFunction(Name: SanCovTraceCmp8, RetTy: VoidTy, Args: Int64Ty, Args: Int64Ty);
474
475 SanCovTraceConstCmpFunction[0] = M.getOrInsertFunction(
476 Name: SanCovTraceConstCmp1, AttributeList: SanCovTraceCmpZeroExtAL, RetTy: VoidTy, Args: Int8Ty, Args: Int8Ty);
477 SanCovTraceConstCmpFunction[1] = M.getOrInsertFunction(
478 Name: SanCovTraceConstCmp2, AttributeList: SanCovTraceCmpZeroExtAL, RetTy: VoidTy, Args: Int16Ty, Args: Int16Ty);
479 SanCovTraceConstCmpFunction[2] = M.getOrInsertFunction(
480 Name: SanCovTraceConstCmp4, AttributeList: SanCovTraceCmpZeroExtAL, RetTy: VoidTy, Args: Int32Ty, Args: Int32Ty);
481 SanCovTraceConstCmpFunction[3] =
482 M.getOrInsertFunction(Name: SanCovTraceConstCmp8, RetTy: VoidTy, Args: Int64Ty, Args: Int64Ty);
483
484 // Loads.
485 SanCovLoadFunction[0] = M.getOrInsertFunction(Name: SanCovLoad1, RetTy: VoidTy, Args: PtrTy);
486 SanCovLoadFunction[1] = M.getOrInsertFunction(Name: SanCovLoad2, RetTy: VoidTy, Args: PtrTy);
487 SanCovLoadFunction[2] = M.getOrInsertFunction(Name: SanCovLoad4, RetTy: VoidTy, Args: PtrTy);
488 SanCovLoadFunction[3] = M.getOrInsertFunction(Name: SanCovLoad8, RetTy: VoidTy, Args: PtrTy);
489 SanCovLoadFunction[4] = M.getOrInsertFunction(Name: SanCovLoad16, RetTy: VoidTy, Args: PtrTy);
490 // Stores.
491 SanCovStoreFunction[0] = M.getOrInsertFunction(Name: SanCovStore1, RetTy: VoidTy, Args: PtrTy);
492 SanCovStoreFunction[1] = M.getOrInsertFunction(Name: SanCovStore2, RetTy: VoidTy, Args: PtrTy);
493 SanCovStoreFunction[2] = M.getOrInsertFunction(Name: SanCovStore4, RetTy: VoidTy, Args: PtrTy);
494 SanCovStoreFunction[3] = M.getOrInsertFunction(Name: SanCovStore8, RetTy: VoidTy, Args: PtrTy);
495 SanCovStoreFunction[4] = M.getOrInsertFunction(Name: SanCovStore16, RetTy: VoidTy, Args: PtrTy);
496
497 {
498 AttributeList AL;
499 AL = AL.addParamAttribute(C&: *C, ArgNo: 0, Kind: Attribute::ZExt);
500 SanCovTraceDivFunction[0] =
501 M.getOrInsertFunction(Name: SanCovTraceDiv4, AttributeList: AL, RetTy: VoidTy, Args: IRB.getInt32Ty());
502 }
503 SanCovTraceDivFunction[1] =
504 M.getOrInsertFunction(Name: SanCovTraceDiv8, RetTy: VoidTy, Args: Int64Ty);
505 SanCovTraceGepFunction =
506 M.getOrInsertFunction(Name: SanCovTraceGep, RetTy: VoidTy, Args: IntptrTy);
507 SanCovTraceSwitchFunction =
508 M.getOrInsertFunction(Name: SanCovTraceSwitchName, RetTy: VoidTy, Args: Int64Ty, Args: PtrTy);
509
510 SanCovLowestStack = M.getOrInsertGlobal(Name: SanCovLowestStackName, Ty: IntptrTy);
511 if (SanCovLowestStack->getValueType() != IntptrTy) {
512 C->emitError(ErrorStr: StringRef("'") + SanCovLowestStackName +
513 "' should not be declared by the user");
514 return true;
515 }
516 SanCovLowestStack->setThreadLocalMode(
517 GlobalValue::ThreadLocalMode::InitialExecTLSModel);
518 if (Options.StackDepth && !SanCovLowestStack->isDeclaration())
519 SanCovLowestStack->setInitializer(Constant::getAllOnesValue(Ty: IntptrTy));
520
521 if (Options.GatedCallbacks) {
522 if (!Options.TracePCGuard && !Options.TraceCmp) {
523 C->emitError(ErrorStr: StringRef("'") + ClGatedCallbacks.ArgStr +
524 "' is only supported with trace-pc-guard or trace-cmp");
525 return true;
526 }
527
528 SanCovCallbackGate = cast<GlobalVariable>(
529 Val: M.getOrInsertGlobal(Name: SanCovCallbackGateName, Ty: Int64Ty));
530 SanCovCallbackGate->setSection(
531 getSectionName(Section: SanCovCallbackGateSectionName));
532 SanCovCallbackGate->setInitializer(Constant::getNullValue(Ty: Int64Ty));
533 SanCovCallbackGate->setLinkage(GlobalVariable::LinkOnceAnyLinkage);
534 SanCovCallbackGate->setVisibility(GlobalVariable::HiddenVisibility);
535 appendToCompilerUsed(M, Values: SanCovCallbackGate);
536 }
537
538 SanCovTracePC = M.getOrInsertFunction(Name: SanCovTracePCName, RetTy: VoidTy);
539 SanCovTracePCEntry = M.getOrInsertFunction(Name: SanCovTracePCEntryName, RetTy: VoidTy);
540 SanCovTracePCExit = M.getOrInsertFunction(Name: SanCovTracePCExitName, RetTy: VoidTy);
541 SanCovTracePCGuard =
542 M.getOrInsertFunction(Name: SanCovTracePCGuardName, RetTy: VoidTy, Args: PtrTy);
543
544 SanCovStackDepthCallback =
545 M.getOrInsertFunction(Name: SanCovStackDepthCallbackName, RetTy: VoidTy);
546
547 for (auto &F : M)
548 instrumentFunction(F);
549
550 Function *Ctor = nullptr;
551
552 if (FunctionGuardArray)
553 Ctor = CreateInitCallsForSections(M, CtorName: SanCovModuleCtorTracePcGuardName,
554 InitFunctionName: SanCovTracePCGuardInitName, Ty: Int32Ty,
555 Section: SanCovGuardsSectionName);
556 if (Function8bitCounterArray)
557 Ctor = CreateInitCallsForSections(M, CtorName: SanCovModuleCtor8bitCountersName,
558 InitFunctionName: SanCov8bitCountersInitName, Ty: Int8Ty,
559 Section: SanCovCountersSectionName);
560 if (FunctionBoolArray) {
561 Ctor = CreateInitCallsForSections(M, CtorName: SanCovModuleCtorBoolFlagName,
562 InitFunctionName: SanCovBoolFlagInitName, Ty: Int1Ty,
563 Section: SanCovBoolFlagSectionName);
564 }
565 if (Ctor && Options.PCTable) {
566 auto SecStartEnd = CreateSecStartEnd(M, Section: SanCovPCsSectionName, Ty: IntptrTy);
567 FunctionCallee InitFunction =
568 declareSanitizerInitFunction(M, InitName: SanCovPCsInitName, InitArgTypes: {PtrTy, PtrTy});
569 IRBuilder<> IRBCtor(Ctor->getEntryBlock().getTerminator());
570 IRBCtor.CreateCall(Callee: InitFunction, Args: {SecStartEnd.first, SecStartEnd.second});
571 }
572
573 if (Ctor && Options.CollectControlFlow) {
574 auto SecStartEnd = CreateSecStartEnd(M, Section: SanCovCFsSectionName, Ty: IntptrTy);
575 FunctionCallee InitFunction =
576 declareSanitizerInitFunction(M, InitName: SanCovCFsInitName, InitArgTypes: {PtrTy, PtrTy});
577 IRBuilder<> IRBCtor(Ctor->getEntryBlock().getTerminator());
578 IRBCtor.CreateCall(Callee: InitFunction, Args: {SecStartEnd.first, SecStartEnd.second});
579 }
580
581 appendToUsed(M, Values: GlobalsToAppendToUsed);
582 appendToCompilerUsed(M, Values: GlobalsToAppendToCompilerUsed);
583 return true;
584}
585
586// True if block has successors and it dominates all of them.
587static bool isFullDominator(const BasicBlock *BB, const DominatorTree &DT) {
588 if (succ_empty(BB))
589 return false;
590
591 return llvm::all_of(Range: successors(BB), P: [&](const BasicBlock *SUCC) {
592 return DT.dominates(A: BB, B: SUCC);
593 });
594}
595
596// True if block has predecessors and it postdominates all of them.
597static bool isFullPostDominator(const BasicBlock *BB,
598 const PostDominatorTree &PDT) {
599 if (pred_empty(BB))
600 return false;
601
602 return llvm::all_of(Range: predecessors(BB), P: [&](const BasicBlock *PRED) {
603 return PDT.dominates(A: BB, B: PRED);
604 });
605}
606
607static bool shouldInstrumentBlock(const Function &F, const BasicBlock *BB,
608 const DominatorTree &DT,
609 const PostDominatorTree &PDT,
610 const SanitizerCoverageOptions &Options) {
611 // Don't insert coverage for blocks containing nothing but unreachable: we
612 // will never call __sanitizer_cov() for them, so counting them in
613 // NumberOfInstrumentedBlocks() might complicate calculation of code coverage
614 // percentage. Also, unreachable instructions frequently have no debug
615 // locations.
616 if (isa<UnreachableInst>(Val: BB->getFirstNonPHIOrDbgOrLifetime()))
617 return false;
618
619 // Don't insert coverage into blocks without a valid insertion point
620 // (catchswitch blocks).
621 if (BB->getFirstInsertionPt() == BB->end())
622 return false;
623
624 if (Options.NoPrune || &F.getEntryBlock() == BB)
625 return true;
626
627 if (Options.CoverageType == SanitizerCoverageOptions::SCK_Function &&
628 &F.getEntryBlock() != BB)
629 return false;
630
631 // Do not instrument full dominators, or full post-dominators with multiple
632 // predecessors.
633 return !isFullDominator(BB, DT) &&
634 !(isFullPostDominator(BB, PDT) && !BB->getSinglePredecessor());
635}
636
637// Returns true iff From->To is a backedge.
638// A twist here is that we treat From->To as a backedge if
639// * To dominates From or
640// * To->UniqueSuccessor dominates From
641static bool IsBackEdge(BasicBlock *From, BasicBlock *To,
642 const DominatorTree &DT) {
643 if (DT.dominates(A: To, B: From))
644 return true;
645 if (auto Next = To->getUniqueSuccessor())
646 if (DT.dominates(A: Next, B: From))
647 return true;
648 return false;
649}
650
651// Prunes uninteresting Cmp instrumentation:
652// * CMP instructions that feed into loop backedge branch.
653//
654// Note that Cmp pruning is controlled by the same flag as the
655// BB pruning.
656static bool IsInterestingCmp(ICmpInst *CMP, const DominatorTree &DT,
657 const SanitizerCoverageOptions &Options) {
658 if (!Options.NoPrune)
659 if (CMP->hasOneUse())
660 if (auto BR = dyn_cast<CondBrInst>(Val: CMP->user_back()))
661 for (BasicBlock *B : BR->successors())
662 if (IsBackEdge(From: BR->getParent(), To: B, DT))
663 return false;
664 return true;
665}
666
667void ModuleSanitizerCoverage::instrumentFunction(Function &F) {
668 if (F.empty())
669 return;
670 if (F.getName().contains(Other: ".module_ctor"))
671 return; // Should not instrument sanitizer init functions.
672 if (F.getName().starts_with(Prefix: "__sanitizer_"))
673 return; // Don't instrument __sanitizer_* callbacks.
674 // Don't touch available_externally functions, their actual body is elewhere.
675 if (F.getLinkage() == GlobalValue::AvailableExternallyLinkage)
676 return;
677 // Don't instrument MSVC CRT configuration helpers. They may run before normal
678 // initialization.
679 if (F.getName() == "__local_stdio_printf_options" ||
680 F.getName() == "__local_stdio_scanf_options")
681 return;
682 if (isa<UnreachableInst>(Val: F.getEntryBlock().getTerminator()))
683 return;
684 // Don't instrument functions using SEH for now. Splitting basic blocks like
685 // we do for coverage breaks WinEHPrepare.
686 // FIXME: Remove this when SEH no longer uses landingpad pattern matching.
687 if (F.hasPersonalityFn() &&
688 isAsynchronousEHPersonality(Pers: classifyEHPersonality(Pers: F.getPersonalityFn())))
689 return;
690 if (Allowlist && !Allowlist->inSection(Section: "coverage", Prefix: "fun", Query: F.getName()))
691 return;
692 if (Blocklist && Blocklist->inSection(Section: "coverage", Prefix: "fun", Query: F.getName()))
693 return;
694 // Do not apply any instrumentation for naked functions.
695 if (F.hasFnAttribute(Kind: Attribute::Naked))
696 return;
697 if (F.hasFnAttribute(Kind: Attribute::NoSanitizeCoverage))
698 return;
699 if (F.hasFnAttribute(Kind: Attribute::DisableSanitizerInstrumentation))
700 return;
701 if (Options.CoverageType >= SanitizerCoverageOptions::SCK_Edge) {
702 SplitAllCriticalEdges(
703 F, Options: CriticalEdgeSplittingOptions().setIgnoreUnreachableDests());
704 }
705 SmallVector<Instruction *, 8> IndirCalls;
706 SmallVector<BasicBlock *, 16> BlocksToInstrument;
707 SmallVector<Instruction *, 8> CmpTraceTargets;
708 SmallVector<Instruction *, 8> SwitchTraceTargets;
709 SmallVector<BinaryOperator *, 8> DivTraceTargets;
710 SmallVector<GetElementPtrInst *, 8> GepTraceTargets;
711 SmallVector<LoadInst *, 8> Loads;
712 SmallVector<StoreInst *, 8> Stores;
713
714 const DominatorTree &DT = DTCallback(F);
715 const PostDominatorTree &PDT = PDTCallback(F);
716 bool IsLeafFunc = true;
717
718 for (auto &BB : F) {
719 if (shouldInstrumentBlock(F, BB: &BB, DT, PDT, Options))
720 BlocksToInstrument.push_back(Elt: &BB);
721 for (auto &Inst : BB) {
722 if (Options.IndirectCalls) {
723 CallBase *CB = dyn_cast<CallBase>(Val: &Inst);
724 if (CB && CB->isIndirectCall())
725 IndirCalls.push_back(Elt: &Inst);
726 }
727 if (Options.TraceCmp) {
728 if (ICmpInst *CMP = dyn_cast<ICmpInst>(Val: &Inst))
729 if (IsInterestingCmp(CMP, DT, Options))
730 CmpTraceTargets.push_back(Elt: &Inst);
731 if (isa<SwitchInst>(Val: &Inst))
732 SwitchTraceTargets.push_back(Elt: &Inst);
733 }
734 if (Options.TraceDiv)
735 if (BinaryOperator *BO = dyn_cast<BinaryOperator>(Val: &Inst))
736 if (BO->getOpcode() == Instruction::SDiv ||
737 BO->getOpcode() == Instruction::UDiv)
738 DivTraceTargets.push_back(Elt: BO);
739 if (Options.TraceGep)
740 if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Val: &Inst))
741 GepTraceTargets.push_back(Elt: GEP);
742 if (Options.TraceLoads)
743 if (LoadInst *LI = dyn_cast<LoadInst>(Val: &Inst))
744 Loads.push_back(Elt: LI);
745 if (Options.TraceStores)
746 if (StoreInst *SI = dyn_cast<StoreInst>(Val: &Inst))
747 Stores.push_back(Elt: SI);
748 if (Options.StackDepth)
749 if (isa<InvokeInst>(Val: Inst) ||
750 (isa<CallInst>(Val: Inst) && !isa<IntrinsicInst>(Val: Inst)))
751 IsLeafFunc = false;
752 }
753 }
754
755 if (Options.CollectControlFlow)
756 createFunctionControlFlow(F);
757
758 Value *FunctionGateCmp = nullptr;
759 InjectCoverage(F, AllBlocks: BlocksToInstrument, FunctionGateCmp, IsLeafFunc);
760 InjectCoverageForIndirectCalls(F, IndirCalls);
761 InjectTraceForCmp(F, CmpTraceTargets, FunctionGateCmp);
762 InjectTraceForSwitch(F, SwitchTraceTargets, FunctionGateCmp);
763 InjectTraceForDiv(F, DivTraceTargets);
764 InjectTraceForGep(F, GepTraceTargets);
765 InjectTraceForLoadsAndStores(F, Loads, Stores);
766
767 if (Options.TracePCEntryExit)
768 InjectTraceForExits(F);
769}
770
771GlobalVariable *ModuleSanitizerCoverage::CreateFunctionLocalArrayInSection(
772 size_t NumElements, Function &F, Type *Ty, const char *Section) {
773 ArrayType *ArrayTy = ArrayType::get(ElementType: Ty, NumElements);
774 auto Array = new GlobalVariable(
775 *CurModule, ArrayTy, false, GlobalVariable::PrivateLinkage,
776 Constant::getNullValue(Ty: ArrayTy), "__sancov_gen_");
777
778 // sancov_pcs parallels the other arrays, so they must be retained or
779 // discarded together. Put them in F's comdat to tie them to F for linker GC
780 // benefit. Outside ELF (nodeduplicate), a new comdat for an interposable F
781 // could prevail over a strong definition (COFF: the weak external becomes a
782 // COMDAT definition; Wasm: comdats deduplicate first). noipa doesn't affect
783 // symbol resolution.
784 if (TargetTriple.supportsCOMDAT() &&
785 (F.hasComdat() || TargetTriple.isOSBinFormatELF() ||
786 !F.isInterposable(/*CheckNoIPA=*/false)))
787 if (auto Comdat = getOrCreateFunctionComdat(F, T&: TargetTriple))
788 Array->setComdat(Comdat);
789 Array->setSection(getSectionName(Section));
790 Array->setAlignment(Align(DL->getTypeStoreSize(Ty).getFixedValue()));
791
792 // Optimizers (e.g. GlobalOpt/ConstantMerge) may not discard the arrays as a
793 // unit, so retain them in the compiler; without a comdat, in the linker too.
794 if (Array->hasComdat())
795 GlobalsToAppendToCompilerUsed.push_back(Elt: Array);
796 else
797 GlobalsToAppendToUsed.push_back(Elt: Array);
798
799 return Array;
800}
801
802GlobalVariable *
803ModuleSanitizerCoverage::CreatePCArray(Function &F,
804 ArrayRef<BasicBlock *> AllBlocks) {
805 size_t N = AllBlocks.size();
806 assert(N);
807 SmallVector<Constant *, 32> PCs;
808 IRBuilder<> IRB(&*F.getEntryBlock().getFirstInsertionPt());
809 for (size_t i = 0; i < N; i++) {
810 if (&F.getEntryBlock() == AllBlocks[i]) {
811 PCs.push_back(Elt: (Constant *)IRB.CreatePointerCast(V: &F, DestTy: PtrTy));
812 PCs.push_back(
813 Elt: (Constant *)IRB.CreateIntToPtr(V: ConstantInt::get(Ty: IntptrTy, V: 1), DestTy: PtrTy));
814 } else {
815 PCs.push_back(Elt: (Constant *)IRB.CreatePointerCast(
816 V: BlockAddress::get(BB: AllBlocks[i]), DestTy: PtrTy));
817 PCs.push_back(Elt: Constant::getNullValue(Ty: PtrTy));
818 }
819 }
820 auto *PCArray =
821 CreateFunctionLocalArrayInSection(NumElements: N * 2, F, Ty: PtrTy, Section: SanCovPCsSectionName);
822 PCArray->setInitializer(
823 ConstantArray::get(T: ArrayType::get(ElementType: PtrTy, NumElements: N * 2), V: PCs));
824 PCArray->setConstant(true);
825
826 return PCArray;
827}
828
829void ModuleSanitizerCoverage::CreateFunctionLocalArrays(
830 Function &F, ArrayRef<BasicBlock *> AllBlocks) {
831 if (Options.TracePCGuard)
832 FunctionGuardArray = CreateFunctionLocalArrayInSection(
833 NumElements: AllBlocks.size(), F, Ty: Int32Ty, Section: SanCovGuardsSectionName);
834
835 if (Options.Inline8bitCounters)
836 Function8bitCounterArray = CreateFunctionLocalArrayInSection(
837 NumElements: AllBlocks.size(), F, Ty: Int8Ty, Section: SanCovCountersSectionName);
838 if (Options.InlineBoolFlag)
839 FunctionBoolArray = CreateFunctionLocalArrayInSection(
840 NumElements: AllBlocks.size(), F, Ty: Int1Ty, Section: SanCovBoolFlagSectionName);
841
842 if (Options.PCTable)
843 FunctionPCsArray = CreatePCArray(F, AllBlocks);
844}
845
846Value *ModuleSanitizerCoverage::CreateFunctionLocalGateCmp(IRBuilder<> &IRB) {
847 auto Load = IRB.CreateLoad(Ty: Int64Ty, Ptr: SanCovCallbackGate);
848 Load->setNoSanitizeMetadata();
849 auto Cmp = IRB.CreateIsNotNull(Arg: Load);
850 Cmp->setName("sancov gate cmp");
851 return Cmp;
852}
853
854Instruction *ModuleSanitizerCoverage::CreateGateBranch(Function &F,
855 Value *&FunctionGateCmp,
856 Instruction *IP) {
857 if (!FunctionGateCmp) {
858 // Create this in the entry block
859 BasicBlock &BB = F.getEntryBlock();
860 BasicBlock::iterator IP = BB.getFirstInsertionPt();
861 IP = PrepareToSplitEntryBlock(BB, IP);
862 IRBuilder<> EntryIRB(&*IP);
863 FunctionGateCmp = CreateFunctionLocalGateCmp(IRB&: EntryIRB);
864 }
865 // Set the branch weights in order to minimize the price paid when the
866 // gate is turned off, allowing the default enablement of this
867 // instrumentation with as little of a performance cost as possible
868 auto Weights = MDBuilder(*C).createBranchWeights(TrueWeight: 1, FalseWeight: 100000);
869 return SplitBlockAndInsertIfThen(Cond: FunctionGateCmp, SplitBefore: IP, Unreachable: false, BranchWeights: Weights);
870}
871
872bool ModuleSanitizerCoverage::InjectCoverage(Function &F,
873 ArrayRef<BasicBlock *> AllBlocks,
874 Value *&FunctionGateCmp,
875 bool IsLeafFunc) {
876 if (AllBlocks.empty())
877 return false;
878 CreateFunctionLocalArrays(F, AllBlocks);
879 for (size_t i = 0, N = AllBlocks.size(); i < N; i++)
880 InjectCoverageAtBlock(F, BB&: *AllBlocks[i], Idx: i, FunctionGateCmp, IsLeafFunc);
881
882 return true;
883}
884
885// On every indirect call we call a run-time function
886// __sanitizer_cov_indir_call* with two parameters:
887// - callee address,
888// - global cache array that contains CacheSize pointers (zero-initialized).
889// The cache is used to speed up recording the caller-callee pairs.
890// The address of the caller is passed implicitly via caller PC.
891// CacheSize is encoded in the name of the run-time function.
892void ModuleSanitizerCoverage::InjectCoverageForIndirectCalls(
893 Function &F, ArrayRef<Instruction *> IndirCalls) {
894 if (IndirCalls.empty())
895 return;
896 assert(Options.TracePC || Options.TracePCEntryExit || Options.TracePCGuard ||
897 Options.Inline8bitCounters || Options.InlineBoolFlag);
898 for (auto *I : IndirCalls) {
899 InstrumentationIRBuilder IRB(I);
900 CallBase &CB = cast<CallBase>(Val&: *I);
901 Value *Callee = CB.getCalledOperand();
902 if (isa<InlineAsm>(Val: Callee))
903 continue;
904 IRB.CreateCall(Callee: SanCovTracePCIndir, Args: IRB.CreatePointerCast(V: Callee, DestTy: IntptrTy));
905 }
906}
907
908// For every switch statement we insert a call:
909// __sanitizer_cov_trace_switch(CondValue,
910// {NumCases, ValueSizeInBits, Case0Value, Case1Value, Case2Value, ... })
911
912void ModuleSanitizerCoverage::InjectTraceForSwitch(
913 Function &F, ArrayRef<Instruction *> SwitchTraceTargets,
914 Value *&FunctionGateCmp) {
915 for (auto *I : SwitchTraceTargets) {
916 if (SwitchInst *SI = dyn_cast<SwitchInst>(Val: I)) {
917 InstrumentationIRBuilder IRB(I);
918 SmallVector<Constant *, 16> Initializers;
919 Value *Cond = SI->getCondition();
920 if (Cond->getType()->getScalarSizeInBits() >
921 Int64Ty->getScalarSizeInBits())
922 continue;
923 Initializers.push_back(Elt: ConstantInt::get(Ty: Int64Ty, V: SI->getNumCases()));
924 Initializers.push_back(
925 Elt: ConstantInt::get(Ty: Int64Ty, V: Cond->getType()->getScalarSizeInBits()));
926 if (Cond->getType()->getScalarSizeInBits() <
927 Int64Ty->getScalarSizeInBits())
928 Cond = IRB.CreateIntCast(V: Cond, DestTy: Int64Ty, isSigned: false);
929 for (auto It : SI->cases()) {
930 ConstantInt *C = It.getCaseValue();
931 if (C->getType()->getScalarSizeInBits() < 64)
932 C = ConstantInt::get(Context&: C->getContext(), V: C->getValue().zext(width: 64));
933 Initializers.push_back(Elt: C);
934 }
935 llvm::sort(C: drop_begin(RangeOrContainer&: Initializers, N: 2),
936 Comp: [](const Constant *A, const Constant *B) {
937 return cast<ConstantInt>(Val: A)->getLimitedValue() <
938 cast<ConstantInt>(Val: B)->getLimitedValue();
939 });
940 ArrayType *ArrayOfInt64Ty = ArrayType::get(ElementType: Int64Ty, NumElements: Initializers.size());
941 GlobalVariable *GV = new GlobalVariable(
942 *CurModule, ArrayOfInt64Ty, false, GlobalVariable::InternalLinkage,
943 ConstantArray::get(T: ArrayOfInt64Ty, V: Initializers),
944 "__sancov_gen_cov_switch_values");
945 if (Options.GatedCallbacks) {
946 auto GateBranch = CreateGateBranch(F, FunctionGateCmp, IP: I);
947 IRBuilder<> GateIRB(GateBranch);
948 GateIRB.CreateCall(Callee: SanCovTraceSwitchFunction, Args: {Cond, GV});
949 } else {
950 IRB.CreateCall(Callee: SanCovTraceSwitchFunction, Args: {Cond, GV});
951 }
952 }
953 }
954}
955
956void ModuleSanitizerCoverage::InjectTraceForDiv(
957 Function &, ArrayRef<BinaryOperator *> DivTraceTargets) {
958 for (auto *BO : DivTraceTargets) {
959 InstrumentationIRBuilder IRB(BO);
960 Value *A1 = BO->getOperand(i_nocapture: 1);
961 if (isa<ConstantInt>(Val: A1))
962 continue;
963 if (!A1->getType()->isIntegerTy())
964 continue;
965 uint64_t TypeSize = DL->getTypeStoreSizeInBits(Ty: A1->getType());
966 int CallbackIdx = TypeSize == 32 ? 0 : TypeSize == 64 ? 1 : -1;
967 if (CallbackIdx < 0)
968 continue;
969 auto Ty = Type::getIntNTy(C&: *C, N: TypeSize);
970 IRB.CreateCall(Callee: SanCovTraceDivFunction[CallbackIdx],
971 Args: {IRB.CreateIntCast(V: A1, DestTy: Ty, isSigned: true)});
972 }
973}
974
975void ModuleSanitizerCoverage::InjectTraceForGep(
976 Function &, ArrayRef<GetElementPtrInst *> GepTraceTargets) {
977 for (auto *GEP : GepTraceTargets) {
978 InstrumentationIRBuilder IRB(GEP);
979 for (Use &Idx : GEP->indices())
980 if (!isa<ConstantInt>(Val: Idx) && Idx->getType()->isIntegerTy())
981 IRB.CreateCall(Callee: SanCovTraceGepFunction,
982 Args: {IRB.CreateIntCast(V: Idx, DestTy: IntptrTy, isSigned: true)});
983 }
984}
985
986void ModuleSanitizerCoverage::InjectTraceForLoadsAndStores(
987 Function &, ArrayRef<LoadInst *> Loads, ArrayRef<StoreInst *> Stores) {
988 auto CallbackIdx = [&](Type *ElementTy) -> int {
989 uint64_t TypeSize = DL->getTypeStoreSizeInBits(Ty: ElementTy);
990 return TypeSize == 8 ? 0
991 : TypeSize == 16 ? 1
992 : TypeSize == 32 ? 2
993 : TypeSize == 64 ? 3
994 : TypeSize == 128 ? 4
995 : -1;
996 };
997 for (auto *LI : Loads) {
998 InstrumentationIRBuilder IRB(LI);
999 auto Ptr = LI->getPointerOperand();
1000 int Idx = CallbackIdx(LI->getType());
1001 if (Idx < 0)
1002 continue;
1003 IRB.CreateCall(Callee: SanCovLoadFunction[Idx], Args: Ptr);
1004 }
1005 for (auto *SI : Stores) {
1006 InstrumentationIRBuilder IRB(SI);
1007 auto Ptr = SI->getPointerOperand();
1008 int Idx = CallbackIdx(SI->getValueOperand()->getType());
1009 if (Idx < 0)
1010 continue;
1011 IRB.CreateCall(Callee: SanCovStoreFunction[Idx], Args: Ptr);
1012 }
1013}
1014
1015void ModuleSanitizerCoverage::InjectTraceForExits(Function &F) {
1016 EscapeEnumerator EE(F, "sancov_exit");
1017 while (IRBuilder<> *AtExit = EE.Next()) {
1018 InstrumentationIRBuilder::ensureDebugInfo(IRB&: *AtExit, F);
1019 AtExit->CreateCall(Callee: SanCovTracePCExit, Args: {})
1020 ->setTailCallKind(CallInst::TCK_NoTail);
1021 }
1022}
1023
1024void ModuleSanitizerCoverage::InjectTraceForCmp(
1025 Function &F, ArrayRef<Instruction *> CmpTraceTargets,
1026 Value *&FunctionGateCmp) {
1027 for (auto *I : CmpTraceTargets) {
1028 if (ICmpInst *ICMP = dyn_cast<ICmpInst>(Val: I)) {
1029 InstrumentationIRBuilder IRB(ICMP);
1030 Value *A0 = ICMP->getOperand(i_nocapture: 0);
1031 Value *A1 = ICMP->getOperand(i_nocapture: 1);
1032 if (!A0->getType()->isIntegerTy())
1033 continue;
1034 uint64_t TypeSize = DL->getTypeStoreSizeInBits(Ty: A0->getType());
1035 int CallbackIdx = TypeSize == 8 ? 0
1036 : TypeSize == 16 ? 1
1037 : TypeSize == 32 ? 2
1038 : TypeSize == 64 ? 3
1039 : -1;
1040 if (CallbackIdx < 0)
1041 continue;
1042 // __sanitizer_cov_trace_cmp((type_size << 32) | predicate, A0, A1);
1043 auto CallbackFunc = SanCovTraceCmpFunction[CallbackIdx];
1044 bool FirstIsConst = isa<ConstantInt>(Val: A0);
1045 bool SecondIsConst = isa<ConstantInt>(Val: A1);
1046 // If both are const, then we don't need such a comparison.
1047 if (FirstIsConst && SecondIsConst)
1048 continue;
1049 // If only one is const, then make it the first callback argument.
1050 if (FirstIsConst || SecondIsConst) {
1051 CallbackFunc = SanCovTraceConstCmpFunction[CallbackIdx];
1052 if (SecondIsConst)
1053 std::swap(a&: A0, b&: A1);
1054 }
1055
1056 auto Ty = Type::getIntNTy(C&: *C, N: TypeSize);
1057 if (Options.GatedCallbacks) {
1058 auto GateBranch = CreateGateBranch(F, FunctionGateCmp, IP: I);
1059 IRBuilder<> GateIRB(GateBranch);
1060 GateIRB.CreateCall(Callee: CallbackFunc, Args: {GateIRB.CreateIntCast(V: A0, DestTy: Ty, isSigned: true),
1061 GateIRB.CreateIntCast(V: A1, DestTy: Ty, isSigned: true)});
1062 } else {
1063 IRB.CreateCall(Callee: CallbackFunc, Args: {IRB.CreateIntCast(V: A0, DestTy: Ty, isSigned: true),
1064 IRB.CreateIntCast(V: A1, DestTy: Ty, isSigned: true)});
1065 }
1066 }
1067 }
1068}
1069
1070void ModuleSanitizerCoverage::InjectCoverageAtBlock(Function &F, BasicBlock &BB,
1071 size_t Idx,
1072 Value *&FunctionGateCmp,
1073 bool IsLeafFunc) {
1074 BasicBlock::iterator IP = BB.getFirstInsertionPt();
1075 bool IsEntryBB = &BB == &F.getEntryBlock();
1076 DebugLoc EntryLoc;
1077 if (IsEntryBB) {
1078 if (auto SP = F.getSubprogram())
1079 EntryLoc = DILocation::get(Context&: SP->getContext(), Line: SP->getScopeLine(), Column: 0, Scope: SP);
1080 // Keep static allocas and llvm.localescape calls in the entry block. Even
1081 // if we aren't splitting the block, it's nice for allocas to be before
1082 // calls.
1083 IP = PrepareToSplitEntryBlock(BB, IP);
1084 }
1085
1086 InstrumentationIRBuilder IRB(&*IP);
1087 if (EntryLoc)
1088 IRB.SetCurrentDebugLocation(EntryLoc);
1089 if (Options.TracePC || (IsEntryBB && Options.TracePCEntryExit)) {
1090 FunctionCallee Callee = IsEntryBB && Options.TracePCEntryExit
1091 ? SanCovTracePCEntry
1092 : SanCovTracePC;
1093 IRB.CreateCall(Callee)
1094 ->setCannotMerge(); // gets the PC using GET_CALLER_PC.
1095 }
1096 if (Options.TracePCGuard) {
1097 auto GuardPtr = IRB.CreateConstInBoundsGEP2_64(
1098 Ty: FunctionGuardArray->getValueType(), Ptr: FunctionGuardArray, Idx0: 0, Idx1: Idx);
1099 if (Options.GatedCallbacks) {
1100 Instruction *I = &*IP;
1101 auto GateBranch = CreateGateBranch(F, FunctionGateCmp, IP: I);
1102 IRBuilder<> GateIRB(GateBranch);
1103 GateIRB.CreateCall(Callee: SanCovTracePCGuard, Args: GuardPtr)->setCannotMerge();
1104 } else {
1105 IRB.CreateCall(Callee: SanCovTracePCGuard, Args: GuardPtr)->setCannotMerge();
1106 }
1107 }
1108 if (Options.Inline8bitCounters) {
1109 auto CounterPtr = IRB.CreateGEP(
1110 Ty: Function8bitCounterArray->getValueType(), Ptr: Function8bitCounterArray,
1111 IdxList: {ConstantInt::get(Ty: IntptrTy, V: 0), ConstantInt::get(Ty: IntptrTy, V: Idx)});
1112 auto Load = IRB.CreateLoad(Ty: Int8Ty, Ptr: CounterPtr);
1113 auto Inc = IRB.CreateAdd(LHS: Load, RHS: ConstantInt::get(Ty: Int8Ty, V: 1));
1114 auto Store = IRB.CreateStore(Val: Inc, Ptr: CounterPtr);
1115 Load->setNoSanitizeMetadata();
1116 Store->setNoSanitizeMetadata();
1117 }
1118 if (Options.InlineBoolFlag) {
1119 auto FlagPtr = IRB.CreateGEP(
1120 Ty: FunctionBoolArray->getValueType(), Ptr: FunctionBoolArray,
1121 IdxList: {ConstantInt::get(Ty: IntptrTy, V: 0), ConstantInt::get(Ty: IntptrTy, V: Idx)});
1122 auto Load = IRB.CreateLoad(Ty: Int1Ty, Ptr: FlagPtr);
1123 auto ThenTerm = SplitBlockAndInsertIfThen(
1124 Cond: IRB.CreateIsNull(Arg: Load), SplitBefore: &*IP, Unreachable: false,
1125 BranchWeights: MDBuilder(IRB.getContext()).createUnlikelyBranchWeights());
1126 InstrumentationIRBuilder ThenIRB(ThenTerm);
1127 auto Store = ThenIRB.CreateStore(Val: ConstantInt::getTrue(Ty: Int1Ty), Ptr: FlagPtr);
1128 if (EntryLoc)
1129 Store->setDebugLoc(EntryLoc);
1130 Load->setNoSanitizeMetadata();
1131 Store->setNoSanitizeMetadata();
1132 }
1133 if (Options.StackDepth && IsEntryBB && !IsLeafFunc) {
1134 Module *M = F.getParent();
1135 const DataLayout &DL = M->getDataLayout();
1136
1137 if (Options.StackDepthCallbackMin) {
1138 // In callback mode, only add call when stack depth reaches minimum.
1139 int EstimatedStackSize = 0;
1140 // If dynamic alloca found, always add call.
1141 bool HasDynamicAlloc = false;
1142 // Find an insertion point after last "alloca".
1143 llvm::Instruction *InsertBefore = nullptr;
1144
1145 // Examine all allocas in the basic block. since we're too early
1146 // to have results from Intrinsic::frameaddress, we have to manually
1147 // estimate the stack size.
1148 for (auto &I : BB) {
1149 if (auto *AI = dyn_cast<AllocaInst>(Val: &I)) {
1150 // Move potential insertion point past the "alloca".
1151 InsertBefore = AI->getNextNode();
1152
1153 // Make an estimate on the stack usage.
1154 if (auto AllocaSize = AI->getAllocationSize(DL)) {
1155 if (AllocaSize->isFixed())
1156 EstimatedStackSize += AllocaSize->getFixedValue();
1157 else
1158 HasDynamicAlloc = true;
1159 } else {
1160 HasDynamicAlloc = true;
1161 }
1162 }
1163 }
1164
1165 if (HasDynamicAlloc ||
1166 EstimatedStackSize >= Options.StackDepthCallbackMin) {
1167 if (InsertBefore)
1168 IRB.SetInsertPoint(InsertBefore);
1169 auto Call = IRB.CreateCall(Callee: SanCovStackDepthCallback);
1170 if (EntryLoc)
1171 Call->setDebugLoc(EntryLoc);
1172 Call->setCannotMerge();
1173 }
1174 } else {
1175 // Check stack depth. If it's the deepest so far, record it.
1176 auto FrameAddrPtr = IRB.CreateIntrinsic(
1177 ID: Intrinsic::frameaddress, OverloadTypes: IRB.getPtrTy(AddrSpace: DL.getAllocaAddrSpace()),
1178 Args: {Constant::getNullValue(Ty: Int32Ty)});
1179 auto FrameAddrInt = IRB.CreatePtrToInt(V: FrameAddrPtr, DestTy: IntptrTy);
1180 auto LowestStack = IRB.CreateLoad(Ty: IntptrTy, Ptr: SanCovLowestStack);
1181 auto IsStackLower = IRB.CreateICmpULT(LHS: FrameAddrInt, RHS: LowestStack);
1182 auto ThenTerm = SplitBlockAndInsertIfThen(
1183 Cond: IsStackLower, SplitBefore: &*IP, Unreachable: false,
1184 BranchWeights: MDBuilder(IRB.getContext()).createUnlikelyBranchWeights());
1185 InstrumentationIRBuilder ThenIRB(ThenTerm);
1186 auto Store = ThenIRB.CreateStore(Val: FrameAddrInt, Ptr: SanCovLowestStack);
1187 if (EntryLoc)
1188 Store->setDebugLoc(EntryLoc);
1189 LowestStack->setNoSanitizeMetadata();
1190 Store->setNoSanitizeMetadata();
1191 }
1192 }
1193}
1194
1195std::string
1196ModuleSanitizerCoverage::getSectionName(const std::string &Section) const {
1197 if (TargetTriple.isOSBinFormatCOFF()) {
1198 if (Section == SanCovCountersSectionName)
1199 return ".SCOV$CM";
1200 if (Section == SanCovBoolFlagSectionName)
1201 return ".SCOV$BM";
1202 if (Section == SanCovPCsSectionName)
1203 return ".SCOVP$M";
1204 return ".SCOV$GM"; // For SanCovGuardsSectionName.
1205 }
1206 if (TargetTriple.isOSBinFormatMachO())
1207 return "__DATA,__" + Section;
1208 return "__" + Section;
1209}
1210
1211std::string
1212ModuleSanitizerCoverage::getSectionStart(const std::string &Section) const {
1213 if (TargetTriple.isOSBinFormatMachO())
1214 return "\1section$start$__DATA$__" + Section;
1215 return "__start___" + Section;
1216}
1217
1218std::string
1219ModuleSanitizerCoverage::getSectionEnd(const std::string &Section) const {
1220 if (TargetTriple.isOSBinFormatMachO())
1221 return "\1section$end$__DATA$__" + Section;
1222 return "__stop___" + Section;
1223}
1224
1225void ModuleSanitizerCoverage::createFunctionControlFlow(Function &F) {
1226 SmallVector<Constant *, 32> CFs;
1227 IRBuilder<> IRB(&*F.getEntryBlock().getFirstInsertionPt());
1228
1229 for (auto &BB : F) {
1230 // blockaddress can not be used on function's entry block.
1231 if (&BB == &F.getEntryBlock())
1232 CFs.push_back(Elt: (Constant *)IRB.CreatePointerCast(V: &F, DestTy: PtrTy));
1233 else
1234 CFs.push_back(
1235 Elt: (Constant *)IRB.CreatePointerCast(V: BlockAddress::get(BB: &BB), DestTy: PtrTy));
1236
1237 for (auto SuccBB : successors(BB: &BB)) {
1238 assert(SuccBB != &F.getEntryBlock());
1239 CFs.push_back(
1240 Elt: (Constant *)IRB.CreatePointerCast(V: BlockAddress::get(BB: SuccBB), DestTy: PtrTy));
1241 }
1242
1243 CFs.push_back(Elt: (Constant *)Constant::getNullValue(Ty: PtrTy));
1244
1245 for (auto &Inst : BB) {
1246 if (CallBase *CB = dyn_cast<CallBase>(Val: &Inst)) {
1247 if (CB->isIndirectCall()) {
1248 // TODO(navidem): handle indirect calls, for now mark its existence.
1249 CFs.push_back(Elt: (Constant *)IRB.CreateIntToPtr(
1250 V: ConstantInt::getAllOnesValue(Ty: IntptrTy), DestTy: PtrTy));
1251 } else {
1252 auto CalledF = CB->getCalledFunction();
1253 if (CalledF && !CalledF->isIntrinsic())
1254 CFs.push_back(Elt: (Constant *)IRB.CreatePointerCast(V: CalledF, DestTy: PtrTy));
1255 }
1256 }
1257 }
1258
1259 CFs.push_back(Elt: (Constant *)Constant::getNullValue(Ty: PtrTy));
1260 }
1261
1262 FunctionCFsArray = CreateFunctionLocalArrayInSection(NumElements: CFs.size(), F, Ty: PtrTy,
1263 Section: SanCovCFsSectionName);
1264 FunctionCFsArray->setInitializer(
1265 ConstantArray::get(T: ArrayType::get(ElementType: PtrTy, NumElements: CFs.size()), V: CFs));
1266 FunctionCFsArray->setConstant(true);
1267}
1268