1//===-- SanitizerCoverage.cpp - coverage instrumentation for sanitizers ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Coverage instrumentation done on LLVM IR level, works with Sanitizers.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/Transforms/Instrumentation/SanitizerCoverage.h"
14#include "llvm/ADT/ArrayRef.h"
15#include "llvm/ADT/SmallVector.h"
16#include "llvm/Analysis/GlobalsModRef.h"
17#include "llvm/Analysis/PostDominators.h"
18#include "llvm/IR/Constant.h"
19#include "llvm/IR/Constants.h"
20#include "llvm/IR/DataLayout.h"
21#include "llvm/IR/Dominators.h"
22#include "llvm/IR/EHPersonalities.h"
23#include "llvm/IR/Function.h"
24#include "llvm/IR/GlobalVariable.h"
25#include "llvm/IR/IRBuilder.h"
26#include "llvm/IR/IntrinsicInst.h"
27#include "llvm/IR/Intrinsics.h"
28#include "llvm/IR/LLVMContext.h"
29#include "llvm/IR/MDBuilder.h"
30#include "llvm/IR/Module.h"
31#include "llvm/IR/Type.h"
32#include "llvm/IR/ValueSymbolTable.h"
33#include "llvm/Support/CommandLine.h"
34#include "llvm/Support/SpecialCaseList.h"
35#include "llvm/Support/VirtualFileSystem.h"
36#include "llvm/TargetParser/Triple.h"
37#include "llvm/Transforms/Utils/BasicBlockUtils.h"
38#include "llvm/Transforms/Utils/EscapeEnumerator.h"
39#include "llvm/Transforms/Utils/ModuleUtils.h"
40
41using namespace llvm;
42
43#define DEBUG_TYPE "sancov"
44
45const char SanCovTracePCIndirName[] = "__sanitizer_cov_trace_pc_indir";
46const char SanCovTracePCName[] = "__sanitizer_cov_trace_pc";
47const char SanCovTracePCEntryName[] = "__sanitizer_cov_trace_pc_entry";
48const char SanCovTracePCExitName[] = "__sanitizer_cov_trace_pc_exit";
49const char SanCovTraceCmp1[] = "__sanitizer_cov_trace_cmp1";
50const char SanCovTraceCmp2[] = "__sanitizer_cov_trace_cmp2";
51const char SanCovTraceCmp4[] = "__sanitizer_cov_trace_cmp4";
52const char SanCovTraceCmp8[] = "__sanitizer_cov_trace_cmp8";
53const char SanCovTraceConstCmp1[] = "__sanitizer_cov_trace_const_cmp1";
54const char SanCovTraceConstCmp2[] = "__sanitizer_cov_trace_const_cmp2";
55const char SanCovTraceConstCmp4[] = "__sanitizer_cov_trace_const_cmp4";
56const char SanCovTraceConstCmp8[] = "__sanitizer_cov_trace_const_cmp8";
57const char SanCovLoad1[] = "__sanitizer_cov_load1";
58const char SanCovLoad2[] = "__sanitizer_cov_load2";
59const char SanCovLoad4[] = "__sanitizer_cov_load4";
60const char SanCovLoad8[] = "__sanitizer_cov_load8";
61const char SanCovLoad16[] = "__sanitizer_cov_load16";
62const char SanCovStore1[] = "__sanitizer_cov_store1";
63const char SanCovStore2[] = "__sanitizer_cov_store2";
64const char SanCovStore4[] = "__sanitizer_cov_store4";
65const char SanCovStore8[] = "__sanitizer_cov_store8";
66const char SanCovStore16[] = "__sanitizer_cov_store16";
67const char SanCovTraceDiv4[] = "__sanitizer_cov_trace_div4";
68const char SanCovTraceDiv8[] = "__sanitizer_cov_trace_div8";
69const char SanCovTraceGep[] = "__sanitizer_cov_trace_gep";
70const char SanCovTraceSwitchName[] = "__sanitizer_cov_trace_switch";
71const char SanCovModuleCtorTracePcGuardName[] =
72 "sancov.module_ctor_trace_pc_guard";
73const char SanCovModuleCtor8bitCountersName[] =
74 "sancov.module_ctor_8bit_counters";
75const char SanCovModuleCtorBoolFlagName[] = "sancov.module_ctor_bool_flag";
76static const uint64_t SanCtorAndDtorPriority = 2;
77
78const char SanCovTracePCGuardName[] = "__sanitizer_cov_trace_pc_guard";
79const char SanCovTracePCGuardInitName[] = "__sanitizer_cov_trace_pc_guard_init";
80const char SanCov8bitCountersInitName[] = "__sanitizer_cov_8bit_counters_init";
81const char SanCovBoolFlagInitName[] = "__sanitizer_cov_bool_flag_init";
82const char SanCovPCsInitName[] = "__sanitizer_cov_pcs_init";
83const char SanCovCFsInitName[] = "__sanitizer_cov_cfs_init";
84
85const char SanCovGuardsSectionName[] = "sancov_guards";
86const char SanCovCountersSectionName[] = "sancov_cntrs";
87const char SanCovBoolFlagSectionName[] = "sancov_bools";
88const char SanCovPCsSectionName[] = "sancov_pcs";
89const char SanCovCFsSectionName[] = "sancov_cfs";
90const char SanCovCallbackGateSectionName[] = "sancov_gate";
91
92const char SanCovStackDepthCallbackName[] = "__sanitizer_cov_stack_depth";
93const char SanCovLowestStackName[] = "__sancov_lowest_stack";
94const char SanCovCallbackGateName[] = "__sancov_should_track";
95
96static cl::opt<int> ClCoverageLevel(
97 "sanitizer-coverage-level",
98 cl::desc("Sanitizer Coverage. 0: none, 1: entry block, 2: all blocks, "
99 "3: all blocks and critical edges"),
100 cl::Hidden);
101
102static cl::opt<bool> ClTracePC("sanitizer-coverage-trace-pc",
103 cl::desc("Experimental pc tracing"), cl::Hidden);
104
105static cl::opt<bool> ClTracePCEntryExit(
106 "sanitizer-coverage-trace-pc-entry-exit",
107 cl::desc("pc tracing with separate entry/exit callbacks"), cl::Hidden);
108
109static cl::opt<bool> ClTracePCGuard("sanitizer-coverage-trace-pc-guard",
110 cl::desc("pc tracing with a guard"),
111 cl::Hidden);
112
113// If true, we create a global variable that contains PCs of all instrumented
114// BBs, put this global into a named section, and pass this section's bounds
115// to __sanitizer_cov_pcs_init.
116// This way the coverage instrumentation does not need to acquire the PCs
117// at run-time. Works with trace-pc-guard, inline-8bit-counters, and
118// inline-bool-flag.
119static cl::opt<bool> ClCreatePCTable("sanitizer-coverage-pc-table",
120 cl::desc("create a static PC table"),
121 cl::Hidden);
122
123static cl::opt<bool>
124 ClInline8bitCounters("sanitizer-coverage-inline-8bit-counters",
125 cl::desc("increments 8-bit counter for every edge"),
126 cl::Hidden);
127
128static cl::opt<bool>
129 ClSancovDropCtors("sanitizer-coverage-drop-ctors",
130 cl::desc("do not emit module ctors for global counters"),
131 cl::Hidden);
132
133static cl::opt<bool>
134 ClInlineBoolFlag("sanitizer-coverage-inline-bool-flag",
135 cl::desc("sets a boolean flag for every edge"),
136 cl::Hidden);
137
138static cl::opt<bool>
139 ClCMPTracing("sanitizer-coverage-trace-compares",
140 cl::desc("Tracing of CMP and similar instructions"),
141 cl::Hidden);
142
143static cl::opt<bool> ClDIVTracing("sanitizer-coverage-trace-divs",
144 cl::desc("Tracing of DIV instructions"),
145 cl::Hidden);
146
147static cl::opt<bool> ClLoadTracing("sanitizer-coverage-trace-loads",
148 cl::desc("Tracing of load instructions"),
149 cl::Hidden);
150
151static cl::opt<bool> ClStoreTracing("sanitizer-coverage-trace-stores",
152 cl::desc("Tracing of store instructions"),
153 cl::Hidden);
154
155static cl::opt<bool> ClGEPTracing("sanitizer-coverage-trace-geps",
156 cl::desc("Tracing of GEP instructions"),
157 cl::Hidden);
158
159static cl::opt<bool>
160 ClPruneBlocks("sanitizer-coverage-prune-blocks",
161 cl::desc("Reduce the number of instrumented blocks"),
162 cl::Hidden, cl::init(Val: true));
163
164static cl::opt<bool> ClStackDepth("sanitizer-coverage-stack-depth",
165 cl::desc("max stack depth tracing"),
166 cl::Hidden);
167
168static cl::opt<int> ClStackDepthCallbackMin(
169 "sanitizer-coverage-stack-depth-callback-min",
170 cl::desc("max stack depth tracing should use callback and only when "
171 "stack depth more than specified"),
172 cl::Hidden);
173
174static cl::opt<bool>
175 ClCollectCF("sanitizer-coverage-control-flow",
176 cl::desc("collect control flow for each function"), cl::Hidden);
177
178static cl::opt<bool> ClGatedCallbacks(
179 "sanitizer-coverage-gated-trace-callbacks",
180 cl::desc("Gate the invocation of the tracing callbacks on a global variable"
181 ". Currently only supported for trace-pc-guard and trace-cmp."),
182 cl::Hidden, cl::init(Val: false));
183
184namespace {
185
186SanitizerCoverageOptions getOptions(int LegacyCoverageLevel) {
187 SanitizerCoverageOptions Res;
188 switch (LegacyCoverageLevel) {
189 case 0:
190 Res.CoverageType = SanitizerCoverageOptions::SCK_None;
191 break;
192 case 1:
193 Res.CoverageType = SanitizerCoverageOptions::SCK_Function;
194 break;
195 case 2:
196 Res.CoverageType = SanitizerCoverageOptions::SCK_BB;
197 break;
198 case 3:
199 Res.CoverageType = SanitizerCoverageOptions::SCK_Edge;
200 break;
201 case 4:
202 Res.CoverageType = SanitizerCoverageOptions::SCK_Edge;
203 Res.IndirectCalls = true;
204 break;
205 }
206 return Res;
207}
208
209SanitizerCoverageOptions OverrideFromCL(SanitizerCoverageOptions Options) {
210 // Sets CoverageType and IndirectCalls.
211 SanitizerCoverageOptions CLOpts = getOptions(LegacyCoverageLevel: ClCoverageLevel);
212 Options.CoverageType = std::max(a: Options.CoverageType, b: CLOpts.CoverageType);
213 Options.IndirectCalls |= CLOpts.IndirectCalls;
214 Options.TraceCmp |= ClCMPTracing;
215 Options.TraceDiv |= ClDIVTracing;
216 Options.TraceGep |= ClGEPTracing;
217 Options.TracePC |= ClTracePC;
218 Options.TracePCEntryExit |= ClTracePCEntryExit;
219 Options.TracePCGuard |= ClTracePCGuard;
220 Options.Inline8bitCounters |= ClInline8bitCounters;
221 Options.InlineBoolFlag |= ClInlineBoolFlag;
222 Options.PCTable |= ClCreatePCTable;
223 Options.NoPrune |= !ClPruneBlocks;
224 Options.StackDepth |= ClStackDepth;
225 Options.StackDepthCallbackMin = std::max(a: Options.StackDepthCallbackMin,
226 b: ClStackDepthCallbackMin.getValue());
227 Options.TraceLoads |= ClLoadTracing;
228 Options.TraceStores |= ClStoreTracing;
229 Options.GatedCallbacks |= ClGatedCallbacks;
230 if (!Options.TracePCGuard && !Options.TracePC && !Options.TracePCEntryExit &&
231 !Options.Inline8bitCounters && !Options.StackDepth &&
232 !Options.InlineBoolFlag && !Options.TraceLoads && !Options.TraceStores)
233 Options.TracePCGuard = true; // TracePCGuard is default.
234 Options.CollectControlFlow |= ClCollectCF;
235 return Options;
236}
237
238class ModuleSanitizerCoverage {
239public:
240 using DomTreeCallback = function_ref<const DominatorTree &(Function &F)>;
241 using PostDomTreeCallback =
242 function_ref<const PostDominatorTree &(Function &F)>;
243
244 ModuleSanitizerCoverage(Module &M, DomTreeCallback DTCallback,
245 PostDomTreeCallback PDTCallback,
246 const SanitizerCoverageOptions &Options,
247 const SpecialCaseList *Allowlist,
248 const SpecialCaseList *Blocklist)
249 : M(M), DTCallback(DTCallback), PDTCallback(PDTCallback),
250 Options(Options), Allowlist(Allowlist), Blocklist(Blocklist) {}
251
252 bool instrumentModule();
253
254private:
255 void createFunctionControlFlow(Function &F);
256 void instrumentFunction(Function &F);
257 void InjectCoverageForIndirectCalls(Function &F,
258 ArrayRef<Instruction *> IndirCalls);
259 void InjectTraceForCmp(Function &F, ArrayRef<Instruction *> CmpTraceTargets,
260 Value *&FunctionGateCmp);
261 void InjectTraceForDiv(Function &F,
262 ArrayRef<BinaryOperator *> DivTraceTargets);
263 void InjectTraceForGep(Function &F,
264 ArrayRef<GetElementPtrInst *> GepTraceTargets);
265 void InjectTraceForLoadsAndStores(Function &F, ArrayRef<LoadInst *> Loads,
266 ArrayRef<StoreInst *> Stores);
267 void InjectTraceForExits(Function &F);
268 void InjectTraceForSwitch(Function &F,
269 ArrayRef<Instruction *> SwitchTraceTargets,
270 Value *&FunctionGateCmp);
271 bool InjectCoverage(Function &F, ArrayRef<BasicBlock *> AllBlocks,
272 Value *&FunctionGateCmp, bool IsLeafFunc);
273 GlobalVariable *CreateFunctionLocalArrayInSection(size_t NumElements,
274 Function &F, Type *Ty,
275 const char *Section);
276 GlobalVariable *CreatePCArray(Function &F, ArrayRef<BasicBlock *> AllBlocks);
277 void CreateFunctionLocalArrays(Function &F, ArrayRef<BasicBlock *> AllBlocks);
278 Instruction *CreateGateBranch(Function &F, Value *&FunctionGateCmp,
279 Instruction *I);
280 Value *CreateFunctionLocalGateCmp(IRBuilder<> &IRB);
281 void InjectCoverageAtBlock(Function &F, BasicBlock &BB, size_t Idx,
282 Value *&FunctionGateCmp, bool IsLeafFunc);
283 Function *CreateInitCallsForSections(Module &M, const char *CtorName,
284 const char *InitFunctionName, Type *Ty,
285 const char *Section);
286 std::pair<Value *, Value *> CreateSecStartEnd(Module &M, const char *Section,
287 Type *Ty);
288
289 std::string getSectionName(const std::string &Section) const;
290 std::string getSectionStart(const std::string &Section) const;
291 std::string getSectionEnd(const std::string &Section) const;
292
293 Module &M;
294 DomTreeCallback DTCallback;
295 PostDomTreeCallback PDTCallback;
296
297 FunctionCallee SanCovStackDepthCallback;
298 FunctionCallee SanCovTracePCIndir;
299 FunctionCallee SanCovTracePC, SanCovTracePCGuard;
300 FunctionCallee SanCovTracePCEntry, SanCovTracePCExit;
301 std::array<FunctionCallee, 4> SanCovTraceCmpFunction;
302 std::array<FunctionCallee, 4> SanCovTraceConstCmpFunction;
303 std::array<FunctionCallee, 5> SanCovLoadFunction;
304 std::array<FunctionCallee, 5> SanCovStoreFunction;
305 std::array<FunctionCallee, 2> SanCovTraceDivFunction;
306 FunctionCallee SanCovTraceGepFunction;
307 FunctionCallee SanCovTraceSwitchFunction;
308 GlobalVariable *SanCovLowestStack;
309 GlobalVariable *SanCovCallbackGate;
310 Type *PtrTy, *IntptrTy, *Int64Ty, *Int32Ty, *Int16Ty, *Int8Ty, *Int1Ty;
311 Module *CurModule;
312 Triple TargetTriple;
313 LLVMContext *C;
314 const DataLayout *DL;
315
316 GlobalVariable *FunctionGuardArray; // for trace-pc-guard.
317 GlobalVariable *Function8bitCounterArray; // for inline-8bit-counters.
318 GlobalVariable *FunctionBoolArray; // for inline-bool-flag.
319 GlobalVariable *FunctionPCsArray; // for pc-table.
320 GlobalVariable *FunctionCFsArray; // for control flow table
321 SmallVector<GlobalValue *, 20> GlobalsToAppendToUsed;
322 SmallVector<GlobalValue *, 20> GlobalsToAppendToCompilerUsed;
323
324 SanitizerCoverageOptions Options;
325
326 const SpecialCaseList *Allowlist;
327 const SpecialCaseList *Blocklist;
328};
329} // namespace
330
331SanitizerCoveragePass::SanitizerCoveragePass(
332 SanitizerCoverageOptions Options, IntrusiveRefCntPtr<vfs::FileSystem> VFS,
333 const std::vector<std::string> &AllowlistFiles,
334 const std::vector<std::string> &BlocklistFiles)
335 : Options(std::move(Options)),
336 VFS(VFS ? std::move(VFS) : vfs::getRealFileSystem()) {
337 if (AllowlistFiles.size() > 0)
338 Allowlist = SpecialCaseList::createOrDie(Paths: AllowlistFiles, FS&: *this->VFS);
339 if (BlocklistFiles.size() > 0)
340 Blocklist = SpecialCaseList::createOrDie(Paths: BlocklistFiles, FS&: *this->VFS);
341}
342
343PreservedAnalyses SanitizerCoveragePass::run(Module &M,
344 ModuleAnalysisManager &MAM) {
345 auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
346 auto DTCallback = [&FAM](Function &F) -> const DominatorTree & {
347 return FAM.getResult<DominatorTreeAnalysis>(IR&: F);
348 };
349 auto PDTCallback = [&FAM](Function &F) -> const PostDominatorTree & {
350 return FAM.getResult<PostDominatorTreeAnalysis>(IR&: F);
351 };
352 ModuleSanitizerCoverage ModuleSancov(M, DTCallback, PDTCallback,
353 OverrideFromCL(Options), Allowlist.get(),
354 Blocklist.get());
355 if (!ModuleSancov.instrumentModule())
356 return PreservedAnalyses::all();
357
358 PreservedAnalyses PA = PreservedAnalyses::none();
359 // GlobalsAA is considered stateless and does not get invalidated unless
360 // explicitly invalidated; PreservedAnalyses::none() is not enough. Sanitizers
361 // make changes that require GlobalsAA to be invalidated.
362 PA.abandon<GlobalsAA>();
363 return PA;
364}
365
366std::pair<Value *, Value *>
367ModuleSanitizerCoverage::CreateSecStartEnd(Module &M, const char *Section,
368 Type *Ty) {
369 // Use ExternalWeak so that if all sections are discarded due to section
370 // garbage collection, the linker will not report undefined symbol errors.
371 // Windows defines the start/stop symbols in compiler-rt so no need for
372 // ExternalWeak.
373 GlobalValue::LinkageTypes Linkage = TargetTriple.isOSBinFormatCOFF()
374 ? GlobalVariable::ExternalLinkage
375 : GlobalVariable::ExternalWeakLinkage;
376 GlobalVariable *SecStart = new GlobalVariable(M, Ty, false, Linkage, nullptr,
377 getSectionStart(Section));
378 SecStart->setVisibility(GlobalValue::HiddenVisibility);
379 GlobalVariable *SecEnd = new GlobalVariable(M, Ty, false, Linkage, nullptr,
380 getSectionEnd(Section));
381 SecEnd->setVisibility(GlobalValue::HiddenVisibility);
382 IRBuilder<> IRB(M.getContext());
383 if (!TargetTriple.isOSBinFormatCOFF())
384 return std::make_pair(x&: SecStart, y&: SecEnd);
385
386 // Account for the fact that on windows-msvc __start_* symbols actually
387 // point to a uint64_t before the start of the array.
388 auto GEP =
389 IRB.CreatePtrAdd(Ptr: SecStart, Offset: ConstantInt::get(Ty: IntptrTy, V: sizeof(uint64_t)));
390 return std::make_pair(x&: GEP, y&: SecEnd);
391}
392
393Function *ModuleSanitizerCoverage::CreateInitCallsForSections(
394 Module &M, const char *CtorName, const char *InitFunctionName, Type *Ty,
395 const char *Section) {
396 if (ClSancovDropCtors)
397 return nullptr;
398 auto SecStartEnd = CreateSecStartEnd(M, Section, Ty);
399 auto SecStart = SecStartEnd.first;
400 auto SecEnd = SecStartEnd.second;
401 Function *CtorFunc;
402 std::tie(args&: CtorFunc, args: std::ignore) = createSanitizerCtorAndInitFunctions(
403 M, CtorName, InitName: InitFunctionName, InitArgTypes: {PtrTy, PtrTy}, InitArgs: {SecStart, SecEnd});
404 assert(CtorFunc->getName() == CtorName);
405
406 if (TargetTriple.supportsCOMDAT()) {
407 // Use comdat to dedup CtorFunc.
408 CtorFunc->setComdat(M.getOrInsertComdat(Name: CtorName));
409 appendToGlobalCtors(M, F: CtorFunc, Priority: SanCtorAndDtorPriority, Data: CtorFunc);
410 } else {
411 appendToGlobalCtors(M, F: CtorFunc, Priority: SanCtorAndDtorPriority);
412 }
413
414 if (TargetTriple.isOSBinFormatCOFF()) {
415 // In COFF files, if the contructors are set as COMDAT (they are because
416 // COFF supports COMDAT) and the linker flag /OPT:REF (strip unreferenced
417 // functions and data) is used, the constructors get stripped. To prevent
418 // this, give the constructors weak ODR linkage and ensure the linker knows
419 // to include the sancov constructor. This way the linker can deduplicate
420 // the constructors but always leave one copy.
421 CtorFunc->setLinkage(GlobalValue::WeakODRLinkage);
422 }
423 return CtorFunc;
424}
425
426bool ModuleSanitizerCoverage::instrumentModule() {
427 if (Options.CoverageType == SanitizerCoverageOptions::SCK_None)
428 return false;
429 if (Allowlist &&
430 !Allowlist->inSection(Section: "coverage", Prefix: "src", Query: M.getSourceFileName()))
431 return false;
432 if (Blocklist &&
433 Blocklist->inSection(Section: "coverage", Prefix: "src", Query: M.getSourceFileName()))
434 return false;
435 C = &(M.getContext());
436 DL = &M.getDataLayout();
437 CurModule = &M;
438 TargetTriple = M.getTargetTriple();
439 FunctionGuardArray = nullptr;
440 Function8bitCounterArray = nullptr;
441 FunctionBoolArray = nullptr;
442 FunctionPCsArray = nullptr;
443 FunctionCFsArray = nullptr;
444 IntptrTy = Type::getIntNTy(C&: *C, N: DL->getPointerSizeInBits());
445 PtrTy = PointerType::getUnqual(C&: *C);
446 Type *VoidTy = Type::getVoidTy(C&: *C);
447 IRBuilder<> IRB(*C);
448 Int64Ty = IRB.getInt64Ty();
449 Int32Ty = IRB.getInt32Ty();
450 Int16Ty = IRB.getInt16Ty();
451 Int8Ty = IRB.getInt8Ty();
452 Int1Ty = IRB.getInt1Ty();
453
454 SanCovTracePCIndir =
455 M.getOrInsertFunction(Name: SanCovTracePCIndirName, RetTy: VoidTy, Args: IntptrTy);
456 // Make sure smaller parameters are zero-extended to i64 if required by the
457 // target ABI.
458 AttributeList SanCovTraceCmpZeroExtAL;
459 SanCovTraceCmpZeroExtAL =
460 SanCovTraceCmpZeroExtAL.addParamAttribute(C&: *C, ArgNo: 0, Kind: Attribute::ZExt);
461 SanCovTraceCmpZeroExtAL =
462 SanCovTraceCmpZeroExtAL.addParamAttribute(C&: *C, ArgNo: 1, Kind: Attribute::ZExt);
463
464 SanCovTraceCmpFunction[0] =
465 M.getOrInsertFunction(Name: SanCovTraceCmp1, AttributeList: SanCovTraceCmpZeroExtAL, RetTy: VoidTy,
466 Args: IRB.getInt8Ty(), Args: IRB.getInt8Ty());
467 SanCovTraceCmpFunction[1] =
468 M.getOrInsertFunction(Name: SanCovTraceCmp2, AttributeList: SanCovTraceCmpZeroExtAL, RetTy: VoidTy,
469 Args: IRB.getInt16Ty(), Args: IRB.getInt16Ty());
470 SanCovTraceCmpFunction[2] =
471 M.getOrInsertFunction(Name: SanCovTraceCmp4, AttributeList: SanCovTraceCmpZeroExtAL, RetTy: VoidTy,
472 Args: IRB.getInt32Ty(), Args: IRB.getInt32Ty());
473 SanCovTraceCmpFunction[3] =
474 M.getOrInsertFunction(Name: SanCovTraceCmp8, RetTy: VoidTy, Args: Int64Ty, Args: Int64Ty);
475
476 SanCovTraceConstCmpFunction[0] = M.getOrInsertFunction(
477 Name: SanCovTraceConstCmp1, AttributeList: SanCovTraceCmpZeroExtAL, RetTy: VoidTy, Args: Int8Ty, Args: Int8Ty);
478 SanCovTraceConstCmpFunction[1] = M.getOrInsertFunction(
479 Name: SanCovTraceConstCmp2, AttributeList: SanCovTraceCmpZeroExtAL, RetTy: VoidTy, Args: Int16Ty, Args: Int16Ty);
480 SanCovTraceConstCmpFunction[2] = M.getOrInsertFunction(
481 Name: SanCovTraceConstCmp4, AttributeList: SanCovTraceCmpZeroExtAL, RetTy: VoidTy, Args: Int32Ty, Args: Int32Ty);
482 SanCovTraceConstCmpFunction[3] =
483 M.getOrInsertFunction(Name: SanCovTraceConstCmp8, RetTy: VoidTy, Args: Int64Ty, Args: Int64Ty);
484
485 // Loads.
486 SanCovLoadFunction[0] = M.getOrInsertFunction(Name: SanCovLoad1, RetTy: VoidTy, Args: PtrTy);
487 SanCovLoadFunction[1] = M.getOrInsertFunction(Name: SanCovLoad2, RetTy: VoidTy, Args: PtrTy);
488 SanCovLoadFunction[2] = M.getOrInsertFunction(Name: SanCovLoad4, RetTy: VoidTy, Args: PtrTy);
489 SanCovLoadFunction[3] = M.getOrInsertFunction(Name: SanCovLoad8, RetTy: VoidTy, Args: PtrTy);
490 SanCovLoadFunction[4] = M.getOrInsertFunction(Name: SanCovLoad16, RetTy: VoidTy, Args: PtrTy);
491 // Stores.
492 SanCovStoreFunction[0] = M.getOrInsertFunction(Name: SanCovStore1, RetTy: VoidTy, Args: PtrTy);
493 SanCovStoreFunction[1] = M.getOrInsertFunction(Name: SanCovStore2, RetTy: VoidTy, Args: PtrTy);
494 SanCovStoreFunction[2] = M.getOrInsertFunction(Name: SanCovStore4, RetTy: VoidTy, Args: PtrTy);
495 SanCovStoreFunction[3] = M.getOrInsertFunction(Name: SanCovStore8, RetTy: VoidTy, Args: PtrTy);
496 SanCovStoreFunction[4] = M.getOrInsertFunction(Name: SanCovStore16, RetTy: VoidTy, Args: PtrTy);
497
498 {
499 AttributeList AL;
500 AL = AL.addParamAttribute(C&: *C, ArgNo: 0, Kind: Attribute::ZExt);
501 SanCovTraceDivFunction[0] =
502 M.getOrInsertFunction(Name: SanCovTraceDiv4, AttributeList: AL, RetTy: VoidTy, Args: IRB.getInt32Ty());
503 }
504 SanCovTraceDivFunction[1] =
505 M.getOrInsertFunction(Name: SanCovTraceDiv8, RetTy: VoidTy, Args: Int64Ty);
506 SanCovTraceGepFunction =
507 M.getOrInsertFunction(Name: SanCovTraceGep, RetTy: VoidTy, Args: IntptrTy);
508 SanCovTraceSwitchFunction =
509 M.getOrInsertFunction(Name: SanCovTraceSwitchName, RetTy: VoidTy, Args: Int64Ty, Args: PtrTy);
510
511 SanCovLowestStack = M.getOrInsertGlobal(Name: SanCovLowestStackName, Ty: IntptrTy);
512 if (SanCovLowestStack->getValueType() != IntptrTy) {
513 C->emitError(ErrorStr: StringRef("'") + SanCovLowestStackName +
514 "' should not be declared by the user");
515 return true;
516 }
517 SanCovLowestStack->setThreadLocalMode(
518 GlobalValue::ThreadLocalMode::InitialExecTLSModel);
519 if (Options.StackDepth && !SanCovLowestStack->isDeclaration())
520 SanCovLowestStack->setInitializer(Constant::getAllOnesValue(Ty: IntptrTy));
521
522 if (Options.GatedCallbacks) {
523 if (!Options.TracePCGuard && !Options.TraceCmp) {
524 C->emitError(ErrorStr: StringRef("'") + ClGatedCallbacks.ArgStr +
525 "' is only supported with trace-pc-guard or trace-cmp");
526 return true;
527 }
528
529 SanCovCallbackGate = cast<GlobalVariable>(
530 Val: M.getOrInsertGlobal(Name: SanCovCallbackGateName, Ty: Int64Ty));
531 SanCovCallbackGate->setSection(
532 getSectionName(Section: SanCovCallbackGateSectionName));
533 SanCovCallbackGate->setInitializer(Constant::getNullValue(Ty: Int64Ty));
534 SanCovCallbackGate->setLinkage(GlobalVariable::LinkOnceAnyLinkage);
535 SanCovCallbackGate->setVisibility(GlobalVariable::HiddenVisibility);
536 appendToCompilerUsed(M, Values: SanCovCallbackGate);
537 }
538
539 SanCovTracePC = M.getOrInsertFunction(Name: SanCovTracePCName, RetTy: VoidTy);
540 SanCovTracePCEntry = M.getOrInsertFunction(Name: SanCovTracePCEntryName, RetTy: VoidTy);
541 SanCovTracePCExit = M.getOrInsertFunction(Name: SanCovTracePCExitName, RetTy: VoidTy);
542 SanCovTracePCGuard =
543 M.getOrInsertFunction(Name: SanCovTracePCGuardName, RetTy: VoidTy, Args: PtrTy);
544
545 SanCovStackDepthCallback =
546 M.getOrInsertFunction(Name: SanCovStackDepthCallbackName, RetTy: VoidTy);
547
548 for (auto &F : M)
549 instrumentFunction(F);
550
551 Function *Ctor = nullptr;
552
553 if (FunctionGuardArray)
554 Ctor = CreateInitCallsForSections(M, CtorName: SanCovModuleCtorTracePcGuardName,
555 InitFunctionName: SanCovTracePCGuardInitName, Ty: Int32Ty,
556 Section: SanCovGuardsSectionName);
557 if (Function8bitCounterArray)
558 Ctor = CreateInitCallsForSections(M, CtorName: SanCovModuleCtor8bitCountersName,
559 InitFunctionName: SanCov8bitCountersInitName, Ty: Int8Ty,
560 Section: SanCovCountersSectionName);
561 if (FunctionBoolArray) {
562 Ctor = CreateInitCallsForSections(M, CtorName: SanCovModuleCtorBoolFlagName,
563 InitFunctionName: SanCovBoolFlagInitName, Ty: Int1Ty,
564 Section: SanCovBoolFlagSectionName);
565 }
566 if (Ctor && Options.PCTable) {
567 auto SecStartEnd = CreateSecStartEnd(M, Section: SanCovPCsSectionName, Ty: IntptrTy);
568 FunctionCallee InitFunction =
569 declareSanitizerInitFunction(M, InitName: SanCovPCsInitName, InitArgTypes: {PtrTy, PtrTy});
570 IRBuilder<> IRBCtor(Ctor->getEntryBlock().getTerminator());
571 IRBCtor.CreateCall(Callee: InitFunction, Args: {SecStartEnd.first, SecStartEnd.second});
572 }
573
574 if (Ctor && Options.CollectControlFlow) {
575 auto SecStartEnd = CreateSecStartEnd(M, Section: SanCovCFsSectionName, Ty: IntptrTy);
576 FunctionCallee InitFunction =
577 declareSanitizerInitFunction(M, InitName: SanCovCFsInitName, InitArgTypes: {PtrTy, PtrTy});
578 IRBuilder<> IRBCtor(Ctor->getEntryBlock().getTerminator());
579 IRBCtor.CreateCall(Callee: InitFunction, Args: {SecStartEnd.first, SecStartEnd.second});
580 }
581
582 appendToUsed(M, Values: GlobalsToAppendToUsed);
583 appendToCompilerUsed(M, Values: GlobalsToAppendToCompilerUsed);
584 return true;
585}
586
587// True if block has successors and it dominates all of them.
588static bool isFullDominator(const BasicBlock *BB, const DominatorTree &DT) {
589 if (succ_empty(BB))
590 return false;
591
592 return llvm::all_of(Range: successors(BB), P: [&](const BasicBlock *SUCC) {
593 return DT.dominates(A: BB, B: SUCC);
594 });
595}
596
597// True if block has predecessors and it postdominates all of them.
598static bool isFullPostDominator(const BasicBlock *BB,
599 const PostDominatorTree &PDT) {
600 if (pred_empty(BB))
601 return false;
602
603 return llvm::all_of(Range: predecessors(BB), P: [&](const BasicBlock *PRED) {
604 return PDT.dominates(A: BB, B: PRED);
605 });
606}
607
608static bool shouldInstrumentBlock(const Function &F, const BasicBlock *BB,
609 const DominatorTree &DT,
610 const PostDominatorTree &PDT,
611 const SanitizerCoverageOptions &Options) {
612 // Don't insert coverage for blocks containing nothing but unreachable: we
613 // will never call __sanitizer_cov() for them, so counting them in
614 // NumberOfInstrumentedBlocks() might complicate calculation of code coverage
615 // percentage. Also, unreachable instructions frequently have no debug
616 // locations.
617 if (isa<UnreachableInst>(Val: BB->getFirstNonPHIOrDbgOrLifetime()))
618 return false;
619
620 // Don't insert coverage into blocks without a valid insertion point
621 // (catchswitch blocks).
622 if (BB->getFirstInsertionPt() == BB->end())
623 return false;
624
625 if (Options.NoPrune || &F.getEntryBlock() == BB)
626 return true;
627
628 if (Options.CoverageType == SanitizerCoverageOptions::SCK_Function &&
629 &F.getEntryBlock() != BB)
630 return false;
631
632 // Do not instrument full dominators, or full post-dominators with multiple
633 // predecessors.
634 return !isFullDominator(BB, DT) &&
635 !(isFullPostDominator(BB, PDT) && !BB->getSinglePredecessor());
636}
637
638// Returns true iff From->To is a backedge.
639// A twist here is that we treat From->To as a backedge if
640// * To dominates From or
641// * To->UniqueSuccessor dominates From
642static bool IsBackEdge(BasicBlock *From, BasicBlock *To,
643 const DominatorTree &DT) {
644 if (DT.dominates(A: To, B: From))
645 return true;
646 if (auto Next = To->getUniqueSuccessor())
647 if (DT.dominates(A: Next, B: From))
648 return true;
649 return false;
650}
651
652// Prunes uninteresting Cmp instrumentation:
653// * CMP instructions that feed into loop backedge branch.
654//
655// Note that Cmp pruning is controlled by the same flag as the
656// BB pruning.
657static bool IsInterestingCmp(ICmpInst *CMP, const DominatorTree &DT,
658 const SanitizerCoverageOptions &Options) {
659 if (!Options.NoPrune)
660 if (CMP->hasOneUse())
661 if (auto BR = dyn_cast<CondBrInst>(Val: CMP->user_back()))
662 for (BasicBlock *B : BR->successors())
663 if (IsBackEdge(From: BR->getParent(), To: B, DT))
664 return false;
665 return true;
666}
667
668void ModuleSanitizerCoverage::instrumentFunction(Function &F) {
669 if (F.empty())
670 return;
671 if (F.getName().contains(Other: ".module_ctor"))
672 return; // Should not instrument sanitizer init functions.
673 if (F.getName().starts_with(Prefix: "__sanitizer_"))
674 return; // Don't instrument __sanitizer_* callbacks.
675 // Don't touch available_externally functions, their actual body is elewhere.
676 if (F.getLinkage() == GlobalValue::AvailableExternallyLinkage)
677 return;
678 // Don't instrument MSVC CRT configuration helpers. They may run before normal
679 // initialization.
680 if (F.getName() == "__local_stdio_printf_options" ||
681 F.getName() == "__local_stdio_scanf_options")
682 return;
683 if (isa<UnreachableInst>(Val: F.getEntryBlock().getTerminator()))
684 return;
685 // Don't instrument functions using SEH for now. Splitting basic blocks like
686 // we do for coverage breaks WinEHPrepare.
687 // FIXME: Remove this when SEH no longer uses landingpad pattern matching.
688 if (F.hasPersonalityFn() &&
689 isAsynchronousEHPersonality(Pers: classifyEHPersonality(Pers: F.getPersonalityFn())))
690 return;
691 if (Allowlist && !Allowlist->inSection(Section: "coverage", Prefix: "fun", Query: F.getName()))
692 return;
693 if (Blocklist && Blocklist->inSection(Section: "coverage", Prefix: "fun", Query: F.getName()))
694 return;
695 // Do not apply any instrumentation for naked functions.
696 if (F.hasFnAttribute(Kind: Attribute::Naked))
697 return;
698 if (F.hasFnAttribute(Kind: Attribute::NoSanitizeCoverage))
699 return;
700 if (F.hasFnAttribute(Kind: Attribute::DisableSanitizerInstrumentation))
701 return;
702 if (Options.CoverageType >= SanitizerCoverageOptions::SCK_Edge) {
703 SplitAllCriticalEdges(
704 F, Options: CriticalEdgeSplittingOptions().setIgnoreUnreachableDests());
705 }
706 SmallVector<Instruction *, 8> IndirCalls;
707 SmallVector<BasicBlock *, 16> BlocksToInstrument;
708 SmallVector<Instruction *, 8> CmpTraceTargets;
709 SmallVector<Instruction *, 8> SwitchTraceTargets;
710 SmallVector<BinaryOperator *, 8> DivTraceTargets;
711 SmallVector<GetElementPtrInst *, 8> GepTraceTargets;
712 SmallVector<LoadInst *, 8> Loads;
713 SmallVector<StoreInst *, 8> Stores;
714
715 const DominatorTree &DT = DTCallback(F);
716 const PostDominatorTree &PDT = PDTCallback(F);
717 bool IsLeafFunc = true;
718
719 for (auto &BB : F) {
720 if (shouldInstrumentBlock(F, BB: &BB, DT, PDT, Options))
721 BlocksToInstrument.push_back(Elt: &BB);
722 for (auto &Inst : BB) {
723 if (Options.IndirectCalls) {
724 CallBase *CB = dyn_cast<CallBase>(Val: &Inst);
725 if (CB && CB->isIndirectCall())
726 IndirCalls.push_back(Elt: &Inst);
727 }
728 if (Options.TraceCmp) {
729 if (ICmpInst *CMP = dyn_cast<ICmpInst>(Val: &Inst))
730 if (IsInterestingCmp(CMP, DT, Options))
731 CmpTraceTargets.push_back(Elt: &Inst);
732 if (isa<SwitchInst>(Val: &Inst))
733 SwitchTraceTargets.push_back(Elt: &Inst);
734 }
735 if (Options.TraceDiv)
736 if (BinaryOperator *BO = dyn_cast<BinaryOperator>(Val: &Inst))
737 if (BO->getOpcode() == Instruction::SDiv ||
738 BO->getOpcode() == Instruction::UDiv)
739 DivTraceTargets.push_back(Elt: BO);
740 if (Options.TraceGep)
741 if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Val: &Inst))
742 GepTraceTargets.push_back(Elt: GEP);
743 if (Options.TraceLoads)
744 if (LoadInst *LI = dyn_cast<LoadInst>(Val: &Inst))
745 Loads.push_back(Elt: LI);
746 if (Options.TraceStores)
747 if (StoreInst *SI = dyn_cast<StoreInst>(Val: &Inst))
748 Stores.push_back(Elt: SI);
749 if (Options.StackDepth)
750 if (isa<InvokeInst>(Val: Inst) ||
751 (isa<CallInst>(Val: Inst) && !isa<IntrinsicInst>(Val: Inst)))
752 IsLeafFunc = false;
753 }
754 }
755
756 if (Options.CollectControlFlow)
757 createFunctionControlFlow(F);
758
759 Value *FunctionGateCmp = nullptr;
760 InjectCoverage(F, AllBlocks: BlocksToInstrument, FunctionGateCmp, IsLeafFunc);
761 InjectCoverageForIndirectCalls(F, IndirCalls);
762 InjectTraceForCmp(F, CmpTraceTargets, FunctionGateCmp);
763 InjectTraceForSwitch(F, SwitchTraceTargets, FunctionGateCmp);
764 InjectTraceForDiv(F, DivTraceTargets);
765 InjectTraceForGep(F, GepTraceTargets);
766 InjectTraceForLoadsAndStores(F, Loads, Stores);
767
768 if (Options.TracePCEntryExit)
769 InjectTraceForExits(F);
770}
771
772GlobalVariable *ModuleSanitizerCoverage::CreateFunctionLocalArrayInSection(
773 size_t NumElements, Function &F, Type *Ty, const char *Section) {
774 ArrayType *ArrayTy = ArrayType::get(ElementType: Ty, NumElements);
775 auto Array = new GlobalVariable(
776 *CurModule, ArrayTy, false, GlobalVariable::PrivateLinkage,
777 Constant::getNullValue(Ty: ArrayTy), "__sancov_gen_");
778
779 if (TargetTriple.supportsCOMDAT() &&
780 (F.hasComdat() || TargetTriple.isOSBinFormatELF() || !F.isInterposable()))
781 if (auto Comdat = getOrCreateFunctionComdat(F, T&: TargetTriple))
782 Array->setComdat(Comdat);
783 Array->setSection(getSectionName(Section));
784 Array->setAlignment(Align(DL->getTypeStoreSize(Ty).getFixedValue()));
785
786 // sancov_pcs parallels the other metadata section(s). Optimizers (e.g.
787 // GlobalOpt/ConstantMerge) may not discard sancov_pcs and the other
788 // section(s) as a unit, so we conservatively retain all unconditionally in
789 // the compiler.
790 //
791 // With comdat (COFF/ELF), the linker can guarantee the associated sections
792 // will be retained or discarded as a unit, so llvm.compiler.used is
793 // sufficient. Otherwise, conservatively make all of them retained by the
794 // linker.
795 if (Array->hasComdat())
796 GlobalsToAppendToCompilerUsed.push_back(Elt: Array);
797 else
798 GlobalsToAppendToUsed.push_back(Elt: Array);
799
800 return Array;
801}
802
803GlobalVariable *
804ModuleSanitizerCoverage::CreatePCArray(Function &F,
805 ArrayRef<BasicBlock *> AllBlocks) {
806 size_t N = AllBlocks.size();
807 assert(N);
808 SmallVector<Constant *, 32> PCs;
809 IRBuilder<> IRB(&*F.getEntryBlock().getFirstInsertionPt());
810 for (size_t i = 0; i < N; i++) {
811 if (&F.getEntryBlock() == AllBlocks[i]) {
812 PCs.push_back(Elt: (Constant *)IRB.CreatePointerCast(V: &F, DestTy: PtrTy));
813 PCs.push_back(
814 Elt: (Constant *)IRB.CreateIntToPtr(V: ConstantInt::get(Ty: IntptrTy, V: 1), DestTy: PtrTy));
815 } else {
816 PCs.push_back(Elt: (Constant *)IRB.CreatePointerCast(
817 V: BlockAddress::get(BB: AllBlocks[i]), DestTy: PtrTy));
818 PCs.push_back(Elt: Constant::getNullValue(Ty: PtrTy));
819 }
820 }
821 auto *PCArray =
822 CreateFunctionLocalArrayInSection(NumElements: N * 2, F, Ty: PtrTy, Section: SanCovPCsSectionName);
823 PCArray->setInitializer(
824 ConstantArray::get(T: ArrayType::get(ElementType: PtrTy, NumElements: N * 2), V: PCs));
825 PCArray->setConstant(true);
826
827 return PCArray;
828}
829
830void ModuleSanitizerCoverage::CreateFunctionLocalArrays(
831 Function &F, ArrayRef<BasicBlock *> AllBlocks) {
832 if (Options.TracePCGuard)
833 FunctionGuardArray = CreateFunctionLocalArrayInSection(
834 NumElements: AllBlocks.size(), F, Ty: Int32Ty, Section: SanCovGuardsSectionName);
835
836 if (Options.Inline8bitCounters)
837 Function8bitCounterArray = CreateFunctionLocalArrayInSection(
838 NumElements: AllBlocks.size(), F, Ty: Int8Ty, Section: SanCovCountersSectionName);
839 if (Options.InlineBoolFlag)
840 FunctionBoolArray = CreateFunctionLocalArrayInSection(
841 NumElements: AllBlocks.size(), F, Ty: Int1Ty, Section: SanCovBoolFlagSectionName);
842
843 if (Options.PCTable)
844 FunctionPCsArray = CreatePCArray(F, AllBlocks);
845}
846
847Value *ModuleSanitizerCoverage::CreateFunctionLocalGateCmp(IRBuilder<> &IRB) {
848 auto Load = IRB.CreateLoad(Ty: Int64Ty, Ptr: SanCovCallbackGate);
849 Load->setNoSanitizeMetadata();
850 auto Cmp = IRB.CreateIsNotNull(Arg: Load);
851 Cmp->setName("sancov gate cmp");
852 return Cmp;
853}
854
855Instruction *ModuleSanitizerCoverage::CreateGateBranch(Function &F,
856 Value *&FunctionGateCmp,
857 Instruction *IP) {
858 if (!FunctionGateCmp) {
859 // Create this in the entry block
860 BasicBlock &BB = F.getEntryBlock();
861 BasicBlock::iterator IP = BB.getFirstInsertionPt();
862 IP = PrepareToSplitEntryBlock(BB, IP);
863 IRBuilder<> EntryIRB(&*IP);
864 FunctionGateCmp = CreateFunctionLocalGateCmp(IRB&: EntryIRB);
865 }
866 // Set the branch weights in order to minimize the price paid when the
867 // gate is turned off, allowing the default enablement of this
868 // instrumentation with as little of a performance cost as possible
869 auto Weights = MDBuilder(*C).createBranchWeights(TrueWeight: 1, FalseWeight: 100000);
870 return SplitBlockAndInsertIfThen(Cond: FunctionGateCmp, SplitBefore: IP, Unreachable: false, BranchWeights: Weights);
871}
872
873bool ModuleSanitizerCoverage::InjectCoverage(Function &F,
874 ArrayRef<BasicBlock *> AllBlocks,
875 Value *&FunctionGateCmp,
876 bool IsLeafFunc) {
877 if (AllBlocks.empty())
878 return false;
879 CreateFunctionLocalArrays(F, AllBlocks);
880 for (size_t i = 0, N = AllBlocks.size(); i < N; i++)
881 InjectCoverageAtBlock(F, BB&: *AllBlocks[i], Idx: i, FunctionGateCmp, IsLeafFunc);
882
883 return true;
884}
885
886// On every indirect call we call a run-time function
887// __sanitizer_cov_indir_call* with two parameters:
888// - callee address,
889// - global cache array that contains CacheSize pointers (zero-initialized).
890// The cache is used to speed up recording the caller-callee pairs.
891// The address of the caller is passed implicitly via caller PC.
892// CacheSize is encoded in the name of the run-time function.
893void ModuleSanitizerCoverage::InjectCoverageForIndirectCalls(
894 Function &F, ArrayRef<Instruction *> IndirCalls) {
895 if (IndirCalls.empty())
896 return;
897 assert(Options.TracePC || Options.TracePCEntryExit || Options.TracePCGuard ||
898 Options.Inline8bitCounters || Options.InlineBoolFlag);
899 for (auto *I : IndirCalls) {
900 InstrumentationIRBuilder IRB(I);
901 CallBase &CB = cast<CallBase>(Val&: *I);
902 Value *Callee = CB.getCalledOperand();
903 if (isa<InlineAsm>(Val: Callee))
904 continue;
905 IRB.CreateCall(Callee: SanCovTracePCIndir, Args: IRB.CreatePointerCast(V: Callee, DestTy: IntptrTy));
906 }
907}
908
909// For every switch statement we insert a call:
910// __sanitizer_cov_trace_switch(CondValue,
911// {NumCases, ValueSizeInBits, Case0Value, Case1Value, Case2Value, ... })
912
913void ModuleSanitizerCoverage::InjectTraceForSwitch(
914 Function &F, ArrayRef<Instruction *> SwitchTraceTargets,
915 Value *&FunctionGateCmp) {
916 for (auto *I : SwitchTraceTargets) {
917 if (SwitchInst *SI = dyn_cast<SwitchInst>(Val: I)) {
918 InstrumentationIRBuilder IRB(I);
919 SmallVector<Constant *, 16> Initializers;
920 Value *Cond = SI->getCondition();
921 if (Cond->getType()->getScalarSizeInBits() >
922 Int64Ty->getScalarSizeInBits())
923 continue;
924 Initializers.push_back(Elt: ConstantInt::get(Ty: Int64Ty, V: SI->getNumCases()));
925 Initializers.push_back(
926 Elt: ConstantInt::get(Ty: Int64Ty, V: Cond->getType()->getScalarSizeInBits()));
927 if (Cond->getType()->getScalarSizeInBits() <
928 Int64Ty->getScalarSizeInBits())
929 Cond = IRB.CreateIntCast(V: Cond, DestTy: Int64Ty, isSigned: false);
930 for (auto It : SI->cases()) {
931 ConstantInt *C = It.getCaseValue();
932 if (C->getType()->getScalarSizeInBits() < 64)
933 C = ConstantInt::get(Context&: C->getContext(), V: C->getValue().zext(width: 64));
934 Initializers.push_back(Elt: C);
935 }
936 llvm::sort(C: drop_begin(RangeOrContainer&: Initializers, N: 2),
937 Comp: [](const Constant *A, const Constant *B) {
938 return cast<ConstantInt>(Val: A)->getLimitedValue() <
939 cast<ConstantInt>(Val: B)->getLimitedValue();
940 });
941 ArrayType *ArrayOfInt64Ty = ArrayType::get(ElementType: Int64Ty, NumElements: Initializers.size());
942 GlobalVariable *GV = new GlobalVariable(
943 *CurModule, ArrayOfInt64Ty, false, GlobalVariable::InternalLinkage,
944 ConstantArray::get(T: ArrayOfInt64Ty, V: Initializers),
945 "__sancov_gen_cov_switch_values");
946 if (Options.GatedCallbacks) {
947 auto GateBranch = CreateGateBranch(F, FunctionGateCmp, IP: I);
948 IRBuilder<> GateIRB(GateBranch);
949 GateIRB.CreateCall(Callee: SanCovTraceSwitchFunction, Args: {Cond, GV});
950 } else {
951 IRB.CreateCall(Callee: SanCovTraceSwitchFunction, Args: {Cond, GV});
952 }
953 }
954 }
955}
956
957void ModuleSanitizerCoverage::InjectTraceForDiv(
958 Function &, ArrayRef<BinaryOperator *> DivTraceTargets) {
959 for (auto *BO : DivTraceTargets) {
960 InstrumentationIRBuilder IRB(BO);
961 Value *A1 = BO->getOperand(i_nocapture: 1);
962 if (isa<ConstantInt>(Val: A1))
963 continue;
964 if (!A1->getType()->isIntegerTy())
965 continue;
966 uint64_t TypeSize = DL->getTypeStoreSizeInBits(Ty: A1->getType());
967 int CallbackIdx = TypeSize == 32 ? 0 : TypeSize == 64 ? 1 : -1;
968 if (CallbackIdx < 0)
969 continue;
970 auto Ty = Type::getIntNTy(C&: *C, N: TypeSize);
971 IRB.CreateCall(Callee: SanCovTraceDivFunction[CallbackIdx],
972 Args: {IRB.CreateIntCast(V: A1, DestTy: Ty, isSigned: true)});
973 }
974}
975
976void ModuleSanitizerCoverage::InjectTraceForGep(
977 Function &, ArrayRef<GetElementPtrInst *> GepTraceTargets) {
978 for (auto *GEP : GepTraceTargets) {
979 InstrumentationIRBuilder IRB(GEP);
980 for (Use &Idx : GEP->indices())
981 if (!isa<ConstantInt>(Val: Idx) && Idx->getType()->isIntegerTy())
982 IRB.CreateCall(Callee: SanCovTraceGepFunction,
983 Args: {IRB.CreateIntCast(V: Idx, DestTy: IntptrTy, isSigned: true)});
984 }
985}
986
987void ModuleSanitizerCoverage::InjectTraceForLoadsAndStores(
988 Function &, ArrayRef<LoadInst *> Loads, ArrayRef<StoreInst *> Stores) {
989 auto CallbackIdx = [&](Type *ElementTy) -> int {
990 uint64_t TypeSize = DL->getTypeStoreSizeInBits(Ty: ElementTy);
991 return TypeSize == 8 ? 0
992 : TypeSize == 16 ? 1
993 : TypeSize == 32 ? 2
994 : TypeSize == 64 ? 3
995 : TypeSize == 128 ? 4
996 : -1;
997 };
998 for (auto *LI : Loads) {
999 InstrumentationIRBuilder IRB(LI);
1000 auto Ptr = LI->getPointerOperand();
1001 int Idx = CallbackIdx(LI->getType());
1002 if (Idx < 0)
1003 continue;
1004 IRB.CreateCall(Callee: SanCovLoadFunction[Idx], Args: Ptr);
1005 }
1006 for (auto *SI : Stores) {
1007 InstrumentationIRBuilder IRB(SI);
1008 auto Ptr = SI->getPointerOperand();
1009 int Idx = CallbackIdx(SI->getValueOperand()->getType());
1010 if (Idx < 0)
1011 continue;
1012 IRB.CreateCall(Callee: SanCovStoreFunction[Idx], Args: Ptr);
1013 }
1014}
1015
1016void ModuleSanitizerCoverage::InjectTraceForExits(Function &F) {
1017 EscapeEnumerator EE(F, "sancov_exit");
1018 while (IRBuilder<> *AtExit = EE.Next()) {
1019 InstrumentationIRBuilder::ensureDebugInfo(IRB&: *AtExit, F);
1020 AtExit->CreateCall(Callee: SanCovTracePCExit, Args: {})
1021 ->setTailCallKind(CallInst::TCK_NoTail);
1022 }
1023}
1024
1025void ModuleSanitizerCoverage::InjectTraceForCmp(
1026 Function &F, ArrayRef<Instruction *> CmpTraceTargets,
1027 Value *&FunctionGateCmp) {
1028 for (auto *I : CmpTraceTargets) {
1029 if (ICmpInst *ICMP = dyn_cast<ICmpInst>(Val: I)) {
1030 InstrumentationIRBuilder IRB(ICMP);
1031 Value *A0 = ICMP->getOperand(i_nocapture: 0);
1032 Value *A1 = ICMP->getOperand(i_nocapture: 1);
1033 if (!A0->getType()->isIntegerTy())
1034 continue;
1035 uint64_t TypeSize = DL->getTypeStoreSizeInBits(Ty: A0->getType());
1036 int CallbackIdx = TypeSize == 8 ? 0
1037 : TypeSize == 16 ? 1
1038 : TypeSize == 32 ? 2
1039 : TypeSize == 64 ? 3
1040 : -1;
1041 if (CallbackIdx < 0)
1042 continue;
1043 // __sanitizer_cov_trace_cmp((type_size << 32) | predicate, A0, A1);
1044 auto CallbackFunc = SanCovTraceCmpFunction[CallbackIdx];
1045 bool FirstIsConst = isa<ConstantInt>(Val: A0);
1046 bool SecondIsConst = isa<ConstantInt>(Val: A1);
1047 // If both are const, then we don't need such a comparison.
1048 if (FirstIsConst && SecondIsConst)
1049 continue;
1050 // If only one is const, then make it the first callback argument.
1051 if (FirstIsConst || SecondIsConst) {
1052 CallbackFunc = SanCovTraceConstCmpFunction[CallbackIdx];
1053 if (SecondIsConst)
1054 std::swap(a&: A0, b&: A1);
1055 }
1056
1057 auto Ty = Type::getIntNTy(C&: *C, N: TypeSize);
1058 if (Options.GatedCallbacks) {
1059 auto GateBranch = CreateGateBranch(F, FunctionGateCmp, IP: I);
1060 IRBuilder<> GateIRB(GateBranch);
1061 GateIRB.CreateCall(Callee: CallbackFunc, Args: {GateIRB.CreateIntCast(V: A0, DestTy: Ty, isSigned: true),
1062 GateIRB.CreateIntCast(V: A1, DestTy: Ty, isSigned: true)});
1063 } else {
1064 IRB.CreateCall(Callee: CallbackFunc, Args: {IRB.CreateIntCast(V: A0, DestTy: Ty, isSigned: true),
1065 IRB.CreateIntCast(V: A1, DestTy: Ty, isSigned: true)});
1066 }
1067 }
1068 }
1069}
1070
1071void ModuleSanitizerCoverage::InjectCoverageAtBlock(Function &F, BasicBlock &BB,
1072 size_t Idx,
1073 Value *&FunctionGateCmp,
1074 bool IsLeafFunc) {
1075 BasicBlock::iterator IP = BB.getFirstInsertionPt();
1076 bool IsEntryBB = &BB == &F.getEntryBlock();
1077 DebugLoc EntryLoc;
1078 if (IsEntryBB) {
1079 if (auto SP = F.getSubprogram())
1080 EntryLoc = DILocation::get(Context&: SP->getContext(), Line: SP->getScopeLine(), Column: 0, Scope: SP);
1081 // Keep static allocas and llvm.localescape calls in the entry block. Even
1082 // if we aren't splitting the block, it's nice for allocas to be before
1083 // calls.
1084 IP = PrepareToSplitEntryBlock(BB, IP);
1085 }
1086
1087 InstrumentationIRBuilder IRB(&*IP);
1088 if (EntryLoc)
1089 IRB.SetCurrentDebugLocation(EntryLoc);
1090 if (Options.TracePC || (IsEntryBB && Options.TracePCEntryExit)) {
1091 FunctionCallee Callee = IsEntryBB && Options.TracePCEntryExit
1092 ? SanCovTracePCEntry
1093 : SanCovTracePC;
1094 IRB.CreateCall(Callee)
1095 ->setCannotMerge(); // gets the PC using GET_CALLER_PC.
1096 }
1097 if (Options.TracePCGuard) {
1098 auto GuardPtr = IRB.CreateConstInBoundsGEP2_64(
1099 Ty: FunctionGuardArray->getValueType(), Ptr: FunctionGuardArray, Idx0: 0, Idx1: Idx);
1100 if (Options.GatedCallbacks) {
1101 Instruction *I = &*IP;
1102 auto GateBranch = CreateGateBranch(F, FunctionGateCmp, IP: I);
1103 IRBuilder<> GateIRB(GateBranch);
1104 GateIRB.CreateCall(Callee: SanCovTracePCGuard, Args: GuardPtr)->setCannotMerge();
1105 } else {
1106 IRB.CreateCall(Callee: SanCovTracePCGuard, Args: GuardPtr)->setCannotMerge();
1107 }
1108 }
1109 if (Options.Inline8bitCounters) {
1110 auto CounterPtr = IRB.CreateGEP(
1111 Ty: Function8bitCounterArray->getValueType(), Ptr: Function8bitCounterArray,
1112 IdxList: {ConstantInt::get(Ty: IntptrTy, V: 0), ConstantInt::get(Ty: IntptrTy, V: Idx)});
1113 auto Load = IRB.CreateLoad(Ty: Int8Ty, Ptr: CounterPtr);
1114 auto Inc = IRB.CreateAdd(LHS: Load, RHS: ConstantInt::get(Ty: Int8Ty, V: 1));
1115 auto Store = IRB.CreateStore(Val: Inc, Ptr: CounterPtr);
1116 Load->setNoSanitizeMetadata();
1117 Store->setNoSanitizeMetadata();
1118 }
1119 if (Options.InlineBoolFlag) {
1120 auto FlagPtr = IRB.CreateGEP(
1121 Ty: FunctionBoolArray->getValueType(), Ptr: FunctionBoolArray,
1122 IdxList: {ConstantInt::get(Ty: IntptrTy, V: 0), ConstantInt::get(Ty: IntptrTy, V: Idx)});
1123 auto Load = IRB.CreateLoad(Ty: Int1Ty, Ptr: FlagPtr);
1124 auto ThenTerm = SplitBlockAndInsertIfThen(
1125 Cond: IRB.CreateIsNull(Arg: Load), SplitBefore: &*IP, Unreachable: false,
1126 BranchWeights: MDBuilder(IRB.getContext()).createUnlikelyBranchWeights());
1127 InstrumentationIRBuilder ThenIRB(ThenTerm);
1128 auto Store = ThenIRB.CreateStore(Val: ConstantInt::getTrue(Ty: Int1Ty), Ptr: FlagPtr);
1129 if (EntryLoc)
1130 Store->setDebugLoc(EntryLoc);
1131 Load->setNoSanitizeMetadata();
1132 Store->setNoSanitizeMetadata();
1133 }
1134 if (Options.StackDepth && IsEntryBB && !IsLeafFunc) {
1135 Module *M = F.getParent();
1136 const DataLayout &DL = M->getDataLayout();
1137
1138 if (Options.StackDepthCallbackMin) {
1139 // In callback mode, only add call when stack depth reaches minimum.
1140 int EstimatedStackSize = 0;
1141 // If dynamic alloca found, always add call.
1142 bool HasDynamicAlloc = false;
1143 // Find an insertion point after last "alloca".
1144 llvm::Instruction *InsertBefore = nullptr;
1145
1146 // Examine all allocas in the basic block. since we're too early
1147 // to have results from Intrinsic::frameaddress, we have to manually
1148 // estimate the stack size.
1149 for (auto &I : BB) {
1150 if (auto *AI = dyn_cast<AllocaInst>(Val: &I)) {
1151 // Move potential insertion point past the "alloca".
1152 InsertBefore = AI->getNextNode();
1153
1154 // Make an estimate on the stack usage.
1155 if (auto AllocaSize = AI->getAllocationSize(DL)) {
1156 if (AllocaSize->isFixed())
1157 EstimatedStackSize += AllocaSize->getFixedValue();
1158 else
1159 HasDynamicAlloc = true;
1160 } else {
1161 HasDynamicAlloc = true;
1162 }
1163 }
1164 }
1165
1166 if (HasDynamicAlloc ||
1167 EstimatedStackSize >= Options.StackDepthCallbackMin) {
1168 if (InsertBefore)
1169 IRB.SetInsertPoint(InsertBefore);
1170 auto Call = IRB.CreateCall(Callee: SanCovStackDepthCallback);
1171 if (EntryLoc)
1172 Call->setDebugLoc(EntryLoc);
1173 Call->setCannotMerge();
1174 }
1175 } else {
1176 // Check stack depth. If it's the deepest so far, record it.
1177 auto FrameAddrPtr = IRB.CreateIntrinsic(
1178 ID: Intrinsic::frameaddress, Types: IRB.getPtrTy(AddrSpace: DL.getAllocaAddrSpace()),
1179 Args: {Constant::getNullValue(Ty: Int32Ty)});
1180 auto FrameAddrInt = IRB.CreatePtrToInt(V: FrameAddrPtr, DestTy: IntptrTy);
1181 auto LowestStack = IRB.CreateLoad(Ty: IntptrTy, Ptr: SanCovLowestStack);
1182 auto IsStackLower = IRB.CreateICmpULT(LHS: FrameAddrInt, RHS: LowestStack);
1183 auto ThenTerm = SplitBlockAndInsertIfThen(
1184 Cond: IsStackLower, SplitBefore: &*IP, Unreachable: false,
1185 BranchWeights: MDBuilder(IRB.getContext()).createUnlikelyBranchWeights());
1186 InstrumentationIRBuilder ThenIRB(ThenTerm);
1187 auto Store = ThenIRB.CreateStore(Val: FrameAddrInt, Ptr: SanCovLowestStack);
1188 if (EntryLoc)
1189 Store->setDebugLoc(EntryLoc);
1190 LowestStack->setNoSanitizeMetadata();
1191 Store->setNoSanitizeMetadata();
1192 }
1193 }
1194}
1195
1196std::string
1197ModuleSanitizerCoverage::getSectionName(const std::string &Section) const {
1198 if (TargetTriple.isOSBinFormatCOFF()) {
1199 if (Section == SanCovCountersSectionName)
1200 return ".SCOV$CM";
1201 if (Section == SanCovBoolFlagSectionName)
1202 return ".SCOV$BM";
1203 if (Section == SanCovPCsSectionName)
1204 return ".SCOVP$M";
1205 return ".SCOV$GM"; // For SanCovGuardsSectionName.
1206 }
1207 if (TargetTriple.isOSBinFormatMachO())
1208 return "__DATA,__" + Section;
1209 return "__" + Section;
1210}
1211
1212std::string
1213ModuleSanitizerCoverage::getSectionStart(const std::string &Section) const {
1214 if (TargetTriple.isOSBinFormatMachO())
1215 return "\1section$start$__DATA$__" + Section;
1216 return "__start___" + Section;
1217}
1218
1219std::string
1220ModuleSanitizerCoverage::getSectionEnd(const std::string &Section) const {
1221 if (TargetTriple.isOSBinFormatMachO())
1222 return "\1section$end$__DATA$__" + Section;
1223 return "__stop___" + Section;
1224}
1225
1226void ModuleSanitizerCoverage::createFunctionControlFlow(Function &F) {
1227 SmallVector<Constant *, 32> CFs;
1228 IRBuilder<> IRB(&*F.getEntryBlock().getFirstInsertionPt());
1229
1230 for (auto &BB : F) {
1231 // blockaddress can not be used on function's entry block.
1232 if (&BB == &F.getEntryBlock())
1233 CFs.push_back(Elt: (Constant *)IRB.CreatePointerCast(V: &F, DestTy: PtrTy));
1234 else
1235 CFs.push_back(
1236 Elt: (Constant *)IRB.CreatePointerCast(V: BlockAddress::get(BB: &BB), DestTy: PtrTy));
1237
1238 for (auto SuccBB : successors(BB: &BB)) {
1239 assert(SuccBB != &F.getEntryBlock());
1240 CFs.push_back(
1241 Elt: (Constant *)IRB.CreatePointerCast(V: BlockAddress::get(BB: SuccBB), DestTy: PtrTy));
1242 }
1243
1244 CFs.push_back(Elt: (Constant *)Constant::getNullValue(Ty: PtrTy));
1245
1246 for (auto &Inst : BB) {
1247 if (CallBase *CB = dyn_cast<CallBase>(Val: &Inst)) {
1248 if (CB->isIndirectCall()) {
1249 // TODO(navidem): handle indirect calls, for now mark its existence.
1250 CFs.push_back(Elt: (Constant *)IRB.CreateIntToPtr(
1251 V: ConstantInt::getAllOnesValue(Ty: IntptrTy), DestTy: PtrTy));
1252 } else {
1253 auto CalledF = CB->getCalledFunction();
1254 if (CalledF && !CalledF->isIntrinsic())
1255 CFs.push_back(Elt: (Constant *)IRB.CreatePointerCast(V: CalledF, DestTy: PtrTy));
1256 }
1257 }
1258 }
1259
1260 CFs.push_back(Elt: (Constant *)Constant::getNullValue(Ty: PtrTy));
1261 }
1262
1263 FunctionCFsArray = CreateFunctionLocalArrayInSection(NumElements: CFs.size(), F, Ty: PtrTy,
1264 Section: SanCovCFsSectionName);
1265 FunctionCFsArray->setInitializer(
1266 ConstantArray::get(T: ArrayType::get(ElementType: PtrTy, NumElements: CFs.size()), V: CFs));
1267 FunctionCFsArray->setConstant(true);
1268}
1269