1//===-- SanitizerCoverage.cpp - coverage instrumentation for sanitizers ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Coverage instrumentation done on LLVM IR level, works with Sanitizers.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/Transforms/Instrumentation/SanitizerCoverage.h"
14#include "llvm/ADT/ArrayRef.h"
15#include "llvm/ADT/SmallVector.h"
16#include "llvm/Analysis/GlobalsModRef.h"
17#include "llvm/Analysis/PostDominators.h"
18#include "llvm/IR/Constant.h"
19#include "llvm/IR/Constants.h"
20#include "llvm/IR/DataLayout.h"
21#include "llvm/IR/Dominators.h"
22#include "llvm/IR/EHPersonalities.h"
23#include "llvm/IR/Function.h"
24#include "llvm/IR/GlobalVariable.h"
25#include "llvm/IR/IRBuilder.h"
26#include "llvm/IR/IntrinsicInst.h"
27#include "llvm/IR/Intrinsics.h"
28#include "llvm/IR/LLVMContext.h"
29#include "llvm/IR/MDBuilder.h"
30#include "llvm/IR/Module.h"
31#include "llvm/IR/Type.h"
32#include "llvm/IR/ValueSymbolTable.h"
33#include "llvm/Support/CommandLine.h"
34#include "llvm/Support/SpecialCaseList.h"
35#include "llvm/Support/VirtualFileSystem.h"
36#include "llvm/TargetParser/Triple.h"
37#include "llvm/Transforms/Utils/BasicBlockUtils.h"
38#include "llvm/Transforms/Utils/ModuleUtils.h"
39
40using namespace llvm;
41
42#define DEBUG_TYPE "sancov"
43
44const char SanCovTracePCIndirName[] = "__sanitizer_cov_trace_pc_indir";
45const char SanCovTracePCName[] = "__sanitizer_cov_trace_pc";
46const char SanCovTraceCmp1[] = "__sanitizer_cov_trace_cmp1";
47const char SanCovTraceCmp2[] = "__sanitizer_cov_trace_cmp2";
48const char SanCovTraceCmp4[] = "__sanitizer_cov_trace_cmp4";
49const char SanCovTraceCmp8[] = "__sanitizer_cov_trace_cmp8";
50const char SanCovTraceConstCmp1[] = "__sanitizer_cov_trace_const_cmp1";
51const char SanCovTraceConstCmp2[] = "__sanitizer_cov_trace_const_cmp2";
52const char SanCovTraceConstCmp4[] = "__sanitizer_cov_trace_const_cmp4";
53const char SanCovTraceConstCmp8[] = "__sanitizer_cov_trace_const_cmp8";
54const char SanCovLoad1[] = "__sanitizer_cov_load1";
55const char SanCovLoad2[] = "__sanitizer_cov_load2";
56const char SanCovLoad4[] = "__sanitizer_cov_load4";
57const char SanCovLoad8[] = "__sanitizer_cov_load8";
58const char SanCovLoad16[] = "__sanitizer_cov_load16";
59const char SanCovStore1[] = "__sanitizer_cov_store1";
60const char SanCovStore2[] = "__sanitizer_cov_store2";
61const char SanCovStore4[] = "__sanitizer_cov_store4";
62const char SanCovStore8[] = "__sanitizer_cov_store8";
63const char SanCovStore16[] = "__sanitizer_cov_store16";
64const char SanCovTraceDiv4[] = "__sanitizer_cov_trace_div4";
65const char SanCovTraceDiv8[] = "__sanitizer_cov_trace_div8";
66const char SanCovTraceGep[] = "__sanitizer_cov_trace_gep";
67const char SanCovTraceSwitchName[] = "__sanitizer_cov_trace_switch";
68const char SanCovModuleCtorTracePcGuardName[] =
69 "sancov.module_ctor_trace_pc_guard";
70const char SanCovModuleCtor8bitCountersName[] =
71 "sancov.module_ctor_8bit_counters";
72const char SanCovModuleCtorBoolFlagName[] = "sancov.module_ctor_bool_flag";
73static const uint64_t SanCtorAndDtorPriority = 2;
74
75const char SanCovTracePCGuardName[] = "__sanitizer_cov_trace_pc_guard";
76const char SanCovTracePCGuardInitName[] = "__sanitizer_cov_trace_pc_guard_init";
77const char SanCov8bitCountersInitName[] = "__sanitizer_cov_8bit_counters_init";
78const char SanCovBoolFlagInitName[] = "__sanitizer_cov_bool_flag_init";
79const char SanCovPCsInitName[] = "__sanitizer_cov_pcs_init";
80const char SanCovCFsInitName[] = "__sanitizer_cov_cfs_init";
81
82const char SanCovGuardsSectionName[] = "sancov_guards";
83const char SanCovCountersSectionName[] = "sancov_cntrs";
84const char SanCovBoolFlagSectionName[] = "sancov_bools";
85const char SanCovPCsSectionName[] = "sancov_pcs";
86const char SanCovCFsSectionName[] = "sancov_cfs";
87const char SanCovCallbackGateSectionName[] = "sancov_gate";
88
89const char SanCovStackDepthCallbackName[] = "__sanitizer_cov_stack_depth";
90const char SanCovLowestStackName[] = "__sancov_lowest_stack";
91const char SanCovCallbackGateName[] = "__sancov_should_track";
92
93static cl::opt<int> ClCoverageLevel(
94 "sanitizer-coverage-level",
95 cl::desc("Sanitizer Coverage. 0: none, 1: entry block, 2: all blocks, "
96 "3: all blocks and critical edges"),
97 cl::Hidden);
98
99static cl::opt<bool> ClTracePC("sanitizer-coverage-trace-pc",
100 cl::desc("Experimental pc tracing"), cl::Hidden);
101
102static cl::opt<bool> ClTracePCGuard("sanitizer-coverage-trace-pc-guard",
103 cl::desc("pc tracing with a guard"),
104 cl::Hidden);
105
106// If true, we create a global variable that contains PCs of all instrumented
107// BBs, put this global into a named section, and pass this section's bounds
108// to __sanitizer_cov_pcs_init.
109// This way the coverage instrumentation does not need to acquire the PCs
110// at run-time. Works with trace-pc-guard, inline-8bit-counters, and
111// inline-bool-flag.
112static cl::opt<bool> ClCreatePCTable("sanitizer-coverage-pc-table",
113 cl::desc("create a static PC table"),
114 cl::Hidden);
115
116static cl::opt<bool>
117 ClInline8bitCounters("sanitizer-coverage-inline-8bit-counters",
118 cl::desc("increments 8-bit counter for every edge"),
119 cl::Hidden);
120
121static cl::opt<bool>
122 ClSancovDropCtors("sanitizer-coverage-drop-ctors",
123 cl::desc("do not emit module ctors for global counters"),
124 cl::Hidden);
125
126static cl::opt<bool>
127 ClInlineBoolFlag("sanitizer-coverage-inline-bool-flag",
128 cl::desc("sets a boolean flag for every edge"),
129 cl::Hidden);
130
131static cl::opt<bool>
132 ClCMPTracing("sanitizer-coverage-trace-compares",
133 cl::desc("Tracing of CMP and similar instructions"),
134 cl::Hidden);
135
136static cl::opt<bool> ClDIVTracing("sanitizer-coverage-trace-divs",
137 cl::desc("Tracing of DIV instructions"),
138 cl::Hidden);
139
140static cl::opt<bool> ClLoadTracing("sanitizer-coverage-trace-loads",
141 cl::desc("Tracing of load instructions"),
142 cl::Hidden);
143
144static cl::opt<bool> ClStoreTracing("sanitizer-coverage-trace-stores",
145 cl::desc("Tracing of store instructions"),
146 cl::Hidden);
147
148static cl::opt<bool> ClGEPTracing("sanitizer-coverage-trace-geps",
149 cl::desc("Tracing of GEP instructions"),
150 cl::Hidden);
151
152static cl::opt<bool>
153 ClPruneBlocks("sanitizer-coverage-prune-blocks",
154 cl::desc("Reduce the number of instrumented blocks"),
155 cl::Hidden, cl::init(Val: true));
156
157static cl::opt<bool> ClStackDepth("sanitizer-coverage-stack-depth",
158 cl::desc("max stack depth tracing"),
159 cl::Hidden);
160
161static cl::opt<int> ClStackDepthCallbackMin(
162 "sanitizer-coverage-stack-depth-callback-min",
163 cl::desc("max stack depth tracing should use callback and only when "
164 "stack depth more than specified"),
165 cl::Hidden);
166
167static cl::opt<bool>
168 ClCollectCF("sanitizer-coverage-control-flow",
169 cl::desc("collect control flow for each function"), cl::Hidden);
170
171static cl::opt<bool> ClGatedCallbacks(
172 "sanitizer-coverage-gated-trace-callbacks",
173 cl::desc("Gate the invocation of the tracing callbacks on a global variable"
174 ". Currently only supported for trace-pc-guard and trace-cmp."),
175 cl::Hidden, cl::init(Val: false));
176
177namespace {
178
179SanitizerCoverageOptions getOptions(int LegacyCoverageLevel) {
180 SanitizerCoverageOptions Res;
181 switch (LegacyCoverageLevel) {
182 case 0:
183 Res.CoverageType = SanitizerCoverageOptions::SCK_None;
184 break;
185 case 1:
186 Res.CoverageType = SanitizerCoverageOptions::SCK_Function;
187 break;
188 case 2:
189 Res.CoverageType = SanitizerCoverageOptions::SCK_BB;
190 break;
191 case 3:
192 Res.CoverageType = SanitizerCoverageOptions::SCK_Edge;
193 break;
194 case 4:
195 Res.CoverageType = SanitizerCoverageOptions::SCK_Edge;
196 Res.IndirectCalls = true;
197 break;
198 }
199 return Res;
200}
201
202SanitizerCoverageOptions OverrideFromCL(SanitizerCoverageOptions Options) {
203 // Sets CoverageType and IndirectCalls.
204 SanitizerCoverageOptions CLOpts = getOptions(LegacyCoverageLevel: ClCoverageLevel);
205 Options.CoverageType = std::max(a: Options.CoverageType, b: CLOpts.CoverageType);
206 Options.IndirectCalls |= CLOpts.IndirectCalls;
207 Options.TraceCmp |= ClCMPTracing;
208 Options.TraceDiv |= ClDIVTracing;
209 Options.TraceGep |= ClGEPTracing;
210 Options.TracePC |= ClTracePC;
211 Options.TracePCGuard |= ClTracePCGuard;
212 Options.Inline8bitCounters |= ClInline8bitCounters;
213 Options.InlineBoolFlag |= ClInlineBoolFlag;
214 Options.PCTable |= ClCreatePCTable;
215 Options.NoPrune |= !ClPruneBlocks;
216 Options.StackDepth |= ClStackDepth;
217 Options.StackDepthCallbackMin = std::max(a: Options.StackDepthCallbackMin,
218 b: ClStackDepthCallbackMin.getValue());
219 Options.TraceLoads |= ClLoadTracing;
220 Options.TraceStores |= ClStoreTracing;
221 Options.GatedCallbacks |= ClGatedCallbacks;
222 if (!Options.TracePCGuard && !Options.TracePC &&
223 !Options.Inline8bitCounters && !Options.StackDepth &&
224 !Options.InlineBoolFlag && !Options.TraceLoads && !Options.TraceStores)
225 Options.TracePCGuard = true; // TracePCGuard is default.
226 Options.CollectControlFlow |= ClCollectCF;
227 return Options;
228}
229
230class ModuleSanitizerCoverage {
231public:
232 using DomTreeCallback = function_ref<const DominatorTree &(Function &F)>;
233 using PostDomTreeCallback =
234 function_ref<const PostDominatorTree &(Function &F)>;
235
236 ModuleSanitizerCoverage(Module &M, DomTreeCallback DTCallback,
237 PostDomTreeCallback PDTCallback,
238 const SanitizerCoverageOptions &Options,
239 const SpecialCaseList *Allowlist,
240 const SpecialCaseList *Blocklist)
241 : M(M), DTCallback(DTCallback), PDTCallback(PDTCallback),
242 Options(Options), Allowlist(Allowlist), Blocklist(Blocklist) {}
243
244 bool instrumentModule();
245
246private:
247 void createFunctionControlFlow(Function &F);
248 void instrumentFunction(Function &F);
249 void InjectCoverageForIndirectCalls(Function &F,
250 ArrayRef<Instruction *> IndirCalls);
251 void InjectTraceForCmp(Function &F, ArrayRef<Instruction *> CmpTraceTargets,
252 Value *&FunctionGateCmp);
253 void InjectTraceForDiv(Function &F,
254 ArrayRef<BinaryOperator *> DivTraceTargets);
255 void InjectTraceForGep(Function &F,
256 ArrayRef<GetElementPtrInst *> GepTraceTargets);
257 void InjectTraceForLoadsAndStores(Function &F, ArrayRef<LoadInst *> Loads,
258 ArrayRef<StoreInst *> Stores);
259 void InjectTraceForSwitch(Function &F,
260 ArrayRef<Instruction *> SwitchTraceTargets,
261 Value *&FunctionGateCmp);
262 bool InjectCoverage(Function &F, ArrayRef<BasicBlock *> AllBlocks,
263 Value *&FunctionGateCmp, bool IsLeafFunc);
264 GlobalVariable *CreateFunctionLocalArrayInSection(size_t NumElements,
265 Function &F, Type *Ty,
266 const char *Section);
267 GlobalVariable *CreatePCArray(Function &F, ArrayRef<BasicBlock *> AllBlocks);
268 void CreateFunctionLocalArrays(Function &F, ArrayRef<BasicBlock *> AllBlocks);
269 Instruction *CreateGateBranch(Function &F, Value *&FunctionGateCmp,
270 Instruction *I);
271 Value *CreateFunctionLocalGateCmp(IRBuilder<> &IRB);
272 void InjectCoverageAtBlock(Function &F, BasicBlock &BB, size_t Idx,
273 Value *&FunctionGateCmp, bool IsLeafFunc);
274 Function *CreateInitCallsForSections(Module &M, const char *CtorName,
275 const char *InitFunctionName, Type *Ty,
276 const char *Section);
277 std::pair<Value *, Value *> CreateSecStartEnd(Module &M, const char *Section,
278 Type *Ty);
279
280 std::string getSectionName(const std::string &Section) const;
281 std::string getSectionStart(const std::string &Section) const;
282 std::string getSectionEnd(const std::string &Section) const;
283
284 Module &M;
285 DomTreeCallback DTCallback;
286 PostDomTreeCallback PDTCallback;
287
288 FunctionCallee SanCovStackDepthCallback;
289 FunctionCallee SanCovTracePCIndir;
290 FunctionCallee SanCovTracePC, SanCovTracePCGuard;
291 std::array<FunctionCallee, 4> SanCovTraceCmpFunction;
292 std::array<FunctionCallee, 4> SanCovTraceConstCmpFunction;
293 std::array<FunctionCallee, 5> SanCovLoadFunction;
294 std::array<FunctionCallee, 5> SanCovStoreFunction;
295 std::array<FunctionCallee, 2> SanCovTraceDivFunction;
296 FunctionCallee SanCovTraceGepFunction;
297 FunctionCallee SanCovTraceSwitchFunction;
298 GlobalVariable *SanCovLowestStack;
299 GlobalVariable *SanCovCallbackGate;
300 Type *PtrTy, *IntptrTy, *Int64Ty, *Int32Ty, *Int16Ty, *Int8Ty, *Int1Ty;
301 Module *CurModule;
302 Triple TargetTriple;
303 LLVMContext *C;
304 const DataLayout *DL;
305
306 GlobalVariable *FunctionGuardArray; // for trace-pc-guard.
307 GlobalVariable *Function8bitCounterArray; // for inline-8bit-counters.
308 GlobalVariable *FunctionBoolArray; // for inline-bool-flag.
309 GlobalVariable *FunctionPCsArray; // for pc-table.
310 GlobalVariable *FunctionCFsArray; // for control flow table
311 SmallVector<GlobalValue *, 20> GlobalsToAppendToUsed;
312 SmallVector<GlobalValue *, 20> GlobalsToAppendToCompilerUsed;
313
314 SanitizerCoverageOptions Options;
315
316 const SpecialCaseList *Allowlist;
317 const SpecialCaseList *Blocklist;
318};
319} // namespace
320
321SanitizerCoveragePass::SanitizerCoveragePass(
322 SanitizerCoverageOptions Options, IntrusiveRefCntPtr<vfs::FileSystem> VFS,
323 const std::vector<std::string> &AllowlistFiles,
324 const std::vector<std::string> &BlocklistFiles)
325 : Options(std::move(Options)),
326 VFS(VFS ? std::move(VFS) : vfs::getRealFileSystem()) {
327 if (AllowlistFiles.size() > 0)
328 Allowlist = SpecialCaseList::createOrDie(Paths: AllowlistFiles, FS&: *this->VFS);
329 if (BlocklistFiles.size() > 0)
330 Blocklist = SpecialCaseList::createOrDie(Paths: BlocklistFiles, FS&: *this->VFS);
331}
332
333PreservedAnalyses SanitizerCoveragePass::run(Module &M,
334 ModuleAnalysisManager &MAM) {
335 auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
336 auto DTCallback = [&FAM](Function &F) -> const DominatorTree & {
337 return FAM.getResult<DominatorTreeAnalysis>(IR&: F);
338 };
339 auto PDTCallback = [&FAM](Function &F) -> const PostDominatorTree & {
340 return FAM.getResult<PostDominatorTreeAnalysis>(IR&: F);
341 };
342 ModuleSanitizerCoverage ModuleSancov(M, DTCallback, PDTCallback,
343 OverrideFromCL(Options), Allowlist.get(),
344 Blocklist.get());
345 if (!ModuleSancov.instrumentModule())
346 return PreservedAnalyses::all();
347
348 PreservedAnalyses PA = PreservedAnalyses::none();
349 // GlobalsAA is considered stateless and does not get invalidated unless
350 // explicitly invalidated; PreservedAnalyses::none() is not enough. Sanitizers
351 // make changes that require GlobalsAA to be invalidated.
352 PA.abandon<GlobalsAA>();
353 return PA;
354}
355
356std::pair<Value *, Value *>
357ModuleSanitizerCoverage::CreateSecStartEnd(Module &M, const char *Section,
358 Type *Ty) {
359 // Use ExternalWeak so that if all sections are discarded due to section
360 // garbage collection, the linker will not report undefined symbol errors.
361 // Windows defines the start/stop symbols in compiler-rt so no need for
362 // ExternalWeak.
363 GlobalValue::LinkageTypes Linkage = TargetTriple.isOSBinFormatCOFF()
364 ? GlobalVariable::ExternalLinkage
365 : GlobalVariable::ExternalWeakLinkage;
366 GlobalVariable *SecStart = new GlobalVariable(M, Ty, false, Linkage, nullptr,
367 getSectionStart(Section));
368 SecStart->setVisibility(GlobalValue::HiddenVisibility);
369 GlobalVariable *SecEnd = new GlobalVariable(M, Ty, false, Linkage, nullptr,
370 getSectionEnd(Section));
371 SecEnd->setVisibility(GlobalValue::HiddenVisibility);
372 IRBuilder<> IRB(M.getContext());
373 if (!TargetTriple.isOSBinFormatCOFF())
374 return std::make_pair(x&: SecStart, y&: SecEnd);
375
376 // Account for the fact that on windows-msvc __start_* symbols actually
377 // point to a uint64_t before the start of the array.
378 auto GEP =
379 IRB.CreatePtrAdd(Ptr: SecStart, Offset: ConstantInt::get(Ty: IntptrTy, V: sizeof(uint64_t)));
380 return std::make_pair(x&: GEP, y&: SecEnd);
381}
382
383Function *ModuleSanitizerCoverage::CreateInitCallsForSections(
384 Module &M, const char *CtorName, const char *InitFunctionName, Type *Ty,
385 const char *Section) {
386 if (ClSancovDropCtors)
387 return nullptr;
388 auto SecStartEnd = CreateSecStartEnd(M, Section, Ty);
389 auto SecStart = SecStartEnd.first;
390 auto SecEnd = SecStartEnd.second;
391 Function *CtorFunc;
392 std::tie(args&: CtorFunc, args: std::ignore) = createSanitizerCtorAndInitFunctions(
393 M, CtorName, InitName: InitFunctionName, InitArgTypes: {PtrTy, PtrTy}, InitArgs: {SecStart, SecEnd});
394 assert(CtorFunc->getName() == CtorName);
395
396 if (TargetTriple.supportsCOMDAT()) {
397 // Use comdat to dedup CtorFunc.
398 CtorFunc->setComdat(M.getOrInsertComdat(Name: CtorName));
399 appendToGlobalCtors(M, F: CtorFunc, Priority: SanCtorAndDtorPriority, Data: CtorFunc);
400 } else {
401 appendToGlobalCtors(M, F: CtorFunc, Priority: SanCtorAndDtorPriority);
402 }
403
404 if (TargetTriple.isOSBinFormatCOFF()) {
405 // In COFF files, if the contructors are set as COMDAT (they are because
406 // COFF supports COMDAT) and the linker flag /OPT:REF (strip unreferenced
407 // functions and data) is used, the constructors get stripped. To prevent
408 // this, give the constructors weak ODR linkage and ensure the linker knows
409 // to include the sancov constructor. This way the linker can deduplicate
410 // the constructors but always leave one copy.
411 CtorFunc->setLinkage(GlobalValue::WeakODRLinkage);
412 }
413 return CtorFunc;
414}
415
416bool ModuleSanitizerCoverage::instrumentModule() {
417 if (Options.CoverageType == SanitizerCoverageOptions::SCK_None)
418 return false;
419 if (Allowlist &&
420 !Allowlist->inSection(Section: "coverage", Prefix: "src", Query: M.getSourceFileName()))
421 return false;
422 if (Blocklist &&
423 Blocklist->inSection(Section: "coverage", Prefix: "src", Query: M.getSourceFileName()))
424 return false;
425 C = &(M.getContext());
426 DL = &M.getDataLayout();
427 CurModule = &M;
428 TargetTriple = M.getTargetTriple();
429 FunctionGuardArray = nullptr;
430 Function8bitCounterArray = nullptr;
431 FunctionBoolArray = nullptr;
432 FunctionPCsArray = nullptr;
433 FunctionCFsArray = nullptr;
434 IntptrTy = Type::getIntNTy(C&: *C, N: DL->getPointerSizeInBits());
435 PtrTy = PointerType::getUnqual(C&: *C);
436 Type *VoidTy = Type::getVoidTy(C&: *C);
437 IRBuilder<> IRB(*C);
438 Int64Ty = IRB.getInt64Ty();
439 Int32Ty = IRB.getInt32Ty();
440 Int16Ty = IRB.getInt16Ty();
441 Int8Ty = IRB.getInt8Ty();
442 Int1Ty = IRB.getInt1Ty();
443
444 SanCovTracePCIndir =
445 M.getOrInsertFunction(Name: SanCovTracePCIndirName, RetTy: VoidTy, Args: IntptrTy);
446 // Make sure smaller parameters are zero-extended to i64 if required by the
447 // target ABI.
448 AttributeList SanCovTraceCmpZeroExtAL;
449 SanCovTraceCmpZeroExtAL =
450 SanCovTraceCmpZeroExtAL.addParamAttribute(C&: *C, ArgNo: 0, Kind: Attribute::ZExt);
451 SanCovTraceCmpZeroExtAL =
452 SanCovTraceCmpZeroExtAL.addParamAttribute(C&: *C, ArgNo: 1, Kind: Attribute::ZExt);
453
454 SanCovTraceCmpFunction[0] =
455 M.getOrInsertFunction(Name: SanCovTraceCmp1, AttributeList: SanCovTraceCmpZeroExtAL, RetTy: VoidTy,
456 Args: IRB.getInt8Ty(), Args: IRB.getInt8Ty());
457 SanCovTraceCmpFunction[1] =
458 M.getOrInsertFunction(Name: SanCovTraceCmp2, AttributeList: SanCovTraceCmpZeroExtAL, RetTy: VoidTy,
459 Args: IRB.getInt16Ty(), Args: IRB.getInt16Ty());
460 SanCovTraceCmpFunction[2] =
461 M.getOrInsertFunction(Name: SanCovTraceCmp4, AttributeList: SanCovTraceCmpZeroExtAL, RetTy: VoidTy,
462 Args: IRB.getInt32Ty(), Args: IRB.getInt32Ty());
463 SanCovTraceCmpFunction[3] =
464 M.getOrInsertFunction(Name: SanCovTraceCmp8, RetTy: VoidTy, Args: Int64Ty, Args: Int64Ty);
465
466 SanCovTraceConstCmpFunction[0] = M.getOrInsertFunction(
467 Name: SanCovTraceConstCmp1, AttributeList: SanCovTraceCmpZeroExtAL, RetTy: VoidTy, Args: Int8Ty, Args: Int8Ty);
468 SanCovTraceConstCmpFunction[1] = M.getOrInsertFunction(
469 Name: SanCovTraceConstCmp2, AttributeList: SanCovTraceCmpZeroExtAL, RetTy: VoidTy, Args: Int16Ty, Args: Int16Ty);
470 SanCovTraceConstCmpFunction[2] = M.getOrInsertFunction(
471 Name: SanCovTraceConstCmp4, AttributeList: SanCovTraceCmpZeroExtAL, RetTy: VoidTy, Args: Int32Ty, Args: Int32Ty);
472 SanCovTraceConstCmpFunction[3] =
473 M.getOrInsertFunction(Name: SanCovTraceConstCmp8, RetTy: VoidTy, Args: Int64Ty, Args: Int64Ty);
474
475 // Loads.
476 SanCovLoadFunction[0] = M.getOrInsertFunction(Name: SanCovLoad1, RetTy: VoidTy, Args: PtrTy);
477 SanCovLoadFunction[1] = M.getOrInsertFunction(Name: SanCovLoad2, RetTy: VoidTy, Args: PtrTy);
478 SanCovLoadFunction[2] = M.getOrInsertFunction(Name: SanCovLoad4, RetTy: VoidTy, Args: PtrTy);
479 SanCovLoadFunction[3] = M.getOrInsertFunction(Name: SanCovLoad8, RetTy: VoidTy, Args: PtrTy);
480 SanCovLoadFunction[4] = M.getOrInsertFunction(Name: SanCovLoad16, RetTy: VoidTy, Args: PtrTy);
481 // Stores.
482 SanCovStoreFunction[0] = M.getOrInsertFunction(Name: SanCovStore1, RetTy: VoidTy, Args: PtrTy);
483 SanCovStoreFunction[1] = M.getOrInsertFunction(Name: SanCovStore2, RetTy: VoidTy, Args: PtrTy);
484 SanCovStoreFunction[2] = M.getOrInsertFunction(Name: SanCovStore4, RetTy: VoidTy, Args: PtrTy);
485 SanCovStoreFunction[3] = M.getOrInsertFunction(Name: SanCovStore8, RetTy: VoidTy, Args: PtrTy);
486 SanCovStoreFunction[4] = M.getOrInsertFunction(Name: SanCovStore16, RetTy: VoidTy, Args: PtrTy);
487
488 {
489 AttributeList AL;
490 AL = AL.addParamAttribute(C&: *C, ArgNo: 0, Kind: Attribute::ZExt);
491 SanCovTraceDivFunction[0] =
492 M.getOrInsertFunction(Name: SanCovTraceDiv4, AttributeList: AL, RetTy: VoidTy, Args: IRB.getInt32Ty());
493 }
494 SanCovTraceDivFunction[1] =
495 M.getOrInsertFunction(Name: SanCovTraceDiv8, RetTy: VoidTy, Args: Int64Ty);
496 SanCovTraceGepFunction =
497 M.getOrInsertFunction(Name: SanCovTraceGep, RetTy: VoidTy, Args: IntptrTy);
498 SanCovTraceSwitchFunction =
499 M.getOrInsertFunction(Name: SanCovTraceSwitchName, RetTy: VoidTy, Args: Int64Ty, Args: PtrTy);
500
501 SanCovLowestStack = M.getOrInsertGlobal(Name: SanCovLowestStackName, Ty: IntptrTy);
502 if (SanCovLowestStack->getValueType() != IntptrTy) {
503 C->emitError(ErrorStr: StringRef("'") + SanCovLowestStackName +
504 "' should not be declared by the user");
505 return true;
506 }
507 SanCovLowestStack->setThreadLocalMode(
508 GlobalValue::ThreadLocalMode::InitialExecTLSModel);
509 if (Options.StackDepth && !SanCovLowestStack->isDeclaration())
510 SanCovLowestStack->setInitializer(Constant::getAllOnesValue(Ty: IntptrTy));
511
512 if (Options.GatedCallbacks) {
513 if (!Options.TracePCGuard && !Options.TraceCmp) {
514 C->emitError(ErrorStr: StringRef("'") + ClGatedCallbacks.ArgStr +
515 "' is only supported with trace-pc-guard or trace-cmp");
516 return true;
517 }
518
519 SanCovCallbackGate = cast<GlobalVariable>(
520 Val: M.getOrInsertGlobal(Name: SanCovCallbackGateName, Ty: Int64Ty));
521 SanCovCallbackGate->setSection(
522 getSectionName(Section: SanCovCallbackGateSectionName));
523 SanCovCallbackGate->setInitializer(Constant::getNullValue(Ty: Int64Ty));
524 SanCovCallbackGate->setLinkage(GlobalVariable::LinkOnceAnyLinkage);
525 SanCovCallbackGate->setVisibility(GlobalVariable::HiddenVisibility);
526 appendToCompilerUsed(M, Values: SanCovCallbackGate);
527 }
528
529 SanCovTracePC = M.getOrInsertFunction(Name: SanCovTracePCName, RetTy: VoidTy);
530 SanCovTracePCGuard =
531 M.getOrInsertFunction(Name: SanCovTracePCGuardName, RetTy: VoidTy, Args: PtrTy);
532
533 SanCovStackDepthCallback =
534 M.getOrInsertFunction(Name: SanCovStackDepthCallbackName, RetTy: VoidTy);
535
536 for (auto &F : M)
537 instrumentFunction(F);
538
539 Function *Ctor = nullptr;
540
541 if (FunctionGuardArray)
542 Ctor = CreateInitCallsForSections(M, CtorName: SanCovModuleCtorTracePcGuardName,
543 InitFunctionName: SanCovTracePCGuardInitName, Ty: Int32Ty,
544 Section: SanCovGuardsSectionName);
545 if (Function8bitCounterArray)
546 Ctor = CreateInitCallsForSections(M, CtorName: SanCovModuleCtor8bitCountersName,
547 InitFunctionName: SanCov8bitCountersInitName, Ty: Int8Ty,
548 Section: SanCovCountersSectionName);
549 if (FunctionBoolArray) {
550 Ctor = CreateInitCallsForSections(M, CtorName: SanCovModuleCtorBoolFlagName,
551 InitFunctionName: SanCovBoolFlagInitName, Ty: Int1Ty,
552 Section: SanCovBoolFlagSectionName);
553 }
554 if (Ctor && Options.PCTable) {
555 auto SecStartEnd = CreateSecStartEnd(M, Section: SanCovPCsSectionName, Ty: IntptrTy);
556 FunctionCallee InitFunction =
557 declareSanitizerInitFunction(M, InitName: SanCovPCsInitName, InitArgTypes: {PtrTy, PtrTy});
558 IRBuilder<> IRBCtor(Ctor->getEntryBlock().getTerminator());
559 IRBCtor.CreateCall(Callee: InitFunction, Args: {SecStartEnd.first, SecStartEnd.second});
560 }
561
562 if (Ctor && Options.CollectControlFlow) {
563 auto SecStartEnd = CreateSecStartEnd(M, Section: SanCovCFsSectionName, Ty: IntptrTy);
564 FunctionCallee InitFunction =
565 declareSanitizerInitFunction(M, InitName: SanCovCFsInitName, InitArgTypes: {PtrTy, PtrTy});
566 IRBuilder<> IRBCtor(Ctor->getEntryBlock().getTerminator());
567 IRBCtor.CreateCall(Callee: InitFunction, Args: {SecStartEnd.first, SecStartEnd.second});
568 }
569
570 appendToUsed(M, Values: GlobalsToAppendToUsed);
571 appendToCompilerUsed(M, Values: GlobalsToAppendToCompilerUsed);
572 return true;
573}
574
575// True if block has successors and it dominates all of them.
576static bool isFullDominator(const BasicBlock *BB, const DominatorTree &DT) {
577 if (succ_empty(BB))
578 return false;
579
580 return llvm::all_of(Range: successors(BB), P: [&](const BasicBlock *SUCC) {
581 return DT.dominates(A: BB, B: SUCC);
582 });
583}
584
585// True if block has predecessors and it postdominates all of them.
586static bool isFullPostDominator(const BasicBlock *BB,
587 const PostDominatorTree &PDT) {
588 if (pred_empty(BB))
589 return false;
590
591 return llvm::all_of(Range: predecessors(BB), P: [&](const BasicBlock *PRED) {
592 return PDT.dominates(A: BB, B: PRED);
593 });
594}
595
596static bool shouldInstrumentBlock(const Function &F, const BasicBlock *BB,
597 const DominatorTree &DT,
598 const PostDominatorTree &PDT,
599 const SanitizerCoverageOptions &Options) {
600 // Don't insert coverage for blocks containing nothing but unreachable: we
601 // will never call __sanitizer_cov() for them, so counting them in
602 // NumberOfInstrumentedBlocks() might complicate calculation of code coverage
603 // percentage. Also, unreachable instructions frequently have no debug
604 // locations.
605 if (isa<UnreachableInst>(Val: BB->getFirstNonPHIOrDbgOrLifetime()))
606 return false;
607
608 // Don't insert coverage into blocks without a valid insertion point
609 // (catchswitch blocks).
610 if (BB->getFirstInsertionPt() == BB->end())
611 return false;
612
613 if (Options.NoPrune || &F.getEntryBlock() == BB)
614 return true;
615
616 if (Options.CoverageType == SanitizerCoverageOptions::SCK_Function &&
617 &F.getEntryBlock() != BB)
618 return false;
619
620 // Do not instrument full dominators, or full post-dominators with multiple
621 // predecessors.
622 return !isFullDominator(BB, DT) &&
623 !(isFullPostDominator(BB, PDT) && !BB->getSinglePredecessor());
624}
625
626// Returns true iff From->To is a backedge.
627// A twist here is that we treat From->To as a backedge if
628// * To dominates From or
629// * To->UniqueSuccessor dominates From
630static bool IsBackEdge(BasicBlock *From, BasicBlock *To,
631 const DominatorTree &DT) {
632 if (DT.dominates(A: To, B: From))
633 return true;
634 if (auto Next = To->getUniqueSuccessor())
635 if (DT.dominates(A: Next, B: From))
636 return true;
637 return false;
638}
639
640// Prunes uninteresting Cmp instrumentation:
641// * CMP instructions that feed into loop backedge branch.
642//
643// Note that Cmp pruning is controlled by the same flag as the
644// BB pruning.
645static bool IsInterestingCmp(ICmpInst *CMP, const DominatorTree &DT,
646 const SanitizerCoverageOptions &Options) {
647 if (!Options.NoPrune)
648 if (CMP->hasOneUse())
649 if (auto BR = dyn_cast<BranchInst>(Val: CMP->user_back()))
650 for (BasicBlock *B : BR->successors())
651 if (IsBackEdge(From: BR->getParent(), To: B, DT))
652 return false;
653 return true;
654}
655
656void ModuleSanitizerCoverage::instrumentFunction(Function &F) {
657 if (F.empty())
658 return;
659 if (F.getName().contains(Other: ".module_ctor"))
660 return; // Should not instrument sanitizer init functions.
661 if (F.getName().starts_with(Prefix: "__sanitizer_"))
662 return; // Don't instrument __sanitizer_* callbacks.
663 // Don't touch available_externally functions, their actual body is elewhere.
664 if (F.getLinkage() == GlobalValue::AvailableExternallyLinkage)
665 return;
666 // Don't instrument MSVC CRT configuration helpers. They may run before normal
667 // initialization.
668 if (F.getName() == "__local_stdio_printf_options" ||
669 F.getName() == "__local_stdio_scanf_options")
670 return;
671 if (isa<UnreachableInst>(Val: F.getEntryBlock().getTerminator()))
672 return;
673 // Don't instrument functions using SEH for now. Splitting basic blocks like
674 // we do for coverage breaks WinEHPrepare.
675 // FIXME: Remove this when SEH no longer uses landingpad pattern matching.
676 if (F.hasPersonalityFn() &&
677 isAsynchronousEHPersonality(Pers: classifyEHPersonality(Pers: F.getPersonalityFn())))
678 return;
679 if (Allowlist && !Allowlist->inSection(Section: "coverage", Prefix: "fun", Query: F.getName()))
680 return;
681 if (Blocklist && Blocklist->inSection(Section: "coverage", Prefix: "fun", Query: F.getName()))
682 return;
683 // Do not apply any instrumentation for naked functions.
684 if (F.hasFnAttribute(Kind: Attribute::Naked))
685 return;
686 if (F.hasFnAttribute(Kind: Attribute::NoSanitizeCoverage))
687 return;
688 if (F.hasFnAttribute(Kind: Attribute::DisableSanitizerInstrumentation))
689 return;
690 if (Options.CoverageType >= SanitizerCoverageOptions::SCK_Edge) {
691 SplitAllCriticalEdges(
692 F, Options: CriticalEdgeSplittingOptions().setIgnoreUnreachableDests());
693 }
694 SmallVector<Instruction *, 8> IndirCalls;
695 SmallVector<BasicBlock *, 16> BlocksToInstrument;
696 SmallVector<Instruction *, 8> CmpTraceTargets;
697 SmallVector<Instruction *, 8> SwitchTraceTargets;
698 SmallVector<BinaryOperator *, 8> DivTraceTargets;
699 SmallVector<GetElementPtrInst *, 8> GepTraceTargets;
700 SmallVector<LoadInst *, 8> Loads;
701 SmallVector<StoreInst *, 8> Stores;
702
703 const DominatorTree &DT = DTCallback(F);
704 const PostDominatorTree &PDT = PDTCallback(F);
705 bool IsLeafFunc = true;
706
707 for (auto &BB : F) {
708 if (shouldInstrumentBlock(F, BB: &BB, DT, PDT, Options))
709 BlocksToInstrument.push_back(Elt: &BB);
710 for (auto &Inst : BB) {
711 if (Options.IndirectCalls) {
712 CallBase *CB = dyn_cast<CallBase>(Val: &Inst);
713 if (CB && CB->isIndirectCall())
714 IndirCalls.push_back(Elt: &Inst);
715 }
716 if (Options.TraceCmp) {
717 if (ICmpInst *CMP = dyn_cast<ICmpInst>(Val: &Inst))
718 if (IsInterestingCmp(CMP, DT, Options))
719 CmpTraceTargets.push_back(Elt: &Inst);
720 if (isa<SwitchInst>(Val: &Inst))
721 SwitchTraceTargets.push_back(Elt: &Inst);
722 }
723 if (Options.TraceDiv)
724 if (BinaryOperator *BO = dyn_cast<BinaryOperator>(Val: &Inst))
725 if (BO->getOpcode() == Instruction::SDiv ||
726 BO->getOpcode() == Instruction::UDiv)
727 DivTraceTargets.push_back(Elt: BO);
728 if (Options.TraceGep)
729 if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Val: &Inst))
730 GepTraceTargets.push_back(Elt: GEP);
731 if (Options.TraceLoads)
732 if (LoadInst *LI = dyn_cast<LoadInst>(Val: &Inst))
733 Loads.push_back(Elt: LI);
734 if (Options.TraceStores)
735 if (StoreInst *SI = dyn_cast<StoreInst>(Val: &Inst))
736 Stores.push_back(Elt: SI);
737 if (Options.StackDepth)
738 if (isa<InvokeInst>(Val: Inst) ||
739 (isa<CallInst>(Val: Inst) && !isa<IntrinsicInst>(Val: Inst)))
740 IsLeafFunc = false;
741 }
742 }
743
744 if (Options.CollectControlFlow)
745 createFunctionControlFlow(F);
746
747 Value *FunctionGateCmp = nullptr;
748 InjectCoverage(F, AllBlocks: BlocksToInstrument, FunctionGateCmp, IsLeafFunc);
749 InjectCoverageForIndirectCalls(F, IndirCalls);
750 InjectTraceForCmp(F, CmpTraceTargets, FunctionGateCmp);
751 InjectTraceForSwitch(F, SwitchTraceTargets, FunctionGateCmp);
752 InjectTraceForDiv(F, DivTraceTargets);
753 InjectTraceForGep(F, GepTraceTargets);
754 InjectTraceForLoadsAndStores(F, Loads, Stores);
755}
756
757GlobalVariable *ModuleSanitizerCoverage::CreateFunctionLocalArrayInSection(
758 size_t NumElements, Function &F, Type *Ty, const char *Section) {
759 ArrayType *ArrayTy = ArrayType::get(ElementType: Ty, NumElements);
760 auto Array = new GlobalVariable(
761 *CurModule, ArrayTy, false, GlobalVariable::PrivateLinkage,
762 Constant::getNullValue(Ty: ArrayTy), "__sancov_gen_");
763
764 if (TargetTriple.supportsCOMDAT() &&
765 (F.hasComdat() || TargetTriple.isOSBinFormatELF() || !F.isInterposable()))
766 if (auto Comdat = getOrCreateFunctionComdat(F, T&: TargetTriple))
767 Array->setComdat(Comdat);
768 Array->setSection(getSectionName(Section));
769 Array->setAlignment(Align(DL->getTypeStoreSize(Ty).getFixedValue()));
770
771 // sancov_pcs parallels the other metadata section(s). Optimizers (e.g.
772 // GlobalOpt/ConstantMerge) may not discard sancov_pcs and the other
773 // section(s) as a unit, so we conservatively retain all unconditionally in
774 // the compiler.
775 //
776 // With comdat (COFF/ELF), the linker can guarantee the associated sections
777 // will be retained or discarded as a unit, so llvm.compiler.used is
778 // sufficient. Otherwise, conservatively make all of them retained by the
779 // linker.
780 if (Array->hasComdat())
781 GlobalsToAppendToCompilerUsed.push_back(Elt: Array);
782 else
783 GlobalsToAppendToUsed.push_back(Elt: Array);
784
785 return Array;
786}
787
788GlobalVariable *
789ModuleSanitizerCoverage::CreatePCArray(Function &F,
790 ArrayRef<BasicBlock *> AllBlocks) {
791 size_t N = AllBlocks.size();
792 assert(N);
793 SmallVector<Constant *, 32> PCs;
794 IRBuilder<> IRB(&*F.getEntryBlock().getFirstInsertionPt());
795 for (size_t i = 0; i < N; i++) {
796 if (&F.getEntryBlock() == AllBlocks[i]) {
797 PCs.push_back(Elt: (Constant *)IRB.CreatePointerCast(V: &F, DestTy: PtrTy));
798 PCs.push_back(
799 Elt: (Constant *)IRB.CreateIntToPtr(V: ConstantInt::get(Ty: IntptrTy, V: 1), DestTy: PtrTy));
800 } else {
801 PCs.push_back(Elt: (Constant *)IRB.CreatePointerCast(
802 V: BlockAddress::get(BB: AllBlocks[i]), DestTy: PtrTy));
803 PCs.push_back(Elt: Constant::getNullValue(Ty: PtrTy));
804 }
805 }
806 auto *PCArray =
807 CreateFunctionLocalArrayInSection(NumElements: N * 2, F, Ty: PtrTy, Section: SanCovPCsSectionName);
808 PCArray->setInitializer(
809 ConstantArray::get(T: ArrayType::get(ElementType: PtrTy, NumElements: N * 2), V: PCs));
810 PCArray->setConstant(true);
811
812 return PCArray;
813}
814
815void ModuleSanitizerCoverage::CreateFunctionLocalArrays(
816 Function &F, ArrayRef<BasicBlock *> AllBlocks) {
817 if (Options.TracePCGuard)
818 FunctionGuardArray = CreateFunctionLocalArrayInSection(
819 NumElements: AllBlocks.size(), F, Ty: Int32Ty, Section: SanCovGuardsSectionName);
820
821 if (Options.Inline8bitCounters)
822 Function8bitCounterArray = CreateFunctionLocalArrayInSection(
823 NumElements: AllBlocks.size(), F, Ty: Int8Ty, Section: SanCovCountersSectionName);
824 if (Options.InlineBoolFlag)
825 FunctionBoolArray = CreateFunctionLocalArrayInSection(
826 NumElements: AllBlocks.size(), F, Ty: Int1Ty, Section: SanCovBoolFlagSectionName);
827
828 if (Options.PCTable)
829 FunctionPCsArray = CreatePCArray(F, AllBlocks);
830}
831
832Value *ModuleSanitizerCoverage::CreateFunctionLocalGateCmp(IRBuilder<> &IRB) {
833 auto Load = IRB.CreateLoad(Ty: Int64Ty, Ptr: SanCovCallbackGate);
834 Load->setNoSanitizeMetadata();
835 auto Cmp = IRB.CreateIsNotNull(Arg: Load);
836 Cmp->setName("sancov gate cmp");
837 return Cmp;
838}
839
840Instruction *ModuleSanitizerCoverage::CreateGateBranch(Function &F,
841 Value *&FunctionGateCmp,
842 Instruction *IP) {
843 if (!FunctionGateCmp) {
844 // Create this in the entry block
845 BasicBlock &BB = F.getEntryBlock();
846 BasicBlock::iterator IP = BB.getFirstInsertionPt();
847 IP = PrepareToSplitEntryBlock(BB, IP);
848 IRBuilder<> EntryIRB(&*IP);
849 FunctionGateCmp = CreateFunctionLocalGateCmp(IRB&: EntryIRB);
850 }
851 // Set the branch weights in order to minimize the price paid when the
852 // gate is turned off, allowing the default enablement of this
853 // instrumentation with as little of a performance cost as possible
854 auto Weights = MDBuilder(*C).createBranchWeights(TrueWeight: 1, FalseWeight: 100000);
855 return SplitBlockAndInsertIfThen(Cond: FunctionGateCmp, SplitBefore: IP, Unreachable: false, BranchWeights: Weights);
856}
857
858bool ModuleSanitizerCoverage::InjectCoverage(Function &F,
859 ArrayRef<BasicBlock *> AllBlocks,
860 Value *&FunctionGateCmp,
861 bool IsLeafFunc) {
862 if (AllBlocks.empty())
863 return false;
864 CreateFunctionLocalArrays(F, AllBlocks);
865 for (size_t i = 0, N = AllBlocks.size(); i < N; i++)
866 InjectCoverageAtBlock(F, BB&: *AllBlocks[i], Idx: i, FunctionGateCmp, IsLeafFunc);
867 return true;
868}
869
870// On every indirect call we call a run-time function
871// __sanitizer_cov_indir_call* with two parameters:
872// - callee address,
873// - global cache array that contains CacheSize pointers (zero-initialized).
874// The cache is used to speed up recording the caller-callee pairs.
875// The address of the caller is passed implicitly via caller PC.
876// CacheSize is encoded in the name of the run-time function.
877void ModuleSanitizerCoverage::InjectCoverageForIndirectCalls(
878 Function &F, ArrayRef<Instruction *> IndirCalls) {
879 if (IndirCalls.empty())
880 return;
881 assert(Options.TracePC || Options.TracePCGuard ||
882 Options.Inline8bitCounters || Options.InlineBoolFlag);
883 for (auto *I : IndirCalls) {
884 InstrumentationIRBuilder IRB(I);
885 CallBase &CB = cast<CallBase>(Val&: *I);
886 Value *Callee = CB.getCalledOperand();
887 if (isa<InlineAsm>(Val: Callee))
888 continue;
889 IRB.CreateCall(Callee: SanCovTracePCIndir, Args: IRB.CreatePointerCast(V: Callee, DestTy: IntptrTy));
890 }
891}
892
893// For every switch statement we insert a call:
894// __sanitizer_cov_trace_switch(CondValue,
895// {NumCases, ValueSizeInBits, Case0Value, Case1Value, Case2Value, ... })
896
897void ModuleSanitizerCoverage::InjectTraceForSwitch(
898 Function &F, ArrayRef<Instruction *> SwitchTraceTargets,
899 Value *&FunctionGateCmp) {
900 for (auto *I : SwitchTraceTargets) {
901 if (SwitchInst *SI = dyn_cast<SwitchInst>(Val: I)) {
902 InstrumentationIRBuilder IRB(I);
903 SmallVector<Constant *, 16> Initializers;
904 Value *Cond = SI->getCondition();
905 if (Cond->getType()->getScalarSizeInBits() >
906 Int64Ty->getScalarSizeInBits())
907 continue;
908 Initializers.push_back(Elt: ConstantInt::get(Ty: Int64Ty, V: SI->getNumCases()));
909 Initializers.push_back(
910 Elt: ConstantInt::get(Ty: Int64Ty, V: Cond->getType()->getScalarSizeInBits()));
911 if (Cond->getType()->getScalarSizeInBits() <
912 Int64Ty->getScalarSizeInBits())
913 Cond = IRB.CreateIntCast(V: Cond, DestTy: Int64Ty, isSigned: false);
914 for (auto It : SI->cases()) {
915 ConstantInt *C = It.getCaseValue();
916 if (C->getType()->getScalarSizeInBits() < 64)
917 C = ConstantInt::get(Context&: C->getContext(), V: C->getValue().zext(width: 64));
918 Initializers.push_back(Elt: C);
919 }
920 llvm::sort(C: drop_begin(RangeOrContainer&: Initializers, N: 2),
921 Comp: [](const Constant *A, const Constant *B) {
922 return cast<ConstantInt>(Val: A)->getLimitedValue() <
923 cast<ConstantInt>(Val: B)->getLimitedValue();
924 });
925 ArrayType *ArrayOfInt64Ty = ArrayType::get(ElementType: Int64Ty, NumElements: Initializers.size());
926 GlobalVariable *GV = new GlobalVariable(
927 *CurModule, ArrayOfInt64Ty, false, GlobalVariable::InternalLinkage,
928 ConstantArray::get(T: ArrayOfInt64Ty, V: Initializers),
929 "__sancov_gen_cov_switch_values");
930 if (Options.GatedCallbacks) {
931 auto GateBranch = CreateGateBranch(F, FunctionGateCmp, IP: I);
932 IRBuilder<> GateIRB(GateBranch);
933 GateIRB.CreateCall(Callee: SanCovTraceSwitchFunction, Args: {Cond, GV});
934 } else {
935 IRB.CreateCall(Callee: SanCovTraceSwitchFunction, Args: {Cond, GV});
936 }
937 }
938 }
939}
940
941void ModuleSanitizerCoverage::InjectTraceForDiv(
942 Function &, ArrayRef<BinaryOperator *> DivTraceTargets) {
943 for (auto *BO : DivTraceTargets) {
944 InstrumentationIRBuilder IRB(BO);
945 Value *A1 = BO->getOperand(i_nocapture: 1);
946 if (isa<ConstantInt>(Val: A1))
947 continue;
948 if (!A1->getType()->isIntegerTy())
949 continue;
950 uint64_t TypeSize = DL->getTypeStoreSizeInBits(Ty: A1->getType());
951 int CallbackIdx = TypeSize == 32 ? 0 : TypeSize == 64 ? 1 : -1;
952 if (CallbackIdx < 0)
953 continue;
954 auto Ty = Type::getIntNTy(C&: *C, N: TypeSize);
955 IRB.CreateCall(Callee: SanCovTraceDivFunction[CallbackIdx],
956 Args: {IRB.CreateIntCast(V: A1, DestTy: Ty, isSigned: true)});
957 }
958}
959
960void ModuleSanitizerCoverage::InjectTraceForGep(
961 Function &, ArrayRef<GetElementPtrInst *> GepTraceTargets) {
962 for (auto *GEP : GepTraceTargets) {
963 InstrumentationIRBuilder IRB(GEP);
964 for (Use &Idx : GEP->indices())
965 if (!isa<ConstantInt>(Val: Idx) && Idx->getType()->isIntegerTy())
966 IRB.CreateCall(Callee: SanCovTraceGepFunction,
967 Args: {IRB.CreateIntCast(V: Idx, DestTy: IntptrTy, isSigned: true)});
968 }
969}
970
971void ModuleSanitizerCoverage::InjectTraceForLoadsAndStores(
972 Function &, ArrayRef<LoadInst *> Loads, ArrayRef<StoreInst *> Stores) {
973 auto CallbackIdx = [&](Type *ElementTy) -> int {
974 uint64_t TypeSize = DL->getTypeStoreSizeInBits(Ty: ElementTy);
975 return TypeSize == 8 ? 0
976 : TypeSize == 16 ? 1
977 : TypeSize == 32 ? 2
978 : TypeSize == 64 ? 3
979 : TypeSize == 128 ? 4
980 : -1;
981 };
982 for (auto *LI : Loads) {
983 InstrumentationIRBuilder IRB(LI);
984 auto Ptr = LI->getPointerOperand();
985 int Idx = CallbackIdx(LI->getType());
986 if (Idx < 0)
987 continue;
988 IRB.CreateCall(Callee: SanCovLoadFunction[Idx], Args: Ptr);
989 }
990 for (auto *SI : Stores) {
991 InstrumentationIRBuilder IRB(SI);
992 auto Ptr = SI->getPointerOperand();
993 int Idx = CallbackIdx(SI->getValueOperand()->getType());
994 if (Idx < 0)
995 continue;
996 IRB.CreateCall(Callee: SanCovStoreFunction[Idx], Args: Ptr);
997 }
998}
999
1000void ModuleSanitizerCoverage::InjectTraceForCmp(
1001 Function &F, ArrayRef<Instruction *> CmpTraceTargets,
1002 Value *&FunctionGateCmp) {
1003 for (auto *I : CmpTraceTargets) {
1004 if (ICmpInst *ICMP = dyn_cast<ICmpInst>(Val: I)) {
1005 InstrumentationIRBuilder IRB(ICMP);
1006 Value *A0 = ICMP->getOperand(i_nocapture: 0);
1007 Value *A1 = ICMP->getOperand(i_nocapture: 1);
1008 if (!A0->getType()->isIntegerTy())
1009 continue;
1010 uint64_t TypeSize = DL->getTypeStoreSizeInBits(Ty: A0->getType());
1011 int CallbackIdx = TypeSize == 8 ? 0
1012 : TypeSize == 16 ? 1
1013 : TypeSize == 32 ? 2
1014 : TypeSize == 64 ? 3
1015 : -1;
1016 if (CallbackIdx < 0)
1017 continue;
1018 // __sanitizer_cov_trace_cmp((type_size << 32) | predicate, A0, A1);
1019 auto CallbackFunc = SanCovTraceCmpFunction[CallbackIdx];
1020 bool FirstIsConst = isa<ConstantInt>(Val: A0);
1021 bool SecondIsConst = isa<ConstantInt>(Val: A1);
1022 // If both are const, then we don't need such a comparison.
1023 if (FirstIsConst && SecondIsConst)
1024 continue;
1025 // If only one is const, then make it the first callback argument.
1026 if (FirstIsConst || SecondIsConst) {
1027 CallbackFunc = SanCovTraceConstCmpFunction[CallbackIdx];
1028 if (SecondIsConst)
1029 std::swap(a&: A0, b&: A1);
1030 }
1031
1032 auto Ty = Type::getIntNTy(C&: *C, N: TypeSize);
1033 if (Options.GatedCallbacks) {
1034 auto GateBranch = CreateGateBranch(F, FunctionGateCmp, IP: I);
1035 IRBuilder<> GateIRB(GateBranch);
1036 GateIRB.CreateCall(Callee: CallbackFunc, Args: {GateIRB.CreateIntCast(V: A0, DestTy: Ty, isSigned: true),
1037 GateIRB.CreateIntCast(V: A1, DestTy: Ty, isSigned: true)});
1038 } else {
1039 IRB.CreateCall(Callee: CallbackFunc, Args: {IRB.CreateIntCast(V: A0, DestTy: Ty, isSigned: true),
1040 IRB.CreateIntCast(V: A1, DestTy: Ty, isSigned: true)});
1041 }
1042 }
1043 }
1044}
1045
1046void ModuleSanitizerCoverage::InjectCoverageAtBlock(Function &F, BasicBlock &BB,
1047 size_t Idx,
1048 Value *&FunctionGateCmp,
1049 bool IsLeafFunc) {
1050 BasicBlock::iterator IP = BB.getFirstInsertionPt();
1051 bool IsEntryBB = &BB == &F.getEntryBlock();
1052 DebugLoc EntryLoc;
1053 if (IsEntryBB) {
1054 if (auto SP = F.getSubprogram())
1055 EntryLoc = DILocation::get(Context&: SP->getContext(), Line: SP->getScopeLine(), Column: 0, Scope: SP);
1056 // Keep static allocas and llvm.localescape calls in the entry block. Even
1057 // if we aren't splitting the block, it's nice for allocas to be before
1058 // calls.
1059 IP = PrepareToSplitEntryBlock(BB, IP);
1060 }
1061
1062 InstrumentationIRBuilder IRB(&*IP);
1063 if (EntryLoc)
1064 IRB.SetCurrentDebugLocation(EntryLoc);
1065 if (Options.TracePC) {
1066 IRB.CreateCall(Callee: SanCovTracePC)
1067 ->setCannotMerge(); // gets the PC using GET_CALLER_PC.
1068 }
1069 if (Options.TracePCGuard) {
1070 auto GuardPtr = IRB.CreateConstInBoundsGEP2_64(
1071 Ty: FunctionGuardArray->getValueType(), Ptr: FunctionGuardArray, Idx0: 0, Idx1: Idx);
1072 if (Options.GatedCallbacks) {
1073 Instruction *I = &*IP;
1074 auto GateBranch = CreateGateBranch(F, FunctionGateCmp, IP: I);
1075 IRBuilder<> GateIRB(GateBranch);
1076 GateIRB.CreateCall(Callee: SanCovTracePCGuard, Args: GuardPtr)->setCannotMerge();
1077 } else {
1078 IRB.CreateCall(Callee: SanCovTracePCGuard, Args: GuardPtr)->setCannotMerge();
1079 }
1080 }
1081 if (Options.Inline8bitCounters) {
1082 auto CounterPtr = IRB.CreateGEP(
1083 Ty: Function8bitCounterArray->getValueType(), Ptr: Function8bitCounterArray,
1084 IdxList: {ConstantInt::get(Ty: IntptrTy, V: 0), ConstantInt::get(Ty: IntptrTy, V: Idx)});
1085 auto Load = IRB.CreateLoad(Ty: Int8Ty, Ptr: CounterPtr);
1086 auto Inc = IRB.CreateAdd(LHS: Load, RHS: ConstantInt::get(Ty: Int8Ty, V: 1));
1087 auto Store = IRB.CreateStore(Val: Inc, Ptr: CounterPtr);
1088 Load->setNoSanitizeMetadata();
1089 Store->setNoSanitizeMetadata();
1090 }
1091 if (Options.InlineBoolFlag) {
1092 auto FlagPtr = IRB.CreateGEP(
1093 Ty: FunctionBoolArray->getValueType(), Ptr: FunctionBoolArray,
1094 IdxList: {ConstantInt::get(Ty: IntptrTy, V: 0), ConstantInt::get(Ty: IntptrTy, V: Idx)});
1095 auto Load = IRB.CreateLoad(Ty: Int1Ty, Ptr: FlagPtr);
1096 auto ThenTerm = SplitBlockAndInsertIfThen(
1097 Cond: IRB.CreateIsNull(Arg: Load), SplitBefore: &*IP, Unreachable: false,
1098 BranchWeights: MDBuilder(IRB.getContext()).createUnlikelyBranchWeights());
1099 InstrumentationIRBuilder ThenIRB(ThenTerm);
1100 auto Store = ThenIRB.CreateStore(Val: ConstantInt::getTrue(Ty: Int1Ty), Ptr: FlagPtr);
1101 if (EntryLoc)
1102 Store->setDebugLoc(EntryLoc);
1103 Load->setNoSanitizeMetadata();
1104 Store->setNoSanitizeMetadata();
1105 }
1106 if (Options.StackDepth && IsEntryBB && !IsLeafFunc) {
1107 Module *M = F.getParent();
1108 const DataLayout &DL = M->getDataLayout();
1109
1110 if (Options.StackDepthCallbackMin) {
1111 // In callback mode, only add call when stack depth reaches minimum.
1112 int EstimatedStackSize = 0;
1113 // If dynamic alloca found, always add call.
1114 bool HasDynamicAlloc = false;
1115 // Find an insertion point after last "alloca".
1116 llvm::Instruction *InsertBefore = nullptr;
1117
1118 // Examine all allocas in the basic block. since we're too early
1119 // to have results from Intrinsic::frameaddress, we have to manually
1120 // estimate the stack size.
1121 for (auto &I : BB) {
1122 if (auto *AI = dyn_cast<AllocaInst>(Val: &I)) {
1123 // Move potential insertion point past the "alloca".
1124 InsertBefore = AI->getNextNode();
1125
1126 // Make an estimate on the stack usage.
1127 if (auto AllocaSize = AI->getAllocationSize(DL)) {
1128 if (AllocaSize->isFixed())
1129 EstimatedStackSize += AllocaSize->getFixedValue();
1130 else
1131 HasDynamicAlloc = true;
1132 } else {
1133 HasDynamicAlloc = true;
1134 }
1135 }
1136 }
1137
1138 if (HasDynamicAlloc ||
1139 EstimatedStackSize >= Options.StackDepthCallbackMin) {
1140 if (InsertBefore)
1141 IRB.SetInsertPoint(InsertBefore);
1142 auto Call = IRB.CreateCall(Callee: SanCovStackDepthCallback);
1143 if (EntryLoc)
1144 Call->setDebugLoc(EntryLoc);
1145 Call->setCannotMerge();
1146 }
1147 } else {
1148 // Check stack depth. If it's the deepest so far, record it.
1149 auto FrameAddrPtr = IRB.CreateIntrinsic(
1150 ID: Intrinsic::frameaddress, Types: IRB.getPtrTy(AddrSpace: DL.getAllocaAddrSpace()),
1151 Args: {Constant::getNullValue(Ty: Int32Ty)});
1152 auto FrameAddrInt = IRB.CreatePtrToInt(V: FrameAddrPtr, DestTy: IntptrTy);
1153 auto LowestStack = IRB.CreateLoad(Ty: IntptrTy, Ptr: SanCovLowestStack);
1154 auto IsStackLower = IRB.CreateICmpULT(LHS: FrameAddrInt, RHS: LowestStack);
1155 auto ThenTerm = SplitBlockAndInsertIfThen(
1156 Cond: IsStackLower, SplitBefore: &*IP, Unreachable: false,
1157 BranchWeights: MDBuilder(IRB.getContext()).createUnlikelyBranchWeights());
1158 InstrumentationIRBuilder ThenIRB(ThenTerm);
1159 auto Store = ThenIRB.CreateStore(Val: FrameAddrInt, Ptr: SanCovLowestStack);
1160 if (EntryLoc)
1161 Store->setDebugLoc(EntryLoc);
1162 LowestStack->setNoSanitizeMetadata();
1163 Store->setNoSanitizeMetadata();
1164 }
1165 }
1166}
1167
1168std::string
1169ModuleSanitizerCoverage::getSectionName(const std::string &Section) const {
1170 if (TargetTriple.isOSBinFormatCOFF()) {
1171 if (Section == SanCovCountersSectionName)
1172 return ".SCOV$CM";
1173 if (Section == SanCovBoolFlagSectionName)
1174 return ".SCOV$BM";
1175 if (Section == SanCovPCsSectionName)
1176 return ".SCOVP$M";
1177 return ".SCOV$GM"; // For SanCovGuardsSectionName.
1178 }
1179 if (TargetTriple.isOSBinFormatMachO())
1180 return "__DATA,__" + Section;
1181 return "__" + Section;
1182}
1183
1184std::string
1185ModuleSanitizerCoverage::getSectionStart(const std::string &Section) const {
1186 if (TargetTriple.isOSBinFormatMachO())
1187 return "\1section$start$__DATA$__" + Section;
1188 return "__start___" + Section;
1189}
1190
1191std::string
1192ModuleSanitizerCoverage::getSectionEnd(const std::string &Section) const {
1193 if (TargetTriple.isOSBinFormatMachO())
1194 return "\1section$end$__DATA$__" + Section;
1195 return "__stop___" + Section;
1196}
1197
1198void ModuleSanitizerCoverage::createFunctionControlFlow(Function &F) {
1199 SmallVector<Constant *, 32> CFs;
1200 IRBuilder<> IRB(&*F.getEntryBlock().getFirstInsertionPt());
1201
1202 for (auto &BB : F) {
1203 // blockaddress can not be used on function's entry block.
1204 if (&BB == &F.getEntryBlock())
1205 CFs.push_back(Elt: (Constant *)IRB.CreatePointerCast(V: &F, DestTy: PtrTy));
1206 else
1207 CFs.push_back(
1208 Elt: (Constant *)IRB.CreatePointerCast(V: BlockAddress::get(BB: &BB), DestTy: PtrTy));
1209
1210 for (auto SuccBB : successors(BB: &BB)) {
1211 assert(SuccBB != &F.getEntryBlock());
1212 CFs.push_back(
1213 Elt: (Constant *)IRB.CreatePointerCast(V: BlockAddress::get(BB: SuccBB), DestTy: PtrTy));
1214 }
1215
1216 CFs.push_back(Elt: (Constant *)Constant::getNullValue(Ty: PtrTy));
1217
1218 for (auto &Inst : BB) {
1219 if (CallBase *CB = dyn_cast<CallBase>(Val: &Inst)) {
1220 if (CB->isIndirectCall()) {
1221 // TODO(navidem): handle indirect calls, for now mark its existence.
1222 CFs.push_back(Elt: (Constant *)IRB.CreateIntToPtr(
1223 V: ConstantInt::getAllOnesValue(Ty: IntptrTy), DestTy: PtrTy));
1224 } else {
1225 auto CalledF = CB->getCalledFunction();
1226 if (CalledF && !CalledF->isIntrinsic())
1227 CFs.push_back(Elt: (Constant *)IRB.CreatePointerCast(V: CalledF, DestTy: PtrTy));
1228 }
1229 }
1230 }
1231
1232 CFs.push_back(Elt: (Constant *)Constant::getNullValue(Ty: PtrTy));
1233 }
1234
1235 FunctionCFsArray = CreateFunctionLocalArrayInSection(NumElements: CFs.size(), F, Ty: PtrTy,
1236 Section: SanCovCFsSectionName);
1237 FunctionCFsArray->setInitializer(
1238 ConstantArray::get(T: ArrayType::get(ElementType: PtrTy, NumElements: CFs.size()), V: CFs));
1239 FunctionCFsArray->setConstant(true);
1240}
1241