1//===-- SanitizerCoverage.cpp - coverage instrumentation for sanitizers ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Coverage instrumentation done on LLVM IR level, works with Sanitizers.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/Transforms/Instrumentation/SanitizerCoverage.h"
14#include "InstrumentationOptions.h"
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/ADT/SmallVector.h"
17#include "llvm/Analysis/GlobalsModRef.h"
18#include "llvm/Analysis/PostDominators.h"
19#include "llvm/IR/Constant.h"
20#include "llvm/IR/Constants.h"
21#include "llvm/IR/DataLayout.h"
22#include "llvm/IR/Dominators.h"
23#include "llvm/IR/EHPersonalities.h"
24#include "llvm/IR/Function.h"
25#include "llvm/IR/GlobalVariable.h"
26#include "llvm/IR/IRBuilder.h"
27#include "llvm/IR/IntrinsicInst.h"
28#include "llvm/IR/Intrinsics.h"
29#include "llvm/IR/LLVMContext.h"
30#include "llvm/IR/MDBuilder.h"
31#include "llvm/IR/Module.h"
32#include "llvm/IR/Type.h"
33#include "llvm/IR/ValueSymbolTable.h"
34#include "llvm/Support/SpecialCaseList.h"
35#include "llvm/Support/VirtualFileSystem.h"
36#include "llvm/TargetParser/Triple.h"
37#include "llvm/Transforms/Utils/BasicBlockUtils.h"
38#include "llvm/Transforms/Utils/EscapeEnumerator.h"
39#include "llvm/Transforms/Utils/ModuleUtils.h"
40
41using namespace llvm;
42
43#define DEBUG_TYPE "sancov"
44
45const char SanCovTracePCIndirName[] = "__sanitizer_cov_trace_pc_indir";
46const char SanCovTracePCName[] = "__sanitizer_cov_trace_pc";
47const char SanCovTracePCEntryName[] = "__sanitizer_cov_trace_pc_entry";
48const char SanCovTracePCExitName[] = "__sanitizer_cov_trace_pc_exit";
49const char SanCovTraceCmp1[] = "__sanitizer_cov_trace_cmp1";
50const char SanCovTraceCmp2[] = "__sanitizer_cov_trace_cmp2";
51const char SanCovTraceCmp4[] = "__sanitizer_cov_trace_cmp4";
52const char SanCovTraceCmp8[] = "__sanitizer_cov_trace_cmp8";
53const char SanCovTraceConstCmp1[] = "__sanitizer_cov_trace_const_cmp1";
54const char SanCovTraceConstCmp2[] = "__sanitizer_cov_trace_const_cmp2";
55const char SanCovTraceConstCmp4[] = "__sanitizer_cov_trace_const_cmp4";
56const char SanCovTraceConstCmp8[] = "__sanitizer_cov_trace_const_cmp8";
57const char SanCovLoad1[] = "__sanitizer_cov_load1";
58const char SanCovLoad2[] = "__sanitizer_cov_load2";
59const char SanCovLoad4[] = "__sanitizer_cov_load4";
60const char SanCovLoad8[] = "__sanitizer_cov_load8";
61const char SanCovLoad16[] = "__sanitizer_cov_load16";
62const char SanCovStore1[] = "__sanitizer_cov_store1";
63const char SanCovStore2[] = "__sanitizer_cov_store2";
64const char SanCovStore4[] = "__sanitizer_cov_store4";
65const char SanCovStore8[] = "__sanitizer_cov_store8";
66const char SanCovStore16[] = "__sanitizer_cov_store16";
67const char SanCovTraceDiv4[] = "__sanitizer_cov_trace_div4";
68const char SanCovTraceDiv8[] = "__sanitizer_cov_trace_div8";
69const char SanCovTraceGep[] = "__sanitizer_cov_trace_gep";
70const char SanCovTraceSwitchName[] = "__sanitizer_cov_trace_switch";
71const char SanCovModuleCtorTracePcGuardName[] =
72 "sancov.module_ctor_trace_pc_guard";
73const char SanCovModuleCtor8bitCountersName[] =
74 "sancov.module_ctor_8bit_counters";
75const char SanCovModuleCtorBoolFlagName[] = "sancov.module_ctor_bool_flag";
76static const uint64_t SanCtorAndDtorPriority = 2;
77
78const char SanCovTracePCGuardName[] = "__sanitizer_cov_trace_pc_guard";
79const char SanCovTracePCGuardInitName[] = "__sanitizer_cov_trace_pc_guard_init";
80const char SanCov8bitCountersInitName[] = "__sanitizer_cov_8bit_counters_init";
81const char SanCovBoolFlagInitName[] = "__sanitizer_cov_bool_flag_init";
82const char SanCovPCsInitName[] = "__sanitizer_cov_pcs_init";
83const char SanCovCFsInitName[] = "__sanitizer_cov_cfs_init";
84
85const char SanCovGuardsSectionName[] = "sancov_guards";
86const char SanCovCountersSectionName[] = "sancov_cntrs";
87const char SanCovBoolFlagSectionName[] = "sancov_bools";
88const char SanCovPCsSectionName[] = "sancov_pcs";
89const char SanCovCFsSectionName[] = "sancov_cfs";
90const char SanCovCallbackGateSectionName[] = "sancov_gate";
91
92const char SanCovStackDepthCallbackName[] = "__sanitizer_cov_stack_depth";
93const char SanCovLowestStackName[] = "__sancov_lowest_stack";
94const char SanCovCallbackGateName[] = "__sancov_should_track";
95
96namespace {
97
98SanitizerCoverageOptions getOptions(int LegacyCoverageLevel) {
99 SanitizerCoverageOptions Res;
100 switch (LegacyCoverageLevel) {
101 case 0:
102 Res.CoverageType = SanitizerCoverageOptions::SCK_None;
103 break;
104 case 1:
105 Res.CoverageType = SanitizerCoverageOptions::SCK_Function;
106 break;
107 case 2:
108 Res.CoverageType = SanitizerCoverageOptions::SCK_BB;
109 break;
110 case 3:
111 Res.CoverageType = SanitizerCoverageOptions::SCK_Edge;
112 break;
113 case 4:
114 Res.CoverageType = SanitizerCoverageOptions::SCK_Edge;
115 Res.IndirectCalls = true;
116 break;
117 }
118 return Res;
119}
120
121SanitizerCoverageOptions overrideFromCL(const InstrumentationOptions &Opts,
122 SanitizerCoverageOptions Options) {
123 // Sets CoverageType and IndirectCalls.
124 SanitizerCoverageOptions CLOpts = getOptions(LegacyCoverageLevel: Opts.sanitizer_coverage_level);
125 Options.CoverageType = std::max(a: Options.CoverageType, b: CLOpts.CoverageType);
126 Options.IndirectCalls |= CLOpts.IndirectCalls;
127 Options.TraceCmp |= Opts.sanitizer_coverage_trace_compares;
128 Options.TraceDiv |= Opts.sanitizer_coverage_trace_divs;
129 Options.TraceGep |= Opts.sanitizer_coverage_trace_geps;
130 Options.TracePC |= Opts.sanitizer_coverage_trace_pc;
131 Options.TracePCEntryExit |= Opts.sanitizer_coverage_trace_pc_entry_exit;
132 Options.TracePCGuard |= Opts.sanitizer_coverage_trace_pc_guard;
133 Options.Inline8bitCounters |= Opts.sanitizer_coverage_inline_8bit_counters;
134 Options.InlineBoolFlag |= Opts.sanitizer_coverage_inline_bool_flag;
135 Options.PCTable |= Opts.sanitizer_coverage_pc_table;
136 Options.NoPrune |= !Opts.sanitizer_coverage_prune_blocks;
137 Options.StackDepth |= Opts.sanitizer_coverage_stack_depth;
138 Options.StackDepthCallbackMin =
139 std::max(a: Options.StackDepthCallbackMin,
140 b: Opts.sanitizer_coverage_stack_depth_callback_min);
141 Options.TraceLoads |= Opts.sanitizer_coverage_trace_loads;
142 Options.TraceStores |= Opts.sanitizer_coverage_trace_stores;
143 Options.GatedCallbacks |= Opts.sanitizer_coverage_gated_trace_callbacks;
144 if (!Options.TracePCGuard && !Options.TracePC && !Options.TracePCEntryExit &&
145 !Options.Inline8bitCounters && !Options.StackDepth &&
146 !Options.InlineBoolFlag && !Options.TraceLoads && !Options.TraceStores)
147 Options.TracePCGuard = true; // TracePCGuard is default.
148 Options.CollectControlFlow |= Opts.sanitizer_coverage_control_flow;
149 return Options;
150}
151
152class ModuleSanitizerCoverage {
153public:
154 using DomTreeCallback = function_ref<const DominatorTree &(Function &F)>;
155 using PostDomTreeCallback =
156 function_ref<const PostDominatorTree &(Function &F)>;
157
158 ModuleSanitizerCoverage(const InstrumentationOptions &Opts, Module &M,
159 DomTreeCallback DTCallback,
160 PostDomTreeCallback PDTCallback,
161 const SanitizerCoverageOptions &Options,
162 const SpecialCaseList *Allowlist,
163 const SpecialCaseList *Blocklist)
164 : Opts(Opts), M(M), DTCallback(DTCallback), PDTCallback(PDTCallback),
165 Options(Options), Allowlist(Allowlist), Blocklist(Blocklist) {}
166
167 bool instrumentModule();
168
169private:
170 void createFunctionControlFlow(Function &F);
171 void instrumentFunction(Function &F);
172 void InjectCoverageForIndirectCalls(Function &F,
173 ArrayRef<Instruction *> IndirCalls);
174 void InjectTraceForCmp(Function &F, ArrayRef<Instruction *> CmpTraceTargets,
175 Value *&FunctionGateCmp);
176 void InjectTraceForDiv(Function &F,
177 ArrayRef<BinaryOperator *> DivTraceTargets);
178 void InjectTraceForGep(Function &F,
179 ArrayRef<GetElementPtrInst *> GepTraceTargets);
180 void InjectTraceForLoadsAndStores(Function &F, ArrayRef<LoadInst *> Loads,
181 ArrayRef<StoreInst *> Stores);
182 void InjectTraceForExits(Function &F);
183 void InjectTraceForSwitch(Function &F,
184 ArrayRef<Instruction *> SwitchTraceTargets,
185 Value *&FunctionGateCmp);
186 bool InjectCoverage(Function &F, ArrayRef<BasicBlock *> AllBlocks,
187 Value *&FunctionGateCmp, bool IsLeafFunc);
188 GlobalVariable *CreateFunctionLocalArrayInSection(size_t NumElements,
189 Function &F, Type *Ty,
190 const char *Section);
191 GlobalVariable *CreatePCArray(Function &F, ArrayRef<BasicBlock *> AllBlocks);
192 void CreateFunctionLocalArrays(Function &F, ArrayRef<BasicBlock *> AllBlocks);
193 Instruction *CreateGateBranch(Function &F, Value *&FunctionGateCmp,
194 Instruction *I);
195 Value *CreateFunctionLocalGateCmp(IRBuilder<> &IRB);
196 void InjectCoverageAtBlock(Function &F, BasicBlock &BB, size_t Idx,
197 Value *&FunctionGateCmp, bool IsLeafFunc);
198 Function *CreateInitCallsForSections(Module &M, const char *CtorName,
199 const char *InitFunctionName, Type *Ty,
200 const char *Section);
201 std::pair<Value *, Value *> CreateSecStartEnd(Module &M, const char *Section,
202 Type *Ty);
203
204 std::string getSectionName(const std::string &Section) const;
205 std::string getSectionStart(const std::string &Section) const;
206 std::string getSectionEnd(const std::string &Section) const;
207
208 const InstrumentationOptions &Opts;
209 Module &M;
210 DomTreeCallback DTCallback;
211 PostDomTreeCallback PDTCallback;
212
213 FunctionCallee SanCovStackDepthCallback;
214 FunctionCallee SanCovTracePCIndir;
215 FunctionCallee SanCovTracePC, SanCovTracePCGuard;
216 FunctionCallee SanCovTracePCEntry, SanCovTracePCExit;
217 std::array<FunctionCallee, 4> SanCovTraceCmpFunction;
218 std::array<FunctionCallee, 4> SanCovTraceConstCmpFunction;
219 std::array<FunctionCallee, 5> SanCovLoadFunction;
220 std::array<FunctionCallee, 5> SanCovStoreFunction;
221 std::array<FunctionCallee, 2> SanCovTraceDivFunction;
222 FunctionCallee SanCovTraceGepFunction;
223 FunctionCallee SanCovTraceSwitchFunction;
224 GlobalVariable *SanCovLowestStack;
225 GlobalVariable *SanCovCallbackGate;
226 Type *PtrTy, *IntptrTy, *Int64Ty, *Int32Ty, *Int16Ty, *Int8Ty, *Int1Ty;
227 Module *CurModule;
228 Triple TargetTriple;
229 LLVMContext *C;
230 const DataLayout *DL;
231
232 GlobalVariable *FunctionGuardArray; // for trace-pc-guard.
233 GlobalVariable *Function8bitCounterArray; // for inline-8bit-counters.
234 GlobalVariable *FunctionBoolArray; // for inline-bool-flag.
235 GlobalVariable *FunctionPCsArray; // for pc-table.
236 GlobalVariable *FunctionCFsArray; // for control flow table
237 SmallVector<GlobalValue *, 20> GlobalsToAppendToUsed;
238 SmallVector<GlobalValue *, 20> GlobalsToAppendToCompilerUsed;
239
240 SanitizerCoverageOptions Options;
241
242 const SpecialCaseList *Allowlist;
243 const SpecialCaseList *Blocklist;
244};
245} // namespace
246
247SanitizerCoveragePass::SanitizerCoveragePass(
248 SanitizerCoverageOptions Options, IntrusiveRefCntPtr<vfs::FileSystem> VFS,
249 const std::vector<std::string> &AllowlistFiles,
250 const std::vector<std::string> &BlocklistFiles)
251 : Options(std::move(Options)),
252 VFS(VFS ? std::move(VFS) : vfs::getRealFileSystem()) {
253 if (AllowlistFiles.size() > 0)
254 Allowlist = SpecialCaseList::createOrDie(Paths: AllowlistFiles, FS&: *this->VFS);
255 if (BlocklistFiles.size() > 0)
256 Blocklist = SpecialCaseList::createOrDie(Paths: BlocklistFiles, FS&: *this->VFS);
257}
258
259PreservedAnalyses SanitizerCoveragePass::run(Module &M,
260 ModuleAnalysisManager &MAM) {
261 auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
262 auto DTCallback = [&FAM](Function &F) -> const DominatorTree & {
263 return FAM.getResult<DominatorTreeAnalysis>(IR&: F);
264 };
265 auto PDTCallback = [&FAM](Function &F) -> const PostDominatorTree & {
266 return FAM.getResult<PostDominatorTreeAnalysis>(IR&: F);
267 };
268 const InstrumentationOptions &Opts = getInstrumentationOptions(M);
269 ModuleSanitizerCoverage ModuleSancov(Opts, M, DTCallback, PDTCallback,
270 overrideFromCL(Opts, Options),
271 Allowlist.get(), Blocklist.get());
272 if (!ModuleSancov.instrumentModule())
273 return PreservedAnalyses::all();
274
275 PreservedAnalyses PA = PreservedAnalyses::none();
276 // GlobalsAA is considered stateless and does not get invalidated unless
277 // explicitly invalidated; PreservedAnalyses::none() is not enough. Sanitizers
278 // make changes that require GlobalsAA to be invalidated.
279 PA.abandon<GlobalsAA>();
280 return PA;
281}
282
283std::pair<Value *, Value *>
284ModuleSanitizerCoverage::CreateSecStartEnd(Module &M, const char *Section,
285 Type *Ty) {
286 // Use ExternalWeak so that if all sections are discarded due to section
287 // garbage collection, the linker will not report undefined symbol errors.
288 // Windows defines the start/stop symbols in compiler-rt so no need for
289 // ExternalWeak.
290 GlobalValue::LinkageTypes Linkage = TargetTriple.isOSBinFormatCOFF()
291 ? GlobalVariable::ExternalLinkage
292 : GlobalVariable::ExternalWeakLinkage;
293 GlobalVariable *SecStart = new GlobalVariable(M, Ty, false, Linkage, nullptr,
294 getSectionStart(Section));
295 SecStart->setVisibility(GlobalValue::HiddenVisibility);
296 GlobalVariable *SecEnd = new GlobalVariable(M, Ty, false, Linkage, nullptr,
297 getSectionEnd(Section));
298 SecEnd->setVisibility(GlobalValue::HiddenVisibility);
299 if (!TargetTriple.isOSBinFormatCOFF())
300 return std::make_pair(x&: SecStart, y&: SecEnd);
301
302 // Account for the fact that on windows-msvc __start_* symbols actually
303 // point to a uint64_t before the start of the array.
304 auto *GEP = ConstantExpr::getPtrAdd(
305 Ptr: SecStart, Offset: ConstantInt::get(Ty: IntptrTy, V: sizeof(uint64_t)));
306 return std::make_pair(x&: GEP, y&: SecEnd);
307}
308
309Function *ModuleSanitizerCoverage::CreateInitCallsForSections(
310 Module &M, const char *CtorName, const char *InitFunctionName, Type *Ty,
311 const char *Section) {
312 if (Opts.sanitizer_coverage_drop_ctors)
313 return nullptr;
314 auto SecStartEnd = CreateSecStartEnd(M, Section, Ty);
315 auto SecStart = SecStartEnd.first;
316 auto SecEnd = SecStartEnd.second;
317 Function *CtorFunc;
318 std::tie(args&: CtorFunc, args: std::ignore) = createSanitizerCtorAndInitFunctions(
319 M, CtorName, InitName: InitFunctionName, InitArgTypes: {PtrTy, PtrTy}, InitArgs: {SecStart, SecEnd});
320 assert(CtorFunc->getName() == CtorName);
321
322 if (TargetTriple.supportsCOMDAT()) {
323 // Use comdat to dedup CtorFunc.
324 CtorFunc->setComdat(M.getOrInsertComdat(Name: CtorName));
325 appendToGlobalCtors(M, F: CtorFunc, Priority: SanCtorAndDtorPriority, Data: CtorFunc);
326 } else {
327 appendToGlobalCtors(M, F: CtorFunc, Priority: SanCtorAndDtorPriority);
328 }
329
330 if (TargetTriple.isOSBinFormatCOFF()) {
331 // In COFF files, if the contructors are set as COMDAT (they are because
332 // COFF supports COMDAT) and the linker flag /OPT:REF (strip unreferenced
333 // functions and data) is used, the constructors get stripped. To prevent
334 // this, give the constructors weak ODR linkage and ensure the linker knows
335 // to include the sancov constructor. This way the linker can deduplicate
336 // the constructors but always leave one copy.
337 CtorFunc->setLinkage(GlobalValue::WeakODRLinkage);
338 }
339 return CtorFunc;
340}
341
342bool ModuleSanitizerCoverage::instrumentModule() {
343 if (Options.CoverageType == SanitizerCoverageOptions::SCK_None)
344 return false;
345 if (Allowlist &&
346 !Allowlist->inSection(Section: "coverage", Prefix: "src", Query: M.getSourceFileName()))
347 return false;
348 if (Blocklist &&
349 Blocklist->inSection(Section: "coverage", Prefix: "src", Query: M.getSourceFileName()))
350 return false;
351 C = &(M.getContext());
352 DL = &M.getDataLayout();
353 CurModule = &M;
354 TargetTriple = M.getTargetTriple();
355 FunctionGuardArray = nullptr;
356 Function8bitCounterArray = nullptr;
357 FunctionBoolArray = nullptr;
358 FunctionPCsArray = nullptr;
359 FunctionCFsArray = nullptr;
360 IntptrTy = Type::getIntNTy(C&: *C, N: DL->getPointerSizeInBits());
361 PtrTy = PointerType::getUnqual(C&: *C);
362 Type *VoidTy = Type::getVoidTy(C&: *C);
363 IRBuilder<> IRB(M);
364 Int64Ty = IRB.getInt64Ty();
365 Int32Ty = IRB.getInt32Ty();
366 Int16Ty = IRB.getInt16Ty();
367 Int8Ty = IRB.getInt8Ty();
368 Int1Ty = IRB.getInt1Ty();
369
370 SanCovTracePCIndir =
371 M.getOrInsertFunction(Name: SanCovTracePCIndirName, RetTy: VoidTy, Args: IntptrTy);
372 // Make sure smaller parameters are zero-extended to i64 if required by the
373 // target ABI.
374 AttributeList SanCovTraceCmpZeroExtAL;
375 SanCovTraceCmpZeroExtAL =
376 SanCovTraceCmpZeroExtAL.addParamAttribute(C&: *C, ArgNo: 0, Kind: Attribute::ZExt);
377 SanCovTraceCmpZeroExtAL =
378 SanCovTraceCmpZeroExtAL.addParamAttribute(C&: *C, ArgNo: 1, Kind: Attribute::ZExt);
379
380 SanCovTraceCmpFunction[0] =
381 M.getOrInsertFunction(Name: SanCovTraceCmp1, AttributeList: SanCovTraceCmpZeroExtAL, RetTy: VoidTy,
382 Args: IRB.getInt8Ty(), Args: IRB.getInt8Ty());
383 SanCovTraceCmpFunction[1] =
384 M.getOrInsertFunction(Name: SanCovTraceCmp2, AttributeList: SanCovTraceCmpZeroExtAL, RetTy: VoidTy,
385 Args: IRB.getInt16Ty(), Args: IRB.getInt16Ty());
386 SanCovTraceCmpFunction[2] =
387 M.getOrInsertFunction(Name: SanCovTraceCmp4, AttributeList: SanCovTraceCmpZeroExtAL, RetTy: VoidTy,
388 Args: IRB.getInt32Ty(), Args: IRB.getInt32Ty());
389 SanCovTraceCmpFunction[3] =
390 M.getOrInsertFunction(Name: SanCovTraceCmp8, RetTy: VoidTy, Args: Int64Ty, Args: Int64Ty);
391
392 SanCovTraceConstCmpFunction[0] = M.getOrInsertFunction(
393 Name: SanCovTraceConstCmp1, AttributeList: SanCovTraceCmpZeroExtAL, RetTy: VoidTy, Args: Int8Ty, Args: Int8Ty);
394 SanCovTraceConstCmpFunction[1] = M.getOrInsertFunction(
395 Name: SanCovTraceConstCmp2, AttributeList: SanCovTraceCmpZeroExtAL, RetTy: VoidTy, Args: Int16Ty, Args: Int16Ty);
396 SanCovTraceConstCmpFunction[2] = M.getOrInsertFunction(
397 Name: SanCovTraceConstCmp4, AttributeList: SanCovTraceCmpZeroExtAL, RetTy: VoidTy, Args: Int32Ty, Args: Int32Ty);
398 SanCovTraceConstCmpFunction[3] =
399 M.getOrInsertFunction(Name: SanCovTraceConstCmp8, RetTy: VoidTy, Args: Int64Ty, Args: Int64Ty);
400
401 // Loads.
402 SanCovLoadFunction[0] = M.getOrInsertFunction(Name: SanCovLoad1, RetTy: VoidTy, Args: PtrTy);
403 SanCovLoadFunction[1] = M.getOrInsertFunction(Name: SanCovLoad2, RetTy: VoidTy, Args: PtrTy);
404 SanCovLoadFunction[2] = M.getOrInsertFunction(Name: SanCovLoad4, RetTy: VoidTy, Args: PtrTy);
405 SanCovLoadFunction[3] = M.getOrInsertFunction(Name: SanCovLoad8, RetTy: VoidTy, Args: PtrTy);
406 SanCovLoadFunction[4] = M.getOrInsertFunction(Name: SanCovLoad16, RetTy: VoidTy, Args: PtrTy);
407 // Stores.
408 SanCovStoreFunction[0] = M.getOrInsertFunction(Name: SanCovStore1, RetTy: VoidTy, Args: PtrTy);
409 SanCovStoreFunction[1] = M.getOrInsertFunction(Name: SanCovStore2, RetTy: VoidTy, Args: PtrTy);
410 SanCovStoreFunction[2] = M.getOrInsertFunction(Name: SanCovStore4, RetTy: VoidTy, Args: PtrTy);
411 SanCovStoreFunction[3] = M.getOrInsertFunction(Name: SanCovStore8, RetTy: VoidTy, Args: PtrTy);
412 SanCovStoreFunction[4] = M.getOrInsertFunction(Name: SanCovStore16, RetTy: VoidTy, Args: PtrTy);
413
414 {
415 AttributeList AL;
416 AL = AL.addParamAttribute(C&: *C, ArgNo: 0, Kind: Attribute::ZExt);
417 SanCovTraceDivFunction[0] =
418 M.getOrInsertFunction(Name: SanCovTraceDiv4, AttributeList: AL, RetTy: VoidTy, Args: IRB.getInt32Ty());
419 }
420 SanCovTraceDivFunction[1] =
421 M.getOrInsertFunction(Name: SanCovTraceDiv8, RetTy: VoidTy, Args: Int64Ty);
422 SanCovTraceGepFunction =
423 M.getOrInsertFunction(Name: SanCovTraceGep, RetTy: VoidTy, Args: IntptrTy);
424 SanCovTraceSwitchFunction =
425 M.getOrInsertFunction(Name: SanCovTraceSwitchName, RetTy: VoidTy, Args: Int64Ty, Args: PtrTy);
426
427 SanCovLowestStack = M.getOrInsertGlobal(Name: SanCovLowestStackName, Ty: IntptrTy);
428 if (SanCovLowestStack->getValueType() != IntptrTy) {
429 C->emitError(ErrorStr: StringRef("'") + SanCovLowestStackName +
430 "' should not be declared by the user");
431 return true;
432 }
433 SanCovLowestStack->setThreadLocalMode(
434 GlobalValue::ThreadLocalMode::InitialExecTLSModel);
435 if (Options.StackDepth && !SanCovLowestStack->isDeclaration())
436 SanCovLowestStack->setInitializer(Constant::getAllOnesValue(Ty: IntptrTy));
437
438 if (Options.GatedCallbacks) {
439 if (!Options.TracePCGuard && !Options.TraceCmp) {
440 C->emitError(ErrorStr: "'sanitizer-coverage-gated-trace-callbacks' is only "
441 "supported with trace-pc-guard or trace-cmp");
442 return true;
443 }
444
445 SanCovCallbackGate = cast<GlobalVariable>(
446 Val: M.getOrInsertGlobal(Name: SanCovCallbackGateName, Ty: Int64Ty));
447 SanCovCallbackGate->setSection(
448 getSectionName(Section: SanCovCallbackGateSectionName));
449 SanCovCallbackGate->setInitializer(Constant::getNullValue(Ty: Int64Ty));
450 SanCovCallbackGate->setLinkage(GlobalVariable::LinkOnceAnyLinkage);
451 SanCovCallbackGate->setVisibility(GlobalVariable::HiddenVisibility);
452 appendToCompilerUsed(M, Values: SanCovCallbackGate);
453 }
454
455 SanCovTracePC = M.getOrInsertFunction(Name: SanCovTracePCName, RetTy: VoidTy);
456 SanCovTracePCEntry = M.getOrInsertFunction(Name: SanCovTracePCEntryName, RetTy: VoidTy);
457 SanCovTracePCExit = M.getOrInsertFunction(Name: SanCovTracePCExitName, RetTy: VoidTy);
458 SanCovTracePCGuard =
459 M.getOrInsertFunction(Name: SanCovTracePCGuardName, RetTy: VoidTy, Args: PtrTy);
460
461 SanCovStackDepthCallback =
462 M.getOrInsertFunction(Name: SanCovStackDepthCallbackName, RetTy: VoidTy);
463
464 for (auto &F : M)
465 instrumentFunction(F);
466
467 Function *Ctor = nullptr;
468
469 if (FunctionGuardArray)
470 Ctor = CreateInitCallsForSections(M, CtorName: SanCovModuleCtorTracePcGuardName,
471 InitFunctionName: SanCovTracePCGuardInitName, Ty: Int32Ty,
472 Section: SanCovGuardsSectionName);
473 if (Function8bitCounterArray)
474 Ctor = CreateInitCallsForSections(M, CtorName: SanCovModuleCtor8bitCountersName,
475 InitFunctionName: SanCov8bitCountersInitName, Ty: Int8Ty,
476 Section: SanCovCountersSectionName);
477 if (FunctionBoolArray) {
478 Ctor = CreateInitCallsForSections(M, CtorName: SanCovModuleCtorBoolFlagName,
479 InitFunctionName: SanCovBoolFlagInitName, Ty: Int1Ty,
480 Section: SanCovBoolFlagSectionName);
481 }
482 if (Ctor && Options.PCTable) {
483 auto SecStartEnd = CreateSecStartEnd(M, Section: SanCovPCsSectionName, Ty: IntptrTy);
484 FunctionCallee InitFunction =
485 declareSanitizerInitFunction(M, InitName: SanCovPCsInitName, InitArgTypes: {PtrTy, PtrTy});
486 IRBuilder<> IRBCtor(Ctor->getEntryBlock().getTerminator());
487 IRBCtor.CreateCall(Callee: InitFunction, Args: {SecStartEnd.first, SecStartEnd.second});
488 }
489
490 if (Ctor && Options.CollectControlFlow) {
491 auto SecStartEnd = CreateSecStartEnd(M, Section: SanCovCFsSectionName, Ty: IntptrTy);
492 FunctionCallee InitFunction =
493 declareSanitizerInitFunction(M, InitName: SanCovCFsInitName, InitArgTypes: {PtrTy, PtrTy});
494 IRBuilder<> IRBCtor(Ctor->getEntryBlock().getTerminator());
495 IRBCtor.CreateCall(Callee: InitFunction, Args: {SecStartEnd.first, SecStartEnd.second});
496 }
497
498 appendToUsed(M, Values: GlobalsToAppendToUsed);
499 appendToCompilerUsed(M, Values: GlobalsToAppendToCompilerUsed);
500 return true;
501}
502
503// True if block has successors and it dominates all of them.
504static bool isFullDominator(const BasicBlock *BB, const DominatorTree &DT) {
505 if (succ_empty(BB))
506 return false;
507
508 return llvm::all_of(Range: successors(BB), P: [&](const BasicBlock *SUCC) {
509 return DT.dominates(A: BB, B: SUCC);
510 });
511}
512
513// True if block has predecessors and it postdominates all of them.
514static bool isFullPostDominator(const BasicBlock *BB,
515 const PostDominatorTree &PDT) {
516 if (pred_empty(BB))
517 return false;
518
519 return llvm::all_of(Range: predecessors(BB), P: [&](const BasicBlock *PRED) {
520 return PDT.dominates(A: BB, B: PRED);
521 });
522}
523
524static bool shouldInstrumentBlock(const Function &F, const BasicBlock *BB,
525 const DominatorTree &DT,
526 const PostDominatorTree &PDT,
527 const SanitizerCoverageOptions &Options) {
528 // Don't insert coverage for blocks containing nothing but unreachable: we
529 // will never call __sanitizer_cov() for them, so counting them in
530 // NumberOfInstrumentedBlocks() might complicate calculation of code coverage
531 // percentage. Also, unreachable instructions frequently have no debug
532 // locations.
533 if (isa<UnreachableInst>(Val: BB->getFirstNonPHIOrLifetime()))
534 return false;
535
536 // Don't insert coverage into blocks without a valid insertion point
537 // (catchswitch blocks).
538 if (BB->getFirstInsertionPt() == BB->end())
539 return false;
540
541 if (Options.NoPrune || &F.getEntryBlock() == BB)
542 return true;
543
544 if (Options.CoverageType == SanitizerCoverageOptions::SCK_Function &&
545 &F.getEntryBlock() != BB)
546 return false;
547
548 // Do not instrument full dominators, or full post-dominators with multiple
549 // predecessors.
550 return !isFullDominator(BB, DT) &&
551 !(isFullPostDominator(BB, PDT) && !BB->getSinglePredecessor());
552}
553
554// Returns true iff From->To is a backedge.
555// A twist here is that we treat From->To as a backedge if
556// * To dominates From or
557// * To->UniqueSuccessor dominates From
558static bool IsBackEdge(BasicBlock *From, BasicBlock *To,
559 const DominatorTree &DT) {
560 if (DT.dominates(A: To, B: From))
561 return true;
562 if (auto Next = To->getUniqueSuccessor())
563 if (DT.dominates(A: Next, B: From))
564 return true;
565 return false;
566}
567
568// Prunes uninteresting Cmp instrumentation:
569// * CMP instructions that feed into loop backedge branch.
570//
571// Note that Cmp pruning is controlled by the same flag as the
572// BB pruning.
573static bool IsInterestingCmp(ICmpInst *CMP, const DominatorTree &DT,
574 const SanitizerCoverageOptions &Options) {
575 if (!Options.NoPrune)
576 if (CMP->hasOneUse())
577 if (auto BR = dyn_cast<CondBrInst>(Val: CMP->user_back()))
578 for (BasicBlock *B : BR->successors())
579 if (IsBackEdge(From: BR->getParent(), To: B, DT))
580 return false;
581 return true;
582}
583
584void ModuleSanitizerCoverage::instrumentFunction(Function &F) {
585 if (F.empty())
586 return;
587 if (F.getName().contains(Other: ".module_ctor"))
588 return; // Should not instrument sanitizer init functions.
589 if (F.getName().starts_with(Prefix: "__sanitizer_"))
590 return; // Don't instrument __sanitizer_* callbacks.
591 // Don't touch available_externally functions, their actual body is elewhere.
592 if (F.getLinkage() == GlobalValue::AvailableExternallyLinkage)
593 return;
594 // Don't instrument MSVC CRT configuration helpers. They may run before normal
595 // initialization.
596 if (F.getName() == "__local_stdio_printf_options" ||
597 F.getName() == "__local_stdio_scanf_options")
598 return;
599 if (isa<UnreachableInst>(Val: F.getEntryBlock().getTerminator()))
600 return;
601 // Don't instrument functions using SEH for now. Splitting basic blocks like
602 // we do for coverage breaks WinEHPrepare.
603 // FIXME: Remove this when SEH no longer uses landingpad pattern matching.
604 if (F.hasPersonalityFn() &&
605 isAsynchronousEHPersonality(Pers: classifyEHPersonality(Pers: F.getPersonalityFn())))
606 return;
607 if (Allowlist && !Allowlist->inSection(Section: "coverage", Prefix: "fun", Query: F.getName()))
608 return;
609 if (Blocklist && Blocklist->inSection(Section: "coverage", Prefix: "fun", Query: F.getName()))
610 return;
611 // Do not apply any instrumentation for naked functions.
612 if (F.hasFnAttribute(Kind: Attribute::Naked))
613 return;
614 if (F.hasFnAttribute(Kind: Attribute::NoSanitizeCoverage))
615 return;
616 if (F.hasFnAttribute(Kind: Attribute::DisableSanitizerInstrumentation))
617 return;
618 if (Options.CoverageType >= SanitizerCoverageOptions::SCK_Edge) {
619 SplitAllCriticalEdges(
620 F, Options: CriticalEdgeSplittingOptions().setIgnoreUnreachableDests());
621 }
622 SmallVector<Instruction *, 8> IndirCalls;
623 SmallVector<BasicBlock *, 16> BlocksToInstrument;
624 SmallVector<Instruction *, 8> CmpTraceTargets;
625 SmallVector<Instruction *, 8> SwitchTraceTargets;
626 SmallVector<BinaryOperator *, 8> DivTraceTargets;
627 SmallVector<GetElementPtrInst *, 8> GepTraceTargets;
628 SmallVector<LoadInst *, 8> Loads;
629 SmallVector<StoreInst *, 8> Stores;
630
631 const DominatorTree &DT = DTCallback(F);
632 const PostDominatorTree &PDT = PDTCallback(F);
633 bool IsLeafFunc = true;
634
635 for (auto &BB : F) {
636 if (shouldInstrumentBlock(F, BB: &BB, DT, PDT, Options))
637 BlocksToInstrument.push_back(Elt: &BB);
638 for (auto &Inst : BB) {
639 if (Options.IndirectCalls) {
640 CallBase *CB = dyn_cast<CallBase>(Val: &Inst);
641 if (CB && CB->isIndirectCall())
642 IndirCalls.push_back(Elt: &Inst);
643 }
644 if (Options.TraceCmp) {
645 if (ICmpInst *CMP = dyn_cast<ICmpInst>(Val: &Inst))
646 if (IsInterestingCmp(CMP, DT, Options))
647 CmpTraceTargets.push_back(Elt: &Inst);
648 if (isa<SwitchInst>(Val: &Inst))
649 SwitchTraceTargets.push_back(Elt: &Inst);
650 }
651 if (Options.TraceDiv)
652 if (BinaryOperator *BO = dyn_cast<BinaryOperator>(Val: &Inst))
653 if (BO->getOpcode() == Instruction::SDiv ||
654 BO->getOpcode() == Instruction::UDiv)
655 DivTraceTargets.push_back(Elt: BO);
656 if (Options.TraceGep)
657 if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Val: &Inst))
658 GepTraceTargets.push_back(Elt: GEP);
659 if (Options.TraceLoads)
660 if (LoadInst *LI = dyn_cast<LoadInst>(Val: &Inst))
661 Loads.push_back(Elt: LI);
662 if (Options.TraceStores)
663 if (StoreInst *SI = dyn_cast<StoreInst>(Val: &Inst))
664 Stores.push_back(Elt: SI);
665 if (Options.StackDepth)
666 if (isa<InvokeInst>(Val: Inst) ||
667 (isa<CallInst>(Val: Inst) && !isa<IntrinsicInst>(Val: Inst)))
668 IsLeafFunc = false;
669 }
670 }
671
672 if (Options.CollectControlFlow)
673 createFunctionControlFlow(F);
674
675 Value *FunctionGateCmp = nullptr;
676 InjectCoverage(F, AllBlocks: BlocksToInstrument, FunctionGateCmp, IsLeafFunc);
677 InjectCoverageForIndirectCalls(F, IndirCalls);
678 InjectTraceForCmp(F, CmpTraceTargets, FunctionGateCmp);
679 InjectTraceForSwitch(F, SwitchTraceTargets, FunctionGateCmp);
680 InjectTraceForDiv(F, DivTraceTargets);
681 InjectTraceForGep(F, GepTraceTargets);
682 InjectTraceForLoadsAndStores(F, Loads, Stores);
683
684 if (Options.TracePCEntryExit)
685 InjectTraceForExits(F);
686}
687
688GlobalVariable *ModuleSanitizerCoverage::CreateFunctionLocalArrayInSection(
689 size_t NumElements, Function &F, Type *Ty, const char *Section) {
690 ArrayType *ArrayTy = ArrayType::get(ElementType: Ty, NumElements);
691 auto Array = new GlobalVariable(
692 *CurModule, ArrayTy, false, GlobalVariable::PrivateLinkage,
693 Constant::getNullValue(Ty: ArrayTy), "__sancov_gen_");
694
695 // sancov_pcs parallels the other arrays, so they must be retained or
696 // discarded together. Put them in F's comdat to tie them to F for linker GC
697 // benefit. Outside ELF (nodeduplicate), a new comdat for an interposable F
698 // could prevail over a strong definition (COFF: the weak external becomes a
699 // COMDAT definition; Wasm: comdats deduplicate first). noipa doesn't affect
700 // symbol resolution.
701 if (TargetTriple.supportsCOMDAT() &&
702 (F.hasComdat() || TargetTriple.isOSBinFormatELF() ||
703 !F.isInterposable(/*CheckNoIPA=*/false)))
704 if (auto Comdat = getOrCreateFunctionComdat(F, T&: TargetTriple))
705 Array->setComdat(Comdat);
706 Array->setSection(getSectionName(Section));
707 Array->setAlignment(Align(DL->getTypeStoreSize(Ty).getFixedValue()));
708
709 // Optimizers (e.g. GlobalOpt/ConstantMerge) may not discard the arrays as a
710 // unit, so retain them in the compiler; without a comdat, in the linker too.
711 if (Array->hasComdat())
712 GlobalsToAppendToCompilerUsed.push_back(Elt: Array);
713 else
714 GlobalsToAppendToUsed.push_back(Elt: Array);
715
716 return Array;
717}
718
719GlobalVariable *
720ModuleSanitizerCoverage::CreatePCArray(Function &F,
721 ArrayRef<BasicBlock *> AllBlocks) {
722 size_t N = AllBlocks.size();
723 assert(N);
724 SmallVector<Constant *, 32> PCs;
725 IRBuilder<> IRB(&*F.getEntryBlock().getFirstInsertionPt());
726 for (size_t i = 0; i < N; i++) {
727 if (&F.getEntryBlock() == AllBlocks[i]) {
728 PCs.push_back(Elt: (Constant *)IRB.CreatePointerCast(V: &F, DestTy: PtrTy));
729 PCs.push_back(
730 Elt: (Constant *)IRB.CreateIntToPtr(V: ConstantInt::get(Ty: IntptrTy, V: 1), DestTy: PtrTy));
731 } else {
732 PCs.push_back(Elt: (Constant *)IRB.CreatePointerCast(
733 V: BlockAddress::get(BB: AllBlocks[i]), DestTy: PtrTy));
734 PCs.push_back(Elt: Constant::getNullValue(Ty: PtrTy));
735 }
736 }
737 auto *PCArray =
738 CreateFunctionLocalArrayInSection(NumElements: N * 2, F, Ty: PtrTy, Section: SanCovPCsSectionName);
739 PCArray->setInitializer(
740 ConstantArray::get(T: ArrayType::get(ElementType: PtrTy, NumElements: N * 2), V: PCs));
741 PCArray->setConstant(true);
742
743 return PCArray;
744}
745
746void ModuleSanitizerCoverage::CreateFunctionLocalArrays(
747 Function &F, ArrayRef<BasicBlock *> AllBlocks) {
748 if (Options.TracePCGuard)
749 FunctionGuardArray = CreateFunctionLocalArrayInSection(
750 NumElements: AllBlocks.size(), F, Ty: Int32Ty, Section: SanCovGuardsSectionName);
751
752 if (Options.Inline8bitCounters)
753 Function8bitCounterArray = CreateFunctionLocalArrayInSection(
754 NumElements: AllBlocks.size(), F, Ty: Int8Ty, Section: SanCovCountersSectionName);
755 if (Options.InlineBoolFlag)
756 FunctionBoolArray = CreateFunctionLocalArrayInSection(
757 NumElements: AllBlocks.size(), F, Ty: Int1Ty, Section: SanCovBoolFlagSectionName);
758
759 if (Options.PCTable)
760 FunctionPCsArray = CreatePCArray(F, AllBlocks);
761}
762
763Value *ModuleSanitizerCoverage::CreateFunctionLocalGateCmp(IRBuilder<> &IRB) {
764 auto Load = IRB.CreateLoad(Ty: Int64Ty, Ptr: SanCovCallbackGate);
765 Load->setNoSanitizeMetadata();
766 auto Cmp = IRB.CreateIsNotNull(Arg: Load);
767 Cmp->setName("sancov gate cmp");
768 return Cmp;
769}
770
771Instruction *ModuleSanitizerCoverage::CreateGateBranch(Function &F,
772 Value *&FunctionGateCmp,
773 Instruction *IP) {
774 if (!FunctionGateCmp) {
775 // Create this in the entry block
776 BasicBlock &BB = F.getEntryBlock();
777 BasicBlock::iterator IP = BB.getFirstInsertionPt();
778 IP = PrepareToSplitEntryBlock(BB, IP);
779 IRBuilder<> EntryIRB(&*IP);
780 FunctionGateCmp = CreateFunctionLocalGateCmp(IRB&: EntryIRB);
781 }
782 // Set the branch weights in order to minimize the price paid when the
783 // gate is turned off, allowing the default enablement of this
784 // instrumentation with as little of a performance cost as possible
785 auto Weights = MDBuilder(*C).createBranchWeights(TrueWeight: 1, FalseWeight: 100000);
786 return SplitBlockAndInsertIfThen(Cond: FunctionGateCmp, SplitBefore: IP, Unreachable: false, BranchWeights: Weights);
787}
788
789bool ModuleSanitizerCoverage::InjectCoverage(Function &F,
790 ArrayRef<BasicBlock *> AllBlocks,
791 Value *&FunctionGateCmp,
792 bool IsLeafFunc) {
793 if (AllBlocks.empty())
794 return false;
795 CreateFunctionLocalArrays(F, AllBlocks);
796 for (size_t i = 0, N = AllBlocks.size(); i < N; i++)
797 InjectCoverageAtBlock(F, BB&: *AllBlocks[i], Idx: i, FunctionGateCmp, IsLeafFunc);
798
799 return true;
800}
801
802// On every indirect call we call a run-time function
803// __sanitizer_cov_indir_call* with two parameters:
804// - callee address,
805// - global cache array that contains CacheSize pointers (zero-initialized).
806// The cache is used to speed up recording the caller-callee pairs.
807// The address of the caller is passed implicitly via caller PC.
808// CacheSize is encoded in the name of the run-time function.
809void ModuleSanitizerCoverage::InjectCoverageForIndirectCalls(
810 Function &F, ArrayRef<Instruction *> IndirCalls) {
811 if (IndirCalls.empty())
812 return;
813 assert(Options.TracePC || Options.TracePCEntryExit || Options.TracePCGuard ||
814 Options.Inline8bitCounters || Options.InlineBoolFlag);
815 for (auto *I : IndirCalls) {
816 InstrumentationIRBuilder IRB(I);
817 CallBase &CB = cast<CallBase>(Val&: *I);
818 Value *Callee = CB.getCalledOperand();
819 if (isa<InlineAsm>(Val: Callee))
820 continue;
821 IRB.CreateCall(Callee: SanCovTracePCIndir, Args: IRB.CreatePointerCast(V: Callee, DestTy: IntptrTy));
822 }
823}
824
825// For every switch statement we insert a call:
826// __sanitizer_cov_trace_switch(CondValue,
827// {NumCases, ValueSizeInBits, Case0Value, Case1Value, Case2Value, ... })
828
829void ModuleSanitizerCoverage::InjectTraceForSwitch(
830 Function &F, ArrayRef<Instruction *> SwitchTraceTargets,
831 Value *&FunctionGateCmp) {
832 for (auto *I : SwitchTraceTargets) {
833 if (SwitchInst *SI = dyn_cast<SwitchInst>(Val: I)) {
834 InstrumentationIRBuilder IRB(I);
835 SmallVector<Constant *, 16> Initializers;
836 Value *Cond = SI->getCondition();
837 if (Cond->getType()->getScalarSizeInBits() >
838 Int64Ty->getScalarSizeInBits())
839 continue;
840 Initializers.push_back(Elt: ConstantInt::get(Ty: Int64Ty, V: SI->getNumCases()));
841 Initializers.push_back(
842 Elt: ConstantInt::get(Ty: Int64Ty, V: Cond->getType()->getScalarSizeInBits()));
843 if (Cond->getType()->getScalarSizeInBits() <
844 Int64Ty->getScalarSizeInBits())
845 Cond = IRB.CreateIntCast(V: Cond, DestTy: Int64Ty, isSigned: false);
846 for (auto It : SI->cases()) {
847 ConstantInt *C = It.getCaseValue();
848 if (C->getType()->getScalarSizeInBits() < 64)
849 C = ConstantInt::get(Context&: C->getContext(), V: C->getValue().zext(width: 64));
850 Initializers.push_back(Elt: C);
851 }
852 llvm::sort(C: drop_begin(RangeOrContainer&: Initializers, N: 2),
853 Comp: [](const Constant *A, const Constant *B) {
854 return cast<ConstantInt>(Val: A)->getLimitedValue() <
855 cast<ConstantInt>(Val: B)->getLimitedValue();
856 });
857 ArrayType *ArrayOfInt64Ty = ArrayType::get(ElementType: Int64Ty, NumElements: Initializers.size());
858 GlobalVariable *GV = new GlobalVariable(
859 *CurModule, ArrayOfInt64Ty, false, GlobalVariable::InternalLinkage,
860 ConstantArray::get(T: ArrayOfInt64Ty, V: Initializers),
861 "__sancov_gen_cov_switch_values");
862 if (Options.GatedCallbacks) {
863 auto GateBranch = CreateGateBranch(F, FunctionGateCmp, IP: I);
864 IRBuilder<> GateIRB(GateBranch);
865 GateIRB.CreateCall(Callee: SanCovTraceSwitchFunction, Args: {Cond, GV});
866 } else {
867 IRB.CreateCall(Callee: SanCovTraceSwitchFunction, Args: {Cond, GV});
868 }
869 }
870 }
871}
872
873void ModuleSanitizerCoverage::InjectTraceForDiv(
874 Function &, ArrayRef<BinaryOperator *> DivTraceTargets) {
875 for (auto *BO : DivTraceTargets) {
876 InstrumentationIRBuilder IRB(BO);
877 Value *A1 = BO->getOperand(i_nocapture: 1);
878 if (isa<ConstantInt>(Val: A1))
879 continue;
880 if (!A1->getType()->isIntegerTy())
881 continue;
882 uint64_t TypeSize = DL->getTypeStoreSizeInBits(Ty: A1->getType());
883 int CallbackIdx = TypeSize == 32 ? 0 : TypeSize == 64 ? 1 : -1;
884 if (CallbackIdx < 0)
885 continue;
886 auto Ty = Type::getIntNTy(C&: *C, N: TypeSize);
887 IRB.CreateCall(Callee: SanCovTraceDivFunction[CallbackIdx],
888 Args: {IRB.CreateIntCast(V: A1, DestTy: Ty, isSigned: true)});
889 }
890}
891
892void ModuleSanitizerCoverage::InjectTraceForGep(
893 Function &, ArrayRef<GetElementPtrInst *> GepTraceTargets) {
894 for (auto *GEP : GepTraceTargets) {
895 InstrumentationIRBuilder IRB(GEP);
896 for (Use &Idx : GEP->indices())
897 if (!isa<ConstantInt>(Val: Idx) && Idx->getType()->isIntegerTy())
898 IRB.CreateCall(Callee: SanCovTraceGepFunction,
899 Args: {IRB.CreateIntCast(V: Idx, DestTy: IntptrTy, isSigned: true)});
900 }
901}
902
903void ModuleSanitizerCoverage::InjectTraceForLoadsAndStores(
904 Function &, ArrayRef<LoadInst *> Loads, ArrayRef<StoreInst *> Stores) {
905 auto CallbackIdx = [&](Type *ElementTy) -> int {
906 uint64_t TypeSize = DL->getTypeStoreSizeInBits(Ty: ElementTy);
907 return TypeSize == 8 ? 0
908 : TypeSize == 16 ? 1
909 : TypeSize == 32 ? 2
910 : TypeSize == 64 ? 3
911 : TypeSize == 128 ? 4
912 : -1;
913 };
914 for (auto *LI : Loads) {
915 InstrumentationIRBuilder IRB(LI);
916 auto Ptr = LI->getPointerOperand();
917 int Idx = CallbackIdx(LI->getType());
918 if (Idx < 0)
919 continue;
920 IRB.CreateCall(Callee: SanCovLoadFunction[Idx], Args: Ptr);
921 }
922 for (auto *SI : Stores) {
923 InstrumentationIRBuilder IRB(SI);
924 auto Ptr = SI->getPointerOperand();
925 int Idx = CallbackIdx(SI->getValueOperand()->getType());
926 if (Idx < 0)
927 continue;
928 IRB.CreateCall(Callee: SanCovStoreFunction[Idx], Args: Ptr);
929 }
930}
931
932void ModuleSanitizerCoverage::InjectTraceForExits(Function &F) {
933 EscapeEnumerator EE(F, "sancov_exit");
934 while (IRBuilder<> *AtExit = EE.Next()) {
935 InstrumentationIRBuilder::ensureDebugInfo(IRB&: *AtExit, F);
936 AtExit->CreateCall(Callee: SanCovTracePCExit, Args: {})
937 ->setTailCallKind(CallInst::TCK_NoTail);
938 }
939}
940
941void ModuleSanitizerCoverage::InjectTraceForCmp(
942 Function &F, ArrayRef<Instruction *> CmpTraceTargets,
943 Value *&FunctionGateCmp) {
944 for (auto *I : CmpTraceTargets) {
945 if (ICmpInst *ICMP = dyn_cast<ICmpInst>(Val: I)) {
946 InstrumentationIRBuilder IRB(ICMP);
947 Value *A0 = ICMP->getOperand(i_nocapture: 0);
948 Value *A1 = ICMP->getOperand(i_nocapture: 1);
949 if (!A0->getType()->isIntegerTy())
950 continue;
951 uint64_t TypeSize = DL->getTypeStoreSizeInBits(Ty: A0->getType());
952 int CallbackIdx = TypeSize == 8 ? 0
953 : TypeSize == 16 ? 1
954 : TypeSize == 32 ? 2
955 : TypeSize == 64 ? 3
956 : -1;
957 if (CallbackIdx < 0)
958 continue;
959 // __sanitizer_cov_trace_cmp((type_size << 32) | predicate, A0, A1);
960 auto CallbackFunc = SanCovTraceCmpFunction[CallbackIdx];
961 bool FirstIsConst = isa<ConstantInt>(Val: A0);
962 bool SecondIsConst = isa<ConstantInt>(Val: A1);
963 // If both are const, then we don't need such a comparison.
964 if (FirstIsConst && SecondIsConst)
965 continue;
966 // If only one is const, then make it the first callback argument.
967 if (FirstIsConst || SecondIsConst) {
968 CallbackFunc = SanCovTraceConstCmpFunction[CallbackIdx];
969 if (SecondIsConst)
970 std::swap(a&: A0, b&: A1);
971 }
972
973 auto Ty = Type::getIntNTy(C&: *C, N: TypeSize);
974 if (Options.GatedCallbacks) {
975 auto GateBranch = CreateGateBranch(F, FunctionGateCmp, IP: I);
976 IRBuilder<> GateIRB(GateBranch);
977 GateIRB.CreateCall(Callee: CallbackFunc, Args: {GateIRB.CreateIntCast(V: A0, DestTy: Ty, isSigned: true),
978 GateIRB.CreateIntCast(V: A1, DestTy: Ty, isSigned: true)});
979 } else {
980 IRB.CreateCall(Callee: CallbackFunc, Args: {IRB.CreateIntCast(V: A0, DestTy: Ty, isSigned: true),
981 IRB.CreateIntCast(V: A1, DestTy: Ty, isSigned: true)});
982 }
983 }
984 }
985}
986
987void ModuleSanitizerCoverage::InjectCoverageAtBlock(Function &F, BasicBlock &BB,
988 size_t Idx,
989 Value *&FunctionGateCmp,
990 bool IsLeafFunc) {
991 BasicBlock::iterator IP = BB.getFirstInsertionPt();
992 bool IsEntryBB = &BB == &F.getEntryBlock();
993 DebugLoc EntryLoc;
994 if (IsEntryBB) {
995 if (auto SP = F.getSubprogram())
996 EntryLoc = DILocation::get(Context&: SP->getContext(), Line: SP->getScopeLine(), Column: 0, Scope: SP);
997 // Keep static allocas and llvm.localescape calls in the entry block. Even
998 // if we aren't splitting the block, it's nice for allocas to be before
999 // calls.
1000 IP = PrepareToSplitEntryBlock(BB, IP);
1001 }
1002
1003 InstrumentationIRBuilder IRB(&*IP);
1004 if (EntryLoc)
1005 IRB.SetCurrentDebugLocation(EntryLoc);
1006 if (Options.TracePC || (IsEntryBB && Options.TracePCEntryExit)) {
1007 FunctionCallee Callee = IsEntryBB && Options.TracePCEntryExit
1008 ? SanCovTracePCEntry
1009 : SanCovTracePC;
1010 IRB.CreateCall(Callee)
1011 ->setCannotMerge(); // gets the PC using GET_CALLER_PC.
1012 }
1013 if (Options.TracePCGuard) {
1014 auto GuardPtr = IRB.CreateConstInBoundsGEP2_64(
1015 Ty: FunctionGuardArray->getValueType(), Ptr: FunctionGuardArray, Idx0: 0, Idx1: Idx);
1016 if (Options.GatedCallbacks) {
1017 Instruction *I = &*IP;
1018 auto GateBranch = CreateGateBranch(F, FunctionGateCmp, IP: I);
1019 IRBuilder<> GateIRB(GateBranch);
1020 GateIRB.CreateCall(Callee: SanCovTracePCGuard, Args: GuardPtr)->setCannotMerge();
1021 } else {
1022 IRB.CreateCall(Callee: SanCovTracePCGuard, Args: GuardPtr)->setCannotMerge();
1023 }
1024 }
1025 if (Options.Inline8bitCounters) {
1026 auto CounterPtr = IRB.CreateGEP(
1027 Ty: Function8bitCounterArray->getValueType(), Ptr: Function8bitCounterArray,
1028 IdxList: {ConstantInt::get(Ty: IntptrTy, V: 0), ConstantInt::get(Ty: IntptrTy, V: Idx)});
1029 auto Load = IRB.CreateLoad(Ty: Int8Ty, Ptr: CounterPtr);
1030 auto Inc = IRB.CreateAdd(LHS: Load, RHS: ConstantInt::get(Ty: Int8Ty, V: 1));
1031 auto Store = IRB.CreateStore(Val: Inc, Ptr: CounterPtr);
1032 Load->setNoSanitizeMetadata();
1033 Store->setNoSanitizeMetadata();
1034 }
1035 if (Options.InlineBoolFlag) {
1036 auto FlagPtr = IRB.CreateGEP(
1037 Ty: FunctionBoolArray->getValueType(), Ptr: FunctionBoolArray,
1038 IdxList: {ConstantInt::get(Ty: IntptrTy, V: 0), ConstantInt::get(Ty: IntptrTy, V: Idx)});
1039 auto Load = IRB.CreateLoad(Ty: Int1Ty, Ptr: FlagPtr);
1040 auto ThenTerm = SplitBlockAndInsertIfThen(
1041 Cond: IRB.CreateIsNull(Arg: Load), SplitBefore: &*IP, Unreachable: false,
1042 BranchWeights: MDBuilder(IRB.getContext()).createUnlikelyBranchWeights());
1043 InstrumentationIRBuilder ThenIRB(ThenTerm);
1044 auto Store = ThenIRB.CreateStore(Val: ConstantInt::getTrue(Ty: Int1Ty), Ptr: FlagPtr);
1045 if (EntryLoc)
1046 Store->setDebugLoc(EntryLoc);
1047 Load->setNoSanitizeMetadata();
1048 Store->setNoSanitizeMetadata();
1049 }
1050 if (Options.StackDepth && IsEntryBB && !IsLeafFunc) {
1051 Module *M = F.getParent();
1052 const DataLayout &DL = M->getDataLayout();
1053
1054 if (Options.StackDepthCallbackMin) {
1055 // In callback mode, only add call when stack depth reaches minimum.
1056 int EstimatedStackSize = 0;
1057 // If dynamic alloca found, always add call.
1058 bool HasDynamicAlloc = false;
1059 // Find an insertion point after last "alloca".
1060 llvm::Instruction *InsertBefore = nullptr;
1061
1062 // Examine all allocas in the basic block. since we're too early
1063 // to have results from Intrinsic::frameaddress, we have to manually
1064 // estimate the stack size.
1065 for (auto &I : BB) {
1066 if (auto *AI = dyn_cast<AllocaInst>(Val: &I)) {
1067 // Move potential insertion point past the "alloca".
1068 InsertBefore = AI->getNextNode();
1069
1070 // Make an estimate on the stack usage.
1071 if (auto AllocaSize = AI->getAllocationSize(DL)) {
1072 if (AllocaSize->isFixed())
1073 EstimatedStackSize += AllocaSize->getFixedValue();
1074 else
1075 HasDynamicAlloc = true;
1076 } else {
1077 HasDynamicAlloc = true;
1078 }
1079 }
1080 }
1081
1082 if (HasDynamicAlloc ||
1083 EstimatedStackSize >= Options.StackDepthCallbackMin) {
1084 if (InsertBefore)
1085 IRB.SetInsertPoint(InsertBefore);
1086 auto Call = IRB.CreateCall(Callee: SanCovStackDepthCallback);
1087 if (EntryLoc)
1088 Call->setDebugLoc(EntryLoc);
1089 Call->setCannotMerge();
1090 }
1091 } else {
1092 // Check stack depth. If it's the deepest so far, record it.
1093 auto FrameAddrPtr = IRB.CreateIntrinsic(
1094 ID: Intrinsic::frameaddress, OverloadTypes: IRB.getPtrTy(AddrSpace: DL.getAllocaAddrSpace()),
1095 Args: {Constant::getNullValue(Ty: Int32Ty)});
1096 auto FrameAddrInt = IRB.CreatePtrToInt(V: FrameAddrPtr, DestTy: IntptrTy);
1097 auto LowestStack = IRB.CreateLoad(Ty: IntptrTy, Ptr: SanCovLowestStack);
1098 auto IsStackLower = IRB.CreateICmpULT(LHS: FrameAddrInt, RHS: LowestStack);
1099 auto ThenTerm = SplitBlockAndInsertIfThen(
1100 Cond: IsStackLower, SplitBefore: &*IP, Unreachable: false,
1101 BranchWeights: MDBuilder(IRB.getContext()).createUnlikelyBranchWeights());
1102 InstrumentationIRBuilder ThenIRB(ThenTerm);
1103 auto Store = ThenIRB.CreateStore(Val: FrameAddrInt, Ptr: SanCovLowestStack);
1104 if (EntryLoc)
1105 Store->setDebugLoc(EntryLoc);
1106 LowestStack->setNoSanitizeMetadata();
1107 Store->setNoSanitizeMetadata();
1108 }
1109 }
1110}
1111
1112std::string
1113ModuleSanitizerCoverage::getSectionName(const std::string &Section) const {
1114 if (TargetTriple.isOSBinFormatCOFF()) {
1115 if (Section == SanCovCountersSectionName)
1116 return ".SCOV$CM";
1117 if (Section == SanCovBoolFlagSectionName)
1118 return ".SCOV$BM";
1119 if (Section == SanCovPCsSectionName)
1120 return ".SCOVP$M";
1121 return ".SCOV$GM"; // For SanCovGuardsSectionName.
1122 }
1123 if (TargetTriple.isOSBinFormatMachO())
1124 return "__DATA,__" + Section;
1125 return "__" + Section;
1126}
1127
1128std::string
1129ModuleSanitizerCoverage::getSectionStart(const std::string &Section) const {
1130 if (TargetTriple.isOSBinFormatMachO())
1131 return "\1section$start$__DATA$__" + Section;
1132 return "__start___" + Section;
1133}
1134
1135std::string
1136ModuleSanitizerCoverage::getSectionEnd(const std::string &Section) const {
1137 if (TargetTriple.isOSBinFormatMachO())
1138 return "\1section$end$__DATA$__" + Section;
1139 return "__stop___" + Section;
1140}
1141
1142void ModuleSanitizerCoverage::createFunctionControlFlow(Function &F) {
1143 SmallVector<Constant *, 32> CFs;
1144 IRBuilder<> IRB(&*F.getEntryBlock().getFirstInsertionPt());
1145
1146 for (auto &BB : F) {
1147 // blockaddress can not be used on function's entry block.
1148 if (&BB == &F.getEntryBlock())
1149 CFs.push_back(Elt: (Constant *)IRB.CreatePointerCast(V: &F, DestTy: PtrTy));
1150 else
1151 CFs.push_back(
1152 Elt: (Constant *)IRB.CreatePointerCast(V: BlockAddress::get(BB: &BB), DestTy: PtrTy));
1153
1154 for (auto SuccBB : successors(BB: &BB)) {
1155 assert(SuccBB != &F.getEntryBlock());
1156 CFs.push_back(
1157 Elt: (Constant *)IRB.CreatePointerCast(V: BlockAddress::get(BB: SuccBB), DestTy: PtrTy));
1158 }
1159
1160 CFs.push_back(Elt: (Constant *)Constant::getNullValue(Ty: PtrTy));
1161
1162 for (auto &Inst : BB) {
1163 if (CallBase *CB = dyn_cast<CallBase>(Val: &Inst)) {
1164 if (CB->isIndirectCall()) {
1165 // TODO(navidem): handle indirect calls, for now mark its existence.
1166 CFs.push_back(Elt: (Constant *)IRB.CreateIntToPtr(
1167 V: ConstantInt::getAllOnesValue(Ty: IntptrTy), DestTy: PtrTy));
1168 } else {
1169 auto CalledF = CB->getCalledFunction();
1170 if (CalledF && !CalledF->isIntrinsic())
1171 CFs.push_back(Elt: (Constant *)IRB.CreatePointerCast(V: CalledF, DestTy: PtrTy));
1172 }
1173 }
1174 }
1175
1176 CFs.push_back(Elt: (Constant *)Constant::getNullValue(Ty: PtrTy));
1177 }
1178
1179 FunctionCFsArray = CreateFunctionLocalArrayInSection(NumElements: CFs.size(), F, Ty: PtrTy,
1180 Section: SanCovCFsSectionName);
1181 FunctionCFsArray->setInitializer(
1182 ConstantArray::get(T: ArrayType::get(ElementType: PtrTy, NumElements: CFs.size()), V: CFs));
1183 FunctionCFsArray->setConstant(true);
1184}
1185