1//===-- SanitizerCoverage.cpp - coverage instrumentation for sanitizers ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Coverage instrumentation done on LLVM IR level, works with Sanitizers.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/Transforms/Instrumentation/SanitizerCoverage.h"
14#include "InstrumentationOptions.h"
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/ADT/SmallVector.h"
17#include "llvm/Analysis/GlobalsModRef.h"
18#include "llvm/Analysis/PostDominators.h"
19#include "llvm/IR/Constant.h"
20#include "llvm/IR/Constants.h"
21#include "llvm/IR/DataLayout.h"
22#include "llvm/IR/Dominators.h"
23#include "llvm/IR/EHPersonalities.h"
24#include "llvm/IR/Function.h"
25#include "llvm/IR/GlobalVariable.h"
26#include "llvm/IR/IRBuilder.h"
27#include "llvm/IR/IntrinsicInst.h"
28#include "llvm/IR/Intrinsics.h"
29#include "llvm/IR/LLVMContext.h"
30#include "llvm/IR/MDBuilder.h"
31#include "llvm/IR/Module.h"
32#include "llvm/IR/Type.h"
33#include "llvm/IR/ValueSymbolTable.h"
34#include "llvm/Support/SpecialCaseList.h"
35#include "llvm/Support/VirtualFileSystem.h"
36#include "llvm/TargetParser/Triple.h"
37#include "llvm/Transforms/Utils/BasicBlockUtils.h"
38#include "llvm/Transforms/Utils/EscapeEnumerator.h"
39#include "llvm/Transforms/Utils/ModuleUtils.h"
40
41using namespace llvm;
42
43#define DEBUG_TYPE "sancov"
44
45const char SanCovTracePCIndirName[] = "__sanitizer_cov_trace_pc_indir";
46const char SanCovTracePCName[] = "__sanitizer_cov_trace_pc";
47const char SanCovTracePCEntryName[] = "__sanitizer_cov_trace_pc_entry";
48const char SanCovTracePCExitName[] = "__sanitizer_cov_trace_pc_exit";
49const char SanCovTraceCmp1[] = "__sanitizer_cov_trace_cmp1";
50const char SanCovTraceCmp2[] = "__sanitizer_cov_trace_cmp2";
51const char SanCovTraceCmp4[] = "__sanitizer_cov_trace_cmp4";
52const char SanCovTraceCmp8[] = "__sanitizer_cov_trace_cmp8";
53const char SanCovTraceConstCmp1[] = "__sanitizer_cov_trace_const_cmp1";
54const char SanCovTraceConstCmp2[] = "__sanitizer_cov_trace_const_cmp2";
55const char SanCovTraceConstCmp4[] = "__sanitizer_cov_trace_const_cmp4";
56const char SanCovTraceConstCmp8[] = "__sanitizer_cov_trace_const_cmp8";
57const char SanCovLoad1[] = "__sanitizer_cov_load1";
58const char SanCovLoad2[] = "__sanitizer_cov_load2";
59const char SanCovLoad4[] = "__sanitizer_cov_load4";
60const char SanCovLoad8[] = "__sanitizer_cov_load8";
61const char SanCovLoad16[] = "__sanitizer_cov_load16";
62const char SanCovStore1[] = "__sanitizer_cov_store1";
63const char SanCovStore2[] = "__sanitizer_cov_store2";
64const char SanCovStore4[] = "__sanitizer_cov_store4";
65const char SanCovStore8[] = "__sanitizer_cov_store8";
66const char SanCovStore16[] = "__sanitizer_cov_store16";
67const char SanCovTraceDiv4[] = "__sanitizer_cov_trace_div4";
68const char SanCovTraceDiv8[] = "__sanitizer_cov_trace_div8";
69const char SanCovTraceGep[] = "__sanitizer_cov_trace_gep";
70const char SanCovTraceSwitchName[] = "__sanitizer_cov_trace_switch";
71const char SanCovModuleCtorTracePcGuardName[] =
72 "sancov.module_ctor_trace_pc_guard";
73const char SanCovModuleCtor8bitCountersName[] =
74 "sancov.module_ctor_8bit_counters";
75const char SanCovModuleCtorBoolFlagName[] = "sancov.module_ctor_bool_flag";
76static const uint64_t SanCtorAndDtorPriority = 2;
77
78const char SanCovTracePCGuardName[] = "__sanitizer_cov_trace_pc_guard";
79const char SanCovTracePCGuardInitName[] = "__sanitizer_cov_trace_pc_guard_init";
80const char SanCov8bitCountersInitName[] = "__sanitizer_cov_8bit_counters_init";
81const char SanCovBoolFlagInitName[] = "__sanitizer_cov_bool_flag_init";
82const char SanCovPCsInitName[] = "__sanitizer_cov_pcs_init";
83const char SanCovCFsInitName[] = "__sanitizer_cov_cfs_init";
84
85const char SanCovGuardsSectionName[] = "sancov_guards";
86const char SanCovCountersSectionName[] = "sancov_cntrs";
87const char SanCovBoolFlagSectionName[] = "sancov_bools";
88const char SanCovPCsSectionName[] = "sancov_pcs";
89const char SanCovCFsSectionName[] = "sancov_cfs";
90const char SanCovCallbackGateSectionName[] = "sancov_gate";
91
92const char SanCovStackDepthCallbackName[] = "__sanitizer_cov_stack_depth";
93const char SanCovLowestStackName[] = "__sancov_lowest_stack";
94const char SanCovCallbackGateName[] = "__sancov_should_track";
95
96namespace {
97
98SanitizerCoverageOptions getOptions(int LegacyCoverageLevel) {
99 SanitizerCoverageOptions Res;
100 switch (LegacyCoverageLevel) {
101 case 0:
102 Res.CoverageType = SanitizerCoverageOptions::SCK_None;
103 break;
104 case 1:
105 Res.CoverageType = SanitizerCoverageOptions::SCK_Function;
106 break;
107 case 2:
108 Res.CoverageType = SanitizerCoverageOptions::SCK_BB;
109 break;
110 case 3:
111 Res.CoverageType = SanitizerCoverageOptions::SCK_Edge;
112 break;
113 case 4:
114 Res.CoverageType = SanitizerCoverageOptions::SCK_Edge;
115 Res.IndirectCalls = true;
116 break;
117 }
118 return Res;
119}
120
121SanitizerCoverageOptions overrideFromCL(const InstrumentationOptions &Opts,
122 SanitizerCoverageOptions Options) {
123 // Sets CoverageType and IndirectCalls.
124 SanitizerCoverageOptions CLOpts = getOptions(LegacyCoverageLevel: Opts.sanitizer_coverage_level);
125 Options.CoverageType = std::max(a: Options.CoverageType, b: CLOpts.CoverageType);
126 Options.IndirectCalls |= CLOpts.IndirectCalls;
127 Options.TraceCmp |= Opts.sanitizer_coverage_trace_compares;
128 Options.TraceDiv |= Opts.sanitizer_coverage_trace_divs;
129 Options.TraceGep |= Opts.sanitizer_coverage_trace_geps;
130 Options.TracePC |= Opts.sanitizer_coverage_trace_pc;
131 Options.TracePCEntryExit |= Opts.sanitizer_coverage_trace_pc_entry_exit;
132 Options.TracePCGuard |= Opts.sanitizer_coverage_trace_pc_guard;
133 Options.Inline8bitCounters |= Opts.sanitizer_coverage_inline_8bit_counters;
134 Options.InlineBoolFlag |= Opts.sanitizer_coverage_inline_bool_flag;
135 Options.PCTable |= Opts.sanitizer_coverage_pc_table;
136 Options.NoPrune |= !Opts.sanitizer_coverage_prune_blocks;
137 Options.StackDepth |= Opts.sanitizer_coverage_stack_depth;
138 Options.StackDepthCallbackMin =
139 std::max(a: Options.StackDepthCallbackMin,
140 b: Opts.sanitizer_coverage_stack_depth_callback_min);
141 Options.TraceLoads |= Opts.sanitizer_coverage_trace_loads;
142 Options.TraceStores |= Opts.sanitizer_coverage_trace_stores;
143 Options.GatedCallbacks |= Opts.sanitizer_coverage_gated_trace_callbacks;
144 if (!Options.TracePCGuard && !Options.TracePC && !Options.TracePCEntryExit &&
145 !Options.Inline8bitCounters && !Options.StackDepth &&
146 !Options.InlineBoolFlag && !Options.TraceLoads && !Options.TraceStores)
147 Options.TracePCGuard = true; // TracePCGuard is default.
148 Options.CollectControlFlow |= Opts.sanitizer_coverage_control_flow;
149 return Options;
150}
151
152class ModuleSanitizerCoverage {
153public:
154 using DomTreeCallback = function_ref<const DominatorTree &(Function &F)>;
155 using PostDomTreeCallback =
156 function_ref<const PostDominatorTree &(Function &F)>;
157
158 ModuleSanitizerCoverage(const InstrumentationOptions &Opts, Module &M,
159 DomTreeCallback DTCallback,
160 PostDomTreeCallback PDTCallback,
161 const SanitizerCoverageOptions &Options,
162 const SpecialCaseList *Allowlist,
163 const SpecialCaseList *Blocklist)
164 : Opts(Opts), M(M), DTCallback(DTCallback), PDTCallback(PDTCallback),
165 Options(Options), Allowlist(Allowlist), Blocklist(Blocklist) {}
166
167 bool instrumentModule();
168
169private:
170 void createFunctionControlFlow(Function &F);
171 void instrumentFunction(Function &F);
172 void InjectCoverageForIndirectCalls(Function &F,
173 ArrayRef<Instruction *> IndirCalls);
174 void InjectTraceForCmp(Function &F, ArrayRef<Instruction *> CmpTraceTargets,
175 Value *&FunctionGateCmp);
176 void InjectTraceForDiv(Function &F,
177 ArrayRef<BinaryOperator *> DivTraceTargets);
178 void InjectTraceForGep(Function &F,
179 ArrayRef<GetElementPtrInst *> GepTraceTargets);
180 void InjectTraceForLoadsAndStores(Function &F, ArrayRef<LoadInst *> Loads,
181 ArrayRef<StoreInst *> Stores);
182 void InjectTraceForExits(Function &F);
183 void InjectTraceForSwitch(Function &F,
184 ArrayRef<Instruction *> SwitchTraceTargets,
185 Value *&FunctionGateCmp);
186 bool InjectCoverage(Function &F, ArrayRef<BasicBlock *> AllBlocks,
187 Value *&FunctionGateCmp, bool IsLeafFunc);
188 GlobalVariable *CreateFunctionLocalArrayInSection(size_t NumElements,
189 Function &F, Type *Ty,
190 const char *Section);
191 GlobalVariable *CreatePCArray(Function &F, ArrayRef<BasicBlock *> AllBlocks);
192 void CreateFunctionLocalArrays(Function &F, ArrayRef<BasicBlock *> AllBlocks);
193 Instruction *CreateGateBranch(Function &F, Value *&FunctionGateCmp,
194 Instruction *I);
195 Value *CreateFunctionLocalGateCmp(IRBuilder<> &IRB);
196 void InjectCoverageAtBlock(Function &F, BasicBlock &BB, size_t Idx,
197 Value *&FunctionGateCmp, bool IsLeafFunc);
198 Function *CreateInitCallsForSections(Module &M, const char *CtorName,
199 const char *InitFunctionName, Type *Ty,
200 const char *Section);
201 std::pair<Value *, Value *> CreateSecStartEnd(Module &M, const char *Section,
202 Type *Ty);
203
204 std::string getSectionName(const std::string &Section) const;
205 std::string getSectionStart(const std::string &Section) const;
206 std::string getSectionEnd(const std::string &Section) const;
207
208 const InstrumentationOptions &Opts;
209 Module &M;
210 DomTreeCallback DTCallback;
211 PostDomTreeCallback PDTCallback;
212
213 FunctionCallee SanCovStackDepthCallback;
214 FunctionCallee SanCovTracePCIndir;
215 FunctionCallee SanCovTracePC, SanCovTracePCGuard;
216 FunctionCallee SanCovTracePCEntry, SanCovTracePCExit;
217 std::array<FunctionCallee, 4> SanCovTraceCmpFunction;
218 std::array<FunctionCallee, 4> SanCovTraceConstCmpFunction;
219 std::array<FunctionCallee, 5> SanCovLoadFunction;
220 std::array<FunctionCallee, 5> SanCovStoreFunction;
221 std::array<FunctionCallee, 2> SanCovTraceDivFunction;
222 FunctionCallee SanCovTraceGepFunction;
223 FunctionCallee SanCovTraceSwitchFunction;
224 GlobalVariable *SanCovLowestStack;
225 GlobalVariable *SanCovCallbackGate;
226 Type *PtrTy, *IntptrTy, *Int64Ty, *Int32Ty, *Int16Ty, *Int8Ty, *Int1Ty;
227 Module *CurModule;
228 Triple TargetTriple;
229 LLVMContext *C;
230 const DataLayout *DL;
231
232 GlobalVariable *FunctionGuardArray; // for trace-pc-guard.
233 GlobalVariable *Function8bitCounterArray; // for inline-8bit-counters.
234 GlobalVariable *FunctionBoolArray; // for inline-bool-flag.
235 GlobalVariable *FunctionPCsArray; // for pc-table.
236 GlobalVariable *FunctionCFsArray; // for control flow table
237 SmallVector<GlobalValue *, 20> GlobalsToAppendToUsed;
238 SmallVector<GlobalValue *, 20> GlobalsToAppendToCompilerUsed;
239
240 SanitizerCoverageOptions Options;
241
242 const SpecialCaseList *Allowlist;
243 const SpecialCaseList *Blocklist;
244};
245} // namespace
246
247SanitizerCoveragePass::SanitizerCoveragePass(
248 SanitizerCoverageOptions Options, IntrusiveRefCntPtr<vfs::FileSystem> VFS,
249 const std::vector<std::string> &AllowlistFiles,
250 const std::vector<std::string> &BlocklistFiles)
251 : Options(std::move(Options)),
252 VFS(VFS ? std::move(VFS) : vfs::getRealFileSystem()) {
253 if (AllowlistFiles.size() > 0)
254 Allowlist = SpecialCaseList::createOrDie(Paths: AllowlistFiles, FS&: *this->VFS);
255 if (BlocklistFiles.size() > 0)
256 Blocklist = SpecialCaseList::createOrDie(Paths: BlocklistFiles, FS&: *this->VFS);
257}
258
259PreservedAnalyses SanitizerCoveragePass::run(Module &M,
260 ModuleAnalysisManager &MAM) {
261 auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
262 auto DTCallback = [&FAM](Function &F) -> const DominatorTree & {
263 return FAM.getResult<DominatorTreeAnalysis>(IR&: F);
264 };
265 auto PDTCallback = [&FAM](Function &F) -> const PostDominatorTree & {
266 return FAM.getResult<PostDominatorTreeAnalysis>(IR&: F);
267 };
268 const InstrumentationOptions &Opts =
269 M.getContext().getLibraryOptions().get<InstrumentationOptions>();
270 ModuleSanitizerCoverage ModuleSancov(Opts, M, DTCallback, PDTCallback,
271 overrideFromCL(Opts, Options),
272 Allowlist.get(), Blocklist.get());
273 if (!ModuleSancov.instrumentModule())
274 return PreservedAnalyses::all();
275
276 PreservedAnalyses PA = PreservedAnalyses::none();
277 // GlobalsAA is considered stateless and does not get invalidated unless
278 // explicitly invalidated; PreservedAnalyses::none() is not enough. Sanitizers
279 // make changes that require GlobalsAA to be invalidated.
280 PA.abandon<GlobalsAA>();
281 return PA;
282}
283
284std::pair<Value *, Value *>
285ModuleSanitizerCoverage::CreateSecStartEnd(Module &M, const char *Section,
286 Type *Ty) {
287 // Use ExternalWeak so that if all sections are discarded due to section
288 // garbage collection, the linker will not report undefined symbol errors.
289 // Windows defines the start/stop symbols in compiler-rt so no need for
290 // ExternalWeak.
291 GlobalValue::LinkageTypes Linkage = TargetTriple.isOSBinFormatCOFF()
292 ? GlobalVariable::ExternalLinkage
293 : GlobalVariable::ExternalWeakLinkage;
294 GlobalVariable *SecStart = new GlobalVariable(M, Ty, false, Linkage, nullptr,
295 getSectionStart(Section));
296 SecStart->setVisibility(GlobalValue::HiddenVisibility);
297 GlobalVariable *SecEnd = new GlobalVariable(M, Ty, false, Linkage, nullptr,
298 getSectionEnd(Section));
299 SecEnd->setVisibility(GlobalValue::HiddenVisibility);
300 if (!TargetTriple.isOSBinFormatCOFF())
301 return std::make_pair(x&: SecStart, y&: SecEnd);
302
303 // Account for the fact that on windows-msvc __start_* symbols actually
304 // point to a uint64_t before the start of the array.
305 auto *GEP = ConstantExpr::getPtrAdd(
306 Ptr: SecStart, Offset: ConstantInt::get(Ty: IntptrTy, V: sizeof(uint64_t)));
307 return std::make_pair(x&: GEP, y&: SecEnd);
308}
309
310Function *ModuleSanitizerCoverage::CreateInitCallsForSections(
311 Module &M, const char *CtorName, const char *InitFunctionName, Type *Ty,
312 const char *Section) {
313 if (Opts.sanitizer_coverage_drop_ctors)
314 return nullptr;
315 auto SecStartEnd = CreateSecStartEnd(M, Section, Ty);
316 auto SecStart = SecStartEnd.first;
317 auto SecEnd = SecStartEnd.second;
318 Function *CtorFunc;
319 std::tie(args&: CtorFunc, args: std::ignore) = createSanitizerCtorAndInitFunctions(
320 M, CtorName, InitName: InitFunctionName, InitArgTypes: {PtrTy, PtrTy}, InitArgs: {SecStart, SecEnd});
321 assert(CtorFunc->getName() == CtorName);
322
323 if (TargetTriple.supportsCOMDAT()) {
324 // Use comdat to dedup CtorFunc.
325 CtorFunc->setComdat(M.getOrInsertComdat(Name: CtorName));
326 appendToGlobalCtors(M, F: CtorFunc, Priority: SanCtorAndDtorPriority, Data: CtorFunc);
327 } else {
328 appendToGlobalCtors(M, F: CtorFunc, Priority: SanCtorAndDtorPriority);
329 }
330
331 if (TargetTriple.isOSBinFormatCOFF()) {
332 // In COFF files, if the contructors are set as COMDAT (they are because
333 // COFF supports COMDAT) and the linker flag /OPT:REF (strip unreferenced
334 // functions and data) is used, the constructors get stripped. To prevent
335 // this, give the constructors weak ODR linkage and ensure the linker knows
336 // to include the sancov constructor. This way the linker can deduplicate
337 // the constructors but always leave one copy.
338 CtorFunc->setLinkage(GlobalValue::WeakODRLinkage);
339 }
340 return CtorFunc;
341}
342
343bool ModuleSanitizerCoverage::instrumentModule() {
344 if (Options.CoverageType == SanitizerCoverageOptions::SCK_None)
345 return false;
346 if (Allowlist &&
347 !Allowlist->inSection(Section: "coverage", Prefix: "src", Query: M.getSourceFileName()))
348 return false;
349 if (Blocklist &&
350 Blocklist->inSection(Section: "coverage", Prefix: "src", Query: M.getSourceFileName()))
351 return false;
352 C = &(M.getContext());
353 DL = &M.getDataLayout();
354 CurModule = &M;
355 TargetTriple = M.getTargetTriple();
356 FunctionGuardArray = nullptr;
357 Function8bitCounterArray = nullptr;
358 FunctionBoolArray = nullptr;
359 FunctionPCsArray = nullptr;
360 FunctionCFsArray = nullptr;
361 IntptrTy = Type::getIntNTy(C&: *C, N: DL->getPointerSizeInBits());
362 PtrTy = PointerType::getUnqual(C&: *C);
363 Type *VoidTy = Type::getVoidTy(C&: *C);
364 IRBuilder<> IRB(M);
365 Int64Ty = IRB.getInt64Ty();
366 Int32Ty = IRB.getInt32Ty();
367 Int16Ty = IRB.getInt16Ty();
368 Int8Ty = IRB.getInt8Ty();
369 Int1Ty = IRB.getInt1Ty();
370
371 SanCovTracePCIndir =
372 M.getOrInsertFunction(Name: SanCovTracePCIndirName, RetTy: VoidTy, Args: IntptrTy);
373 // Make sure smaller parameters are zero-extended to i64 if required by the
374 // target ABI.
375 AttributeList SanCovTraceCmpZeroExtAL;
376 SanCovTraceCmpZeroExtAL =
377 SanCovTraceCmpZeroExtAL.addParamAttribute(C&: *C, ArgNo: 0, Kind: Attribute::ZExt);
378 SanCovTraceCmpZeroExtAL =
379 SanCovTraceCmpZeroExtAL.addParamAttribute(C&: *C, ArgNo: 1, Kind: Attribute::ZExt);
380
381 SanCovTraceCmpFunction[0] =
382 M.getOrInsertFunction(Name: SanCovTraceCmp1, AttributeList: SanCovTraceCmpZeroExtAL, RetTy: VoidTy,
383 Args: IRB.getInt8Ty(), Args: IRB.getInt8Ty());
384 SanCovTraceCmpFunction[1] =
385 M.getOrInsertFunction(Name: SanCovTraceCmp2, AttributeList: SanCovTraceCmpZeroExtAL, RetTy: VoidTy,
386 Args: IRB.getInt16Ty(), Args: IRB.getInt16Ty());
387 SanCovTraceCmpFunction[2] =
388 M.getOrInsertFunction(Name: SanCovTraceCmp4, AttributeList: SanCovTraceCmpZeroExtAL, RetTy: VoidTy,
389 Args: IRB.getInt32Ty(), Args: IRB.getInt32Ty());
390 SanCovTraceCmpFunction[3] =
391 M.getOrInsertFunction(Name: SanCovTraceCmp8, RetTy: VoidTy, Args: Int64Ty, Args: Int64Ty);
392
393 SanCovTraceConstCmpFunction[0] = M.getOrInsertFunction(
394 Name: SanCovTraceConstCmp1, AttributeList: SanCovTraceCmpZeroExtAL, RetTy: VoidTy, Args: Int8Ty, Args: Int8Ty);
395 SanCovTraceConstCmpFunction[1] = M.getOrInsertFunction(
396 Name: SanCovTraceConstCmp2, AttributeList: SanCovTraceCmpZeroExtAL, RetTy: VoidTy, Args: Int16Ty, Args: Int16Ty);
397 SanCovTraceConstCmpFunction[2] = M.getOrInsertFunction(
398 Name: SanCovTraceConstCmp4, AttributeList: SanCovTraceCmpZeroExtAL, RetTy: VoidTy, Args: Int32Ty, Args: Int32Ty);
399 SanCovTraceConstCmpFunction[3] =
400 M.getOrInsertFunction(Name: SanCovTraceConstCmp8, RetTy: VoidTy, Args: Int64Ty, Args: Int64Ty);
401
402 // Loads.
403 SanCovLoadFunction[0] = M.getOrInsertFunction(Name: SanCovLoad1, RetTy: VoidTy, Args: PtrTy);
404 SanCovLoadFunction[1] = M.getOrInsertFunction(Name: SanCovLoad2, RetTy: VoidTy, Args: PtrTy);
405 SanCovLoadFunction[2] = M.getOrInsertFunction(Name: SanCovLoad4, RetTy: VoidTy, Args: PtrTy);
406 SanCovLoadFunction[3] = M.getOrInsertFunction(Name: SanCovLoad8, RetTy: VoidTy, Args: PtrTy);
407 SanCovLoadFunction[4] = M.getOrInsertFunction(Name: SanCovLoad16, RetTy: VoidTy, Args: PtrTy);
408 // Stores.
409 SanCovStoreFunction[0] = M.getOrInsertFunction(Name: SanCovStore1, RetTy: VoidTy, Args: PtrTy);
410 SanCovStoreFunction[1] = M.getOrInsertFunction(Name: SanCovStore2, RetTy: VoidTy, Args: PtrTy);
411 SanCovStoreFunction[2] = M.getOrInsertFunction(Name: SanCovStore4, RetTy: VoidTy, Args: PtrTy);
412 SanCovStoreFunction[3] = M.getOrInsertFunction(Name: SanCovStore8, RetTy: VoidTy, Args: PtrTy);
413 SanCovStoreFunction[4] = M.getOrInsertFunction(Name: SanCovStore16, RetTy: VoidTy, Args: PtrTy);
414
415 {
416 AttributeList AL;
417 AL = AL.addParamAttribute(C&: *C, ArgNo: 0, Kind: Attribute::ZExt);
418 SanCovTraceDivFunction[0] =
419 M.getOrInsertFunction(Name: SanCovTraceDiv4, AttributeList: AL, RetTy: VoidTy, Args: IRB.getInt32Ty());
420 }
421 SanCovTraceDivFunction[1] =
422 M.getOrInsertFunction(Name: SanCovTraceDiv8, RetTy: VoidTy, Args: Int64Ty);
423 SanCovTraceGepFunction =
424 M.getOrInsertFunction(Name: SanCovTraceGep, RetTy: VoidTy, Args: IntptrTy);
425 SanCovTraceSwitchFunction =
426 M.getOrInsertFunction(Name: SanCovTraceSwitchName, RetTy: VoidTy, Args: Int64Ty, Args: PtrTy);
427
428 SanCovLowestStack = M.getOrInsertGlobal(Name: SanCovLowestStackName, Ty: IntptrTy);
429 if (SanCovLowestStack->getValueType() != IntptrTy) {
430 C->emitError(ErrorStr: StringRef("'") + SanCovLowestStackName +
431 "' should not be declared by the user");
432 return true;
433 }
434 SanCovLowestStack->setThreadLocalMode(
435 GlobalValue::ThreadLocalMode::InitialExecTLSModel);
436 if (Options.StackDepth && !SanCovLowestStack->isDeclaration())
437 SanCovLowestStack->setInitializer(Constant::getAllOnesValue(Ty: IntptrTy));
438
439 if (Options.GatedCallbacks) {
440 if (!Options.TracePCGuard && !Options.TraceCmp) {
441 C->emitError(ErrorStr: "'sanitizer-coverage-gated-trace-callbacks' is only "
442 "supported with trace-pc-guard or trace-cmp");
443 return true;
444 }
445
446 SanCovCallbackGate = cast<GlobalVariable>(
447 Val: M.getOrInsertGlobal(Name: SanCovCallbackGateName, Ty: Int64Ty));
448 SanCovCallbackGate->setSection(
449 getSectionName(Section: SanCovCallbackGateSectionName));
450 SanCovCallbackGate->setInitializer(Constant::getNullValue(Ty: Int64Ty));
451 SanCovCallbackGate->setLinkage(GlobalVariable::LinkOnceAnyLinkage);
452 SanCovCallbackGate->setVisibility(GlobalVariable::HiddenVisibility);
453 appendToCompilerUsed(M, Values: SanCovCallbackGate);
454 }
455
456 SanCovTracePC = M.getOrInsertFunction(Name: SanCovTracePCName, RetTy: VoidTy);
457 SanCovTracePCEntry = M.getOrInsertFunction(Name: SanCovTracePCEntryName, RetTy: VoidTy);
458 SanCovTracePCExit = M.getOrInsertFunction(Name: SanCovTracePCExitName, RetTy: VoidTy);
459 SanCovTracePCGuard =
460 M.getOrInsertFunction(Name: SanCovTracePCGuardName, RetTy: VoidTy, Args: PtrTy);
461
462 SanCovStackDepthCallback =
463 M.getOrInsertFunction(Name: SanCovStackDepthCallbackName, RetTy: VoidTy);
464
465 for (auto &F : M)
466 instrumentFunction(F);
467
468 Function *Ctor = nullptr;
469
470 if (FunctionGuardArray)
471 Ctor = CreateInitCallsForSections(M, CtorName: SanCovModuleCtorTracePcGuardName,
472 InitFunctionName: SanCovTracePCGuardInitName, Ty: Int32Ty,
473 Section: SanCovGuardsSectionName);
474 if (Function8bitCounterArray)
475 Ctor = CreateInitCallsForSections(M, CtorName: SanCovModuleCtor8bitCountersName,
476 InitFunctionName: SanCov8bitCountersInitName, Ty: Int8Ty,
477 Section: SanCovCountersSectionName);
478 if (FunctionBoolArray) {
479 Ctor = CreateInitCallsForSections(M, CtorName: SanCovModuleCtorBoolFlagName,
480 InitFunctionName: SanCovBoolFlagInitName, Ty: Int1Ty,
481 Section: SanCovBoolFlagSectionName);
482 }
483 if (Ctor && Options.PCTable) {
484 auto SecStartEnd = CreateSecStartEnd(M, Section: SanCovPCsSectionName, Ty: IntptrTy);
485 FunctionCallee InitFunction =
486 declareSanitizerInitFunction(M, InitName: SanCovPCsInitName, InitArgTypes: {PtrTy, PtrTy});
487 IRBuilder<> IRBCtor(Ctor->getEntryBlock().getTerminator());
488 IRBCtor.CreateCall(Callee: InitFunction, Args: {SecStartEnd.first, SecStartEnd.second});
489 }
490
491 if (Ctor && Options.CollectControlFlow) {
492 auto SecStartEnd = CreateSecStartEnd(M, Section: SanCovCFsSectionName, Ty: IntptrTy);
493 FunctionCallee InitFunction =
494 declareSanitizerInitFunction(M, InitName: SanCovCFsInitName, InitArgTypes: {PtrTy, PtrTy});
495 IRBuilder<> IRBCtor(Ctor->getEntryBlock().getTerminator());
496 IRBCtor.CreateCall(Callee: InitFunction, Args: {SecStartEnd.first, SecStartEnd.second});
497 }
498
499 appendToUsed(M, Values: GlobalsToAppendToUsed);
500 appendToCompilerUsed(M, Values: GlobalsToAppendToCompilerUsed);
501 return true;
502}
503
504// True if block has successors and it dominates all of them.
505static bool isFullDominator(const BasicBlock *BB, const DominatorTree &DT) {
506 if (succ_empty(BB))
507 return false;
508
509 return llvm::all_of(Range: successors(BB), P: [&](const BasicBlock *SUCC) {
510 return DT.dominates(A: BB, B: SUCC);
511 });
512}
513
514// True if block has predecessors and it postdominates all of them.
515static bool isFullPostDominator(const BasicBlock *BB,
516 const PostDominatorTree &PDT) {
517 if (pred_empty(BB))
518 return false;
519
520 return llvm::all_of(Range: predecessors(BB), P: [&](const BasicBlock *PRED) {
521 return PDT.dominates(A: BB, B: PRED);
522 });
523}
524
525static bool shouldInstrumentBlock(const Function &F, const BasicBlock *BB,
526 const DominatorTree &DT,
527 const PostDominatorTree &PDT,
528 const SanitizerCoverageOptions &Options) {
529 // Don't insert coverage for blocks containing nothing but unreachable: we
530 // will never call __sanitizer_cov() for them, so counting them in
531 // NumberOfInstrumentedBlocks() might complicate calculation of code coverage
532 // percentage. Also, unreachable instructions frequently have no debug
533 // locations.
534 if (isa<UnreachableInst>(Val: BB->getFirstNonPHIOrLifetime()))
535 return false;
536
537 // Don't insert coverage into blocks without a valid insertion point
538 // (catchswitch blocks).
539 if (BB->getFirstInsertionPt() == BB->end())
540 return false;
541
542 if (Options.NoPrune || &F.getEntryBlock() == BB)
543 return true;
544
545 if (Options.CoverageType == SanitizerCoverageOptions::SCK_Function &&
546 &F.getEntryBlock() != BB)
547 return false;
548
549 // Do not instrument full dominators, or full post-dominators with multiple
550 // predecessors.
551 return !isFullDominator(BB, DT) &&
552 !(isFullPostDominator(BB, PDT) && !BB->getSinglePredecessor());
553}
554
555// Returns true iff From->To is a backedge.
556// A twist here is that we treat From->To as a backedge if
557// * To dominates From or
558// * To->UniqueSuccessor dominates From
559static bool IsBackEdge(BasicBlock *From, BasicBlock *To,
560 const DominatorTree &DT) {
561 if (DT.dominates(A: To, B: From))
562 return true;
563 if (auto Next = To->getUniqueSuccessor())
564 if (DT.dominates(A: Next, B: From))
565 return true;
566 return false;
567}
568
569// Prunes uninteresting Cmp instrumentation:
570// * CMP instructions that feed into loop backedge branch.
571//
572// Note that Cmp pruning is controlled by the same flag as the
573// BB pruning.
574static bool IsInterestingCmp(ICmpInst *CMP, const DominatorTree &DT,
575 const SanitizerCoverageOptions &Options) {
576 if (!Options.NoPrune)
577 if (CMP->hasOneUse())
578 if (auto BR = dyn_cast<CondBrInst>(Val: CMP->user_back()))
579 for (BasicBlock *B : BR->successors())
580 if (IsBackEdge(From: BR->getParent(), To: B, DT))
581 return false;
582 return true;
583}
584
585void ModuleSanitizerCoverage::instrumentFunction(Function &F) {
586 if (F.empty())
587 return;
588 if (F.getName().contains(Other: ".module_ctor"))
589 return; // Should not instrument sanitizer init functions.
590 if (F.getName().starts_with(Prefix: "__sanitizer_"))
591 return; // Don't instrument __sanitizer_* callbacks.
592 // Don't touch available_externally functions, their actual body is elewhere.
593 if (F.getLinkage() == GlobalValue::AvailableExternallyLinkage)
594 return;
595 // Don't instrument MSVC CRT configuration helpers. They may run before normal
596 // initialization.
597 if (F.getName() == "__local_stdio_printf_options" ||
598 F.getName() == "__local_stdio_scanf_options")
599 return;
600 if (isa<UnreachableInst>(Val: F.getEntryBlock().getTerminator()))
601 return;
602 // Don't instrument functions using SEH for now. Splitting basic blocks like
603 // we do for coverage breaks WinEHPrepare.
604 // FIXME: Remove this when SEH no longer uses landingpad pattern matching.
605 if (F.hasPersonalityFn() &&
606 isAsynchronousEHPersonality(Pers: classifyEHPersonality(Pers: F.getPersonalityFn())))
607 return;
608 if (Allowlist && !Allowlist->inSection(Section: "coverage", Prefix: "fun", Query: F.getName()))
609 return;
610 if (Blocklist && Blocklist->inSection(Section: "coverage", Prefix: "fun", Query: F.getName()))
611 return;
612 // Do not apply any instrumentation for naked functions.
613 if (F.hasFnAttribute(Kind: Attribute::Naked))
614 return;
615 if (F.hasFnAttribute(Kind: Attribute::NoSanitizeCoverage))
616 return;
617 if (F.hasFnAttribute(Kind: Attribute::DisableSanitizerInstrumentation))
618 return;
619 if (Options.CoverageType >= SanitizerCoverageOptions::SCK_Edge) {
620 SplitAllCriticalEdges(
621 F, Options: CriticalEdgeSplittingOptions().setIgnoreUnreachableDests());
622 }
623 SmallVector<Instruction *, 8> IndirCalls;
624 SmallVector<BasicBlock *, 16> BlocksToInstrument;
625 SmallVector<Instruction *, 8> CmpTraceTargets;
626 SmallVector<Instruction *, 8> SwitchTraceTargets;
627 SmallVector<BinaryOperator *, 8> DivTraceTargets;
628 SmallVector<GetElementPtrInst *, 8> GepTraceTargets;
629 SmallVector<LoadInst *, 8> Loads;
630 SmallVector<StoreInst *, 8> Stores;
631
632 const DominatorTree &DT = DTCallback(F);
633 const PostDominatorTree &PDT = PDTCallback(F);
634 bool IsLeafFunc = true;
635
636 for (auto &BB : F) {
637 if (shouldInstrumentBlock(F, BB: &BB, DT, PDT, Options))
638 BlocksToInstrument.push_back(Elt: &BB);
639 for (auto &Inst : BB) {
640 if (Options.IndirectCalls) {
641 CallBase *CB = dyn_cast<CallBase>(Val: &Inst);
642 if (CB && CB->isIndirectCall())
643 IndirCalls.push_back(Elt: &Inst);
644 }
645 if (Options.TraceCmp) {
646 if (ICmpInst *CMP = dyn_cast<ICmpInst>(Val: &Inst))
647 if (IsInterestingCmp(CMP, DT, Options))
648 CmpTraceTargets.push_back(Elt: &Inst);
649 if (isa<SwitchInst>(Val: &Inst))
650 SwitchTraceTargets.push_back(Elt: &Inst);
651 }
652 if (Options.TraceDiv)
653 if (BinaryOperator *BO = dyn_cast<BinaryOperator>(Val: &Inst))
654 if (BO->getOpcode() == Instruction::SDiv ||
655 BO->getOpcode() == Instruction::UDiv)
656 DivTraceTargets.push_back(Elt: BO);
657 if (Options.TraceGep)
658 if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Val: &Inst))
659 GepTraceTargets.push_back(Elt: GEP);
660 if (Options.TraceLoads)
661 if (LoadInst *LI = dyn_cast<LoadInst>(Val: &Inst))
662 Loads.push_back(Elt: LI);
663 if (Options.TraceStores)
664 if (StoreInst *SI = dyn_cast<StoreInst>(Val: &Inst))
665 Stores.push_back(Elt: SI);
666 if (Options.StackDepth)
667 if (isa<InvokeInst>(Val: Inst) ||
668 (isa<CallInst>(Val: Inst) && !isa<IntrinsicInst>(Val: Inst)))
669 IsLeafFunc = false;
670 }
671 }
672
673 if (Options.CollectControlFlow)
674 createFunctionControlFlow(F);
675
676 Value *FunctionGateCmp = nullptr;
677 InjectCoverage(F, AllBlocks: BlocksToInstrument, FunctionGateCmp, IsLeafFunc);
678 InjectCoverageForIndirectCalls(F, IndirCalls);
679 InjectTraceForCmp(F, CmpTraceTargets, FunctionGateCmp);
680 InjectTraceForSwitch(F, SwitchTraceTargets, FunctionGateCmp);
681 InjectTraceForDiv(F, DivTraceTargets);
682 InjectTraceForGep(F, GepTraceTargets);
683 InjectTraceForLoadsAndStores(F, Loads, Stores);
684
685 if (Options.TracePCEntryExit)
686 InjectTraceForExits(F);
687}
688
689GlobalVariable *ModuleSanitizerCoverage::CreateFunctionLocalArrayInSection(
690 size_t NumElements, Function &F, Type *Ty, const char *Section) {
691 ArrayType *ArrayTy = ArrayType::get(ElementType: Ty, NumElements);
692 auto Array = new GlobalVariable(
693 *CurModule, ArrayTy, false, GlobalVariable::PrivateLinkage,
694 Constant::getNullValue(Ty: ArrayTy), "__sancov_gen_");
695
696 // sancov_pcs parallels the other arrays, so they must be retained or
697 // discarded together. Put them in F's comdat to tie them to F for linker GC
698 // benefit. Outside ELF (nodeduplicate), a new comdat for an interposable F
699 // could prevail over a strong definition (COFF: the weak external becomes a
700 // COMDAT definition; Wasm: comdats deduplicate first). noipa doesn't affect
701 // symbol resolution.
702 if (TargetTriple.supportsCOMDAT() &&
703 (F.hasComdat() || TargetTriple.isOSBinFormatELF() ||
704 !F.isInterposable(/*CheckNoIPA=*/false)))
705 if (auto Comdat = getOrCreateFunctionComdat(F, T&: TargetTriple))
706 Array->setComdat(Comdat);
707 Array->setSection(getSectionName(Section));
708 Array->setAlignment(Align(DL->getTypeStoreSize(Ty).getFixedValue()));
709
710 // Optimizers (e.g. GlobalOpt/ConstantMerge) may not discard the arrays as a
711 // unit, so retain them in the compiler; without a comdat, in the linker too.
712 if (Array->hasComdat())
713 GlobalsToAppendToCompilerUsed.push_back(Elt: Array);
714 else
715 GlobalsToAppendToUsed.push_back(Elt: Array);
716
717 return Array;
718}
719
720GlobalVariable *
721ModuleSanitizerCoverage::CreatePCArray(Function &F,
722 ArrayRef<BasicBlock *> AllBlocks) {
723 size_t N = AllBlocks.size();
724 assert(N);
725 SmallVector<Constant *, 32> PCs;
726 IRBuilder<> IRB(&*F.getEntryBlock().getFirstInsertionPt());
727 for (size_t i = 0; i < N; i++) {
728 if (&F.getEntryBlock() == AllBlocks[i]) {
729 PCs.push_back(Elt: (Constant *)IRB.CreatePointerCast(V: &F, DestTy: PtrTy));
730 PCs.push_back(
731 Elt: (Constant *)IRB.CreateIntToPtr(V: ConstantInt::get(Ty: IntptrTy, V: 1), DestTy: PtrTy));
732 } else {
733 PCs.push_back(Elt: (Constant *)IRB.CreatePointerCast(
734 V: BlockAddress::get(BB: AllBlocks[i]), DestTy: PtrTy));
735 PCs.push_back(Elt: Constant::getNullValue(Ty: PtrTy));
736 }
737 }
738 auto *PCArray =
739 CreateFunctionLocalArrayInSection(NumElements: N * 2, F, Ty: PtrTy, Section: SanCovPCsSectionName);
740 PCArray->setInitializer(
741 ConstantArray::get(T: ArrayType::get(ElementType: PtrTy, NumElements: N * 2), V: PCs));
742 PCArray->setConstant(true);
743
744 return PCArray;
745}
746
747void ModuleSanitizerCoverage::CreateFunctionLocalArrays(
748 Function &F, ArrayRef<BasicBlock *> AllBlocks) {
749 if (Options.TracePCGuard)
750 FunctionGuardArray = CreateFunctionLocalArrayInSection(
751 NumElements: AllBlocks.size(), F, Ty: Int32Ty, Section: SanCovGuardsSectionName);
752
753 if (Options.Inline8bitCounters)
754 Function8bitCounterArray = CreateFunctionLocalArrayInSection(
755 NumElements: AllBlocks.size(), F, Ty: Int8Ty, Section: SanCovCountersSectionName);
756 if (Options.InlineBoolFlag)
757 FunctionBoolArray = CreateFunctionLocalArrayInSection(
758 NumElements: AllBlocks.size(), F, Ty: Int1Ty, Section: SanCovBoolFlagSectionName);
759
760 if (Options.PCTable)
761 FunctionPCsArray = CreatePCArray(F, AllBlocks);
762}
763
764Value *ModuleSanitizerCoverage::CreateFunctionLocalGateCmp(IRBuilder<> &IRB) {
765 auto Load = IRB.CreateLoad(Ty: Int64Ty, Ptr: SanCovCallbackGate);
766 Load->setNoSanitizeMetadata();
767 auto Cmp = IRB.CreateIsNotNull(Arg: Load);
768 Cmp->setName("sancov gate cmp");
769 return Cmp;
770}
771
772Instruction *ModuleSanitizerCoverage::CreateGateBranch(Function &F,
773 Value *&FunctionGateCmp,
774 Instruction *IP) {
775 if (!FunctionGateCmp) {
776 // Create this in the entry block
777 BasicBlock &BB = F.getEntryBlock();
778 BasicBlock::iterator IP = BB.getFirstInsertionPt();
779 IP = PrepareToSplitEntryBlock(BB, IP);
780 IRBuilder<> EntryIRB(&*IP);
781 FunctionGateCmp = CreateFunctionLocalGateCmp(IRB&: EntryIRB);
782 }
783 // Set the branch weights in order to minimize the price paid when the
784 // gate is turned off, allowing the default enablement of this
785 // instrumentation with as little of a performance cost as possible
786 auto Weights = MDBuilder(*C).createBranchWeights(TrueWeight: 1, FalseWeight: 100000);
787 return SplitBlockAndInsertIfThen(Cond: FunctionGateCmp, SplitBefore: IP, Unreachable: false, BranchWeights: Weights);
788}
789
790bool ModuleSanitizerCoverage::InjectCoverage(Function &F,
791 ArrayRef<BasicBlock *> AllBlocks,
792 Value *&FunctionGateCmp,
793 bool IsLeafFunc) {
794 if (AllBlocks.empty())
795 return false;
796 CreateFunctionLocalArrays(F, AllBlocks);
797 for (size_t i = 0, N = AllBlocks.size(); i < N; i++)
798 InjectCoverageAtBlock(F, BB&: *AllBlocks[i], Idx: i, FunctionGateCmp, IsLeafFunc);
799
800 return true;
801}
802
803// On every indirect call we call a run-time function
804// __sanitizer_cov_indir_call* with two parameters:
805// - callee address,
806// - global cache array that contains CacheSize pointers (zero-initialized).
807// The cache is used to speed up recording the caller-callee pairs.
808// The address of the caller is passed implicitly via caller PC.
809// CacheSize is encoded in the name of the run-time function.
810void ModuleSanitizerCoverage::InjectCoverageForIndirectCalls(
811 Function &F, ArrayRef<Instruction *> IndirCalls) {
812 if (IndirCalls.empty())
813 return;
814 assert(Options.TracePC || Options.TracePCEntryExit || Options.TracePCGuard ||
815 Options.Inline8bitCounters || Options.InlineBoolFlag);
816 for (auto *I : IndirCalls) {
817 InstrumentationIRBuilder IRB(I);
818 CallBase &CB = cast<CallBase>(Val&: *I);
819 Value *Callee = CB.getCalledOperand();
820 if (isa<InlineAsm>(Val: Callee))
821 continue;
822 IRB.CreateCall(Callee: SanCovTracePCIndir, Args: IRB.CreatePointerCast(V: Callee, DestTy: IntptrTy));
823 }
824}
825
826// For every switch statement we insert a call:
827// __sanitizer_cov_trace_switch(CondValue,
828// {NumCases, ValueSizeInBits, Case0Value, Case1Value, Case2Value, ... })
829
830void ModuleSanitizerCoverage::InjectTraceForSwitch(
831 Function &F, ArrayRef<Instruction *> SwitchTraceTargets,
832 Value *&FunctionGateCmp) {
833 for (auto *I : SwitchTraceTargets) {
834 if (SwitchInst *SI = dyn_cast<SwitchInst>(Val: I)) {
835 InstrumentationIRBuilder IRB(I);
836 SmallVector<Constant *, 16> Initializers;
837 Value *Cond = SI->getCondition();
838 if (Cond->getType()->getScalarSizeInBits() >
839 Int64Ty->getScalarSizeInBits())
840 continue;
841 Initializers.push_back(Elt: ConstantInt::get(Ty: Int64Ty, V: SI->getNumCases()));
842 Initializers.push_back(
843 Elt: ConstantInt::get(Ty: Int64Ty, V: Cond->getType()->getScalarSizeInBits()));
844 if (Cond->getType()->getScalarSizeInBits() <
845 Int64Ty->getScalarSizeInBits())
846 Cond = IRB.CreateIntCast(V: Cond, DestTy: Int64Ty, isSigned: false);
847 for (auto It : SI->cases()) {
848 ConstantInt *C = It.getCaseValue();
849 if (C->getType()->getScalarSizeInBits() < 64)
850 C = ConstantInt::get(Context&: C->getContext(), V: C->getValue().zext(width: 64));
851 Initializers.push_back(Elt: C);
852 }
853 llvm::sort(C: drop_begin(RangeOrContainer&: Initializers, N: 2),
854 Comp: [](const Constant *A, const Constant *B) {
855 return cast<ConstantInt>(Val: A)->getLimitedValue() <
856 cast<ConstantInt>(Val: B)->getLimitedValue();
857 });
858 ArrayType *ArrayOfInt64Ty = ArrayType::get(ElementType: Int64Ty, NumElements: Initializers.size());
859 GlobalVariable *GV = new GlobalVariable(
860 *CurModule, ArrayOfInt64Ty, false, GlobalVariable::InternalLinkage,
861 ConstantArray::get(T: ArrayOfInt64Ty, V: Initializers),
862 "__sancov_gen_cov_switch_values");
863 if (Options.GatedCallbacks) {
864 auto GateBranch = CreateGateBranch(F, FunctionGateCmp, IP: I);
865 IRBuilder<> GateIRB(GateBranch);
866 GateIRB.CreateCall(Callee: SanCovTraceSwitchFunction, Args: {Cond, GV});
867 } else {
868 IRB.CreateCall(Callee: SanCovTraceSwitchFunction, Args: {Cond, GV});
869 }
870 }
871 }
872}
873
874void ModuleSanitizerCoverage::InjectTraceForDiv(
875 Function &, ArrayRef<BinaryOperator *> DivTraceTargets) {
876 for (auto *BO : DivTraceTargets) {
877 InstrumentationIRBuilder IRB(BO);
878 Value *A1 = BO->getOperand(i_nocapture: 1);
879 if (isa<ConstantInt>(Val: A1))
880 continue;
881 if (!A1->getType()->isIntegerTy())
882 continue;
883 uint64_t TypeSize = DL->getTypeStoreSizeInBits(Ty: A1->getType());
884 int CallbackIdx = TypeSize == 32 ? 0 : TypeSize == 64 ? 1 : -1;
885 if (CallbackIdx < 0)
886 continue;
887 auto Ty = Type::getIntNTy(C&: *C, N: TypeSize);
888 IRB.CreateCall(Callee: SanCovTraceDivFunction[CallbackIdx],
889 Args: {IRB.CreateIntCast(V: A1, DestTy: Ty, isSigned: true)});
890 }
891}
892
893void ModuleSanitizerCoverage::InjectTraceForGep(
894 Function &, ArrayRef<GetElementPtrInst *> GepTraceTargets) {
895 for (auto *GEP : GepTraceTargets) {
896 InstrumentationIRBuilder IRB(GEP);
897 for (Use &Idx : GEP->indices())
898 if (!isa<ConstantInt>(Val: Idx) && Idx->getType()->isIntegerTy())
899 IRB.CreateCall(Callee: SanCovTraceGepFunction,
900 Args: {IRB.CreateIntCast(V: Idx, DestTy: IntptrTy, isSigned: true)});
901 }
902}
903
904void ModuleSanitizerCoverage::InjectTraceForLoadsAndStores(
905 Function &, ArrayRef<LoadInst *> Loads, ArrayRef<StoreInst *> Stores) {
906 auto CallbackIdx = [&](Type *ElementTy) -> int {
907 uint64_t TypeSize = DL->getTypeStoreSizeInBits(Ty: ElementTy);
908 return TypeSize == 8 ? 0
909 : TypeSize == 16 ? 1
910 : TypeSize == 32 ? 2
911 : TypeSize == 64 ? 3
912 : TypeSize == 128 ? 4
913 : -1;
914 };
915 for (auto *LI : Loads) {
916 InstrumentationIRBuilder IRB(LI);
917 auto Ptr = LI->getPointerOperand();
918 int Idx = CallbackIdx(LI->getType());
919 if (Idx < 0)
920 continue;
921 IRB.CreateCall(Callee: SanCovLoadFunction[Idx], Args: Ptr);
922 }
923 for (auto *SI : Stores) {
924 InstrumentationIRBuilder IRB(SI);
925 auto Ptr = SI->getPointerOperand();
926 int Idx = CallbackIdx(SI->getValueOperand()->getType());
927 if (Idx < 0)
928 continue;
929 IRB.CreateCall(Callee: SanCovStoreFunction[Idx], Args: Ptr);
930 }
931}
932
933void ModuleSanitizerCoverage::InjectTraceForExits(Function &F) {
934 EscapeEnumerator EE(F, "sancov_exit");
935 while (IRBuilder<> *AtExit = EE.Next()) {
936 InstrumentationIRBuilder::ensureDebugInfo(IRB&: *AtExit, F);
937 AtExit->CreateCall(Callee: SanCovTracePCExit, Args: {})
938 ->setTailCallKind(CallInst::TCK_NoTail);
939 }
940}
941
942void ModuleSanitizerCoverage::InjectTraceForCmp(
943 Function &F, ArrayRef<Instruction *> CmpTraceTargets,
944 Value *&FunctionGateCmp) {
945 for (auto *I : CmpTraceTargets) {
946 if (ICmpInst *ICMP = dyn_cast<ICmpInst>(Val: I)) {
947 InstrumentationIRBuilder IRB(ICMP);
948 Value *A0 = ICMP->getOperand(i_nocapture: 0);
949 Value *A1 = ICMP->getOperand(i_nocapture: 1);
950 if (!A0->getType()->isIntegerTy())
951 continue;
952 uint64_t TypeSize = DL->getTypeStoreSizeInBits(Ty: A0->getType());
953 int CallbackIdx = TypeSize == 8 ? 0
954 : TypeSize == 16 ? 1
955 : TypeSize == 32 ? 2
956 : TypeSize == 64 ? 3
957 : -1;
958 if (CallbackIdx < 0)
959 continue;
960 // __sanitizer_cov_trace_cmp((type_size << 32) | predicate, A0, A1);
961 auto CallbackFunc = SanCovTraceCmpFunction[CallbackIdx];
962 bool FirstIsConst = isa<ConstantInt>(Val: A0);
963 bool SecondIsConst = isa<ConstantInt>(Val: A1);
964 // If both are const, then we don't need such a comparison.
965 if (FirstIsConst && SecondIsConst)
966 continue;
967 // If only one is const, then make it the first callback argument.
968 if (FirstIsConst || SecondIsConst) {
969 CallbackFunc = SanCovTraceConstCmpFunction[CallbackIdx];
970 if (SecondIsConst)
971 std::swap(a&: A0, b&: A1);
972 }
973
974 auto Ty = Type::getIntNTy(C&: *C, N: TypeSize);
975 if (Options.GatedCallbacks) {
976 auto GateBranch = CreateGateBranch(F, FunctionGateCmp, IP: I);
977 IRBuilder<> GateIRB(GateBranch);
978 GateIRB.CreateCall(Callee: CallbackFunc, Args: {GateIRB.CreateIntCast(V: A0, DestTy: Ty, isSigned: true),
979 GateIRB.CreateIntCast(V: A1, DestTy: Ty, isSigned: true)});
980 } else {
981 IRB.CreateCall(Callee: CallbackFunc, Args: {IRB.CreateIntCast(V: A0, DestTy: Ty, isSigned: true),
982 IRB.CreateIntCast(V: A1, DestTy: Ty, isSigned: true)});
983 }
984 }
985 }
986}
987
988void ModuleSanitizerCoverage::InjectCoverageAtBlock(Function &F, BasicBlock &BB,
989 size_t Idx,
990 Value *&FunctionGateCmp,
991 bool IsLeafFunc) {
992 BasicBlock::iterator IP = BB.getFirstInsertionPt();
993 bool IsEntryBB = &BB == &F.getEntryBlock();
994 DebugLoc EntryLoc;
995 if (IsEntryBB) {
996 if (auto SP = F.getSubprogram())
997 EntryLoc = DILocation::get(Context&: SP->getContext(), Line: SP->getScopeLine(), Column: 0, Scope: SP);
998 // Keep static allocas and llvm.localescape calls in the entry block. Even
999 // if we aren't splitting the block, it's nice for allocas to be before
1000 // calls.
1001 IP = PrepareToSplitEntryBlock(BB, IP);
1002 }
1003
1004 InstrumentationIRBuilder IRB(&*IP);
1005 if (EntryLoc)
1006 IRB.SetCurrentDebugLocation(EntryLoc);
1007 if (Options.TracePC || (IsEntryBB && Options.TracePCEntryExit)) {
1008 FunctionCallee Callee = IsEntryBB && Options.TracePCEntryExit
1009 ? SanCovTracePCEntry
1010 : SanCovTracePC;
1011 IRB.CreateCall(Callee)
1012 ->setCannotMerge(); // gets the PC using GET_CALLER_PC.
1013 }
1014 if (Options.TracePCGuard) {
1015 auto GuardPtr = IRB.CreateConstInBoundsGEP2_64(
1016 Ty: FunctionGuardArray->getValueType(), Ptr: FunctionGuardArray, Idx0: 0, Idx1: Idx);
1017 if (Options.GatedCallbacks) {
1018 Instruction *I = &*IP;
1019 auto GateBranch = CreateGateBranch(F, FunctionGateCmp, IP: I);
1020 IRBuilder<> GateIRB(GateBranch);
1021 GateIRB.CreateCall(Callee: SanCovTracePCGuard, Args: GuardPtr)->setCannotMerge();
1022 } else {
1023 IRB.CreateCall(Callee: SanCovTracePCGuard, Args: GuardPtr)->setCannotMerge();
1024 }
1025 }
1026 if (Options.Inline8bitCounters) {
1027 auto CounterPtr = IRB.CreateGEP(
1028 Ty: Function8bitCounterArray->getValueType(), Ptr: Function8bitCounterArray,
1029 IdxList: {ConstantInt::get(Ty: IntptrTy, V: 0), ConstantInt::get(Ty: IntptrTy, V: Idx)});
1030 auto Load = IRB.CreateLoad(Ty: Int8Ty, Ptr: CounterPtr);
1031 auto Inc = IRB.CreateAdd(LHS: Load, RHS: ConstantInt::get(Ty: Int8Ty, V: 1));
1032 auto Store = IRB.CreateStore(Val: Inc, Ptr: CounterPtr);
1033 Load->setNoSanitizeMetadata();
1034 Store->setNoSanitizeMetadata();
1035 }
1036 if (Options.InlineBoolFlag) {
1037 auto FlagPtr = IRB.CreateGEP(
1038 Ty: FunctionBoolArray->getValueType(), Ptr: FunctionBoolArray,
1039 IdxList: {ConstantInt::get(Ty: IntptrTy, V: 0), ConstantInt::get(Ty: IntptrTy, V: Idx)});
1040 auto Load = IRB.CreateLoad(Ty: Int1Ty, Ptr: FlagPtr);
1041 auto ThenTerm = SplitBlockAndInsertIfThen(
1042 Cond: IRB.CreateIsNull(Arg: Load), SplitBefore: &*IP, Unreachable: false,
1043 BranchWeights: MDBuilder(IRB.getContext()).createUnlikelyBranchWeights());
1044 InstrumentationIRBuilder ThenIRB(ThenTerm);
1045 auto Store = ThenIRB.CreateStore(Val: ConstantInt::getTrue(Ty: Int1Ty), Ptr: FlagPtr);
1046 if (EntryLoc)
1047 Store->setDebugLoc(EntryLoc);
1048 Load->setNoSanitizeMetadata();
1049 Store->setNoSanitizeMetadata();
1050 }
1051 if (Options.StackDepth && IsEntryBB && !IsLeafFunc) {
1052 Module *M = F.getParent();
1053 const DataLayout &DL = M->getDataLayout();
1054
1055 if (Options.StackDepthCallbackMin) {
1056 // In callback mode, only add call when stack depth reaches minimum.
1057 int EstimatedStackSize = 0;
1058 // If dynamic alloca found, always add call.
1059 bool HasDynamicAlloc = false;
1060 // Find an insertion point after last "alloca".
1061 llvm::Instruction *InsertBefore = nullptr;
1062
1063 // Examine all allocas in the basic block. since we're too early
1064 // to have results from Intrinsic::frameaddress, we have to manually
1065 // estimate the stack size.
1066 for (auto &I : BB) {
1067 if (auto *AI = dyn_cast<AllocaInst>(Val: &I)) {
1068 // Move potential insertion point past the "alloca".
1069 InsertBefore = AI->getNextNode();
1070
1071 // Make an estimate on the stack usage.
1072 if (auto AllocaSize = AI->getAllocationSize(DL)) {
1073 if (AllocaSize->isFixed())
1074 EstimatedStackSize += AllocaSize->getFixedValue();
1075 else
1076 HasDynamicAlloc = true;
1077 } else {
1078 HasDynamicAlloc = true;
1079 }
1080 }
1081 }
1082
1083 if (HasDynamicAlloc ||
1084 EstimatedStackSize >= Options.StackDepthCallbackMin) {
1085 if (InsertBefore)
1086 IRB.SetInsertPoint(InsertBefore);
1087 auto Call = IRB.CreateCall(Callee: SanCovStackDepthCallback);
1088 if (EntryLoc)
1089 Call->setDebugLoc(EntryLoc);
1090 Call->setCannotMerge();
1091 }
1092 } else {
1093 // Check stack depth. If it's the deepest so far, record it.
1094 auto FrameAddrPtr = IRB.CreateIntrinsic(
1095 ID: Intrinsic::frameaddress, OverloadTypes: IRB.getPtrTy(AddrSpace: DL.getAllocaAddrSpace()),
1096 Args: {Constant::getNullValue(Ty: Int32Ty)});
1097 auto FrameAddrInt = IRB.CreatePtrToInt(V: FrameAddrPtr, DestTy: IntptrTy);
1098 auto LowestStack = IRB.CreateLoad(Ty: IntptrTy, Ptr: SanCovLowestStack);
1099 auto IsStackLower = IRB.CreateICmpULT(LHS: FrameAddrInt, RHS: LowestStack);
1100 auto ThenTerm = SplitBlockAndInsertIfThen(
1101 Cond: IsStackLower, SplitBefore: &*IP, Unreachable: false,
1102 BranchWeights: MDBuilder(IRB.getContext()).createUnlikelyBranchWeights());
1103 InstrumentationIRBuilder ThenIRB(ThenTerm);
1104 auto Store = ThenIRB.CreateStore(Val: FrameAddrInt, Ptr: SanCovLowestStack);
1105 if (EntryLoc)
1106 Store->setDebugLoc(EntryLoc);
1107 LowestStack->setNoSanitizeMetadata();
1108 Store->setNoSanitizeMetadata();
1109 }
1110 }
1111}
1112
1113std::string
1114ModuleSanitizerCoverage::getSectionName(const std::string &Section) const {
1115 if (TargetTriple.isOSBinFormatCOFF()) {
1116 if (Section == SanCovCountersSectionName)
1117 return ".SCOV$CM";
1118 if (Section == SanCovBoolFlagSectionName)
1119 return ".SCOV$BM";
1120 if (Section == SanCovPCsSectionName)
1121 return ".SCOVP$M";
1122 return ".SCOV$GM"; // For SanCovGuardsSectionName.
1123 }
1124 if (TargetTriple.isOSBinFormatMachO())
1125 return "__DATA,__" + Section;
1126 return "__" + Section;
1127}
1128
1129std::string
1130ModuleSanitizerCoverage::getSectionStart(const std::string &Section) const {
1131 if (TargetTriple.isOSBinFormatMachO())
1132 return "\1section$start$__DATA$__" + Section;
1133 return "__start___" + Section;
1134}
1135
1136std::string
1137ModuleSanitizerCoverage::getSectionEnd(const std::string &Section) const {
1138 if (TargetTriple.isOSBinFormatMachO())
1139 return "\1section$end$__DATA$__" + Section;
1140 return "__stop___" + Section;
1141}
1142
1143void ModuleSanitizerCoverage::createFunctionControlFlow(Function &F) {
1144 SmallVector<Constant *, 32> CFs;
1145 IRBuilder<> IRB(&*F.getEntryBlock().getFirstInsertionPt());
1146
1147 for (auto &BB : F) {
1148 // blockaddress can not be used on function's entry block.
1149 if (&BB == &F.getEntryBlock())
1150 CFs.push_back(Elt: (Constant *)IRB.CreatePointerCast(V: &F, DestTy: PtrTy));
1151 else
1152 CFs.push_back(
1153 Elt: (Constant *)IRB.CreatePointerCast(V: BlockAddress::get(BB: &BB), DestTy: PtrTy));
1154
1155 for (auto SuccBB : successors(BB: &BB)) {
1156 assert(SuccBB != &F.getEntryBlock());
1157 CFs.push_back(
1158 Elt: (Constant *)IRB.CreatePointerCast(V: BlockAddress::get(BB: SuccBB), DestTy: PtrTy));
1159 }
1160
1161 CFs.push_back(Elt: (Constant *)Constant::getNullValue(Ty: PtrTy));
1162
1163 for (auto &Inst : BB) {
1164 if (CallBase *CB = dyn_cast<CallBase>(Val: &Inst)) {
1165 if (CB->isIndirectCall()) {
1166 // TODO(navidem): handle indirect calls, for now mark its existence.
1167 CFs.push_back(Elt: (Constant *)IRB.CreateIntToPtr(
1168 V: ConstantInt::getAllOnesValue(Ty: IntptrTy), DestTy: PtrTy));
1169 } else {
1170 auto CalledF = CB->getCalledFunction();
1171 if (CalledF && !CalledF->isIntrinsic())
1172 CFs.push_back(Elt: (Constant *)IRB.CreatePointerCast(V: CalledF, DestTy: PtrTy));
1173 }
1174 }
1175 }
1176
1177 CFs.push_back(Elt: (Constant *)Constant::getNullValue(Ty: PtrTy));
1178 }
1179
1180 FunctionCFsArray = CreateFunctionLocalArrayInSection(NumElements: CFs.size(), F, Ty: PtrTy,
1181 Section: SanCovCFsSectionName);
1182 FunctionCFsArray->setInitializer(
1183 ConstantArray::get(T: ArrayType::get(ElementType: PtrTy, NumElements: CFs.size()), V: CFs));
1184 FunctionCFsArray->setConstant(true);
1185}
1186