1//===- GCOVProfiling.cpp - Insert edge counters for gcov profiling --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass implements GCOV-style profiling. When this pass is run it emits
10// "gcno" files next to the existing source, and instruments the code that runs
11// to records the edges between blocks that run and emit a complementary "gcda"
12// file on exit.
13//
14//===----------------------------------------------------------------------===//
15
16#include "llvm/ADT/Hashing.h"
17#include "llvm/ADT/MapVector.h"
18#include "llvm/ADT/STLExtras.h"
19#include "llvm/ADT/Sequence.h"
20#include "llvm/ADT/StringMap.h"
21#include "llvm/Analysis/BlockFrequencyInfo.h"
22#include "llvm/Analysis/BranchProbabilityInfo.h"
23#include "llvm/Analysis/TargetLibraryInfo.h"
24#include "llvm/IR/DebugInfo.h"
25#include "llvm/IR/DebugLoc.h"
26#include "llvm/IR/EHPersonalities.h"
27#include "llvm/IR/IRBuilder.h"
28#include "llvm/IR/InstIterator.h"
29#include "llvm/IR/Instructions.h"
30#include "llvm/IR/Module.h"
31#include "llvm/ProfileData/InstrProf.h"
32#include "llvm/Support/CRC.h"
33#include "llvm/Support/CommandLine.h"
34#include "llvm/Support/Debug.h"
35#include "llvm/Support/FileSystem.h"
36#include "llvm/Support/Path.h"
37#include "llvm/Support/Regex.h"
38#include "llvm/Support/VirtualFileSystem.h"
39#include "llvm/Support/raw_ostream.h"
40#include "llvm/Transforms/Instrumentation/CFGMST.h"
41#include "llvm/Transforms/Instrumentation/GCOVProfiler.h"
42#include "llvm/Transforms/Utils/Instrumentation.h"
43#include "llvm/Transforms/Utils/ModuleUtils.h"
44#include <algorithm>
45#include <memory>
46#include <string>
47#include <utility>
48
49using namespace llvm;
50namespace endian = llvm::support::endian;
51
52#define DEBUG_TYPE "insert-gcov-profiling"
53
54enum : uint32_t {
55 GCOV_ARC_ON_TREE = 1 << 0,
56
57 GCOV_TAG_FUNCTION = 0x01000000,
58 GCOV_TAG_BLOCKS = 0x01410000,
59 GCOV_TAG_ARCS = 0x01430000,
60 GCOV_TAG_LINES = 0x01450000,
61};
62
63static cl::opt<std::string> DefaultGCOVVersion("default-gcov-version",
64 cl::init(Val: "0000"), cl::Hidden,
65 cl::ValueRequired);
66
67static cl::opt<bool> AtomicCounter("gcov-atomic-counter", cl::Hidden,
68 cl::desc("Make counter updates atomic"));
69
70// Returns the number of words which will be used to represent this string.
71static unsigned wordsOfString(StringRef s) {
72 // Length + NUL-terminated string + 0~3 padding NULs.
73 return (s.size() / 4) + 2;
74}
75
76GCOVOptions GCOVOptions::getDefault() {
77 GCOVOptions Options;
78 Options.EmitNotes = true;
79 Options.EmitData = true;
80 Options.NoRedZone = false;
81 Options.Atomic = AtomicCounter;
82
83 if (DefaultGCOVVersion.size() != 4) {
84 reportFatalUsageError(reason: Twine("Invalid -default-gcov-version: ") +
85 DefaultGCOVVersion);
86 }
87 memcpy(dest: Options.Version, src: DefaultGCOVVersion.c_str(), n: 4);
88 return Options;
89}
90
91namespace {
92class GCOVFunction;
93
94class GCOVProfiler {
95public:
96 GCOVProfiler()
97 : GCOVProfiler(GCOVOptions::getDefault(), *vfs::getRealFileSystem()) {}
98 GCOVProfiler(const GCOVOptions &Opts, vfs::FileSystem &VFS)
99 : Options(Opts), VFS(VFS) {}
100 bool
101 runOnModule(Module &M, function_ref<BlockFrequencyInfo *(Function &F)> GetBFI,
102 function_ref<BranchProbabilityInfo *(Function &F)> GetBPI,
103 std::function<const TargetLibraryInfo &(Function &F)> GetTLI);
104
105 void write(uint32_t i) {
106 char Bytes[4];
107 endian::write32(P: Bytes, V: i, E: Endian);
108 os->write(Ptr: Bytes, Size: 4);
109 }
110 void writeString(StringRef s) {
111 write(i: wordsOfString(s) - 1);
112 os->write(Ptr: s.data(), Size: s.size());
113 os->write_zeros(NumZeros: 4 - s.size() % 4);
114 }
115 void writeBytes(const char *Bytes, int Size) { os->write(Ptr: Bytes, Size); }
116 vfs::FileSystem &getVirtualFileSystem() const { return VFS; }
117
118private:
119 // Create the .gcno files for the Module based on DebugInfo.
120 bool
121 emitProfileNotes(NamedMDNode *CUNode, bool HasExecOrFork,
122 function_ref<BlockFrequencyInfo *(Function &F)> GetBFI,
123 function_ref<BranchProbabilityInfo *(Function &F)> GetBPI,
124 function_ref<const TargetLibraryInfo &(Function &F)> GetTLI);
125
126 Function *createInternalFunction(FunctionType *FTy, StringRef Name,
127 StringRef MangledType = "");
128
129 void emitGlobalConstructor(
130 SmallVectorImpl<std::pair<GlobalVariable *, MDNode *>> &CountersBySP);
131 void emitModuleInitFunctionPtrs(
132 SmallVectorImpl<std::pair<GlobalVariable *, MDNode *>> &CountersBySP);
133
134 bool isFunctionInstrumented(const Function &F);
135 std::vector<Regex> createRegexesFromString(StringRef RegexesStr);
136 static bool doesFilenameMatchARegex(StringRef Filename,
137 std::vector<Regex> &Regexes);
138
139 // Get pointers to the functions in the runtime library.
140 FunctionCallee getStartFileFunc(const TargetLibraryInfo *TLI);
141 FunctionCallee getEmitFunctionFunc(const TargetLibraryInfo *TLI);
142 FunctionCallee getEmitArcsFunc(const TargetLibraryInfo *TLI);
143 FunctionCallee getSummaryInfoFunc();
144 FunctionCallee getEndFileFunc();
145
146 // Add the function to write out all our counters to the global destructor
147 // list.
148 Function *
149 insertCounterWriteout(ArrayRef<std::pair<GlobalVariable *, MDNode *>>);
150 Function *insertReset(ArrayRef<std::pair<GlobalVariable *, MDNode *>>);
151
152 bool AddFlushBeforeForkAndExec();
153
154 enum class GCovFileType { GCNO, GCDA };
155 std::string mangleName(const DICompileUnit *CU, GCovFileType FileType);
156
157 GCOVOptions Options;
158 llvm::endianness Endian;
159 raw_ostream *os;
160 int Version = 0;
161
162 // Checksum, produced by hash of EdgeDestinations
163 SmallVector<uint32_t, 4> FileChecksums;
164
165 Module *M = nullptr;
166 std::function<const TargetLibraryInfo &(Function &F)> GetTLI;
167 LLVMContext *Ctx = nullptr;
168 SmallVector<std::unique_ptr<GCOVFunction>, 16> Funcs;
169 std::vector<Regex> FilterRe;
170 std::vector<Regex> ExcludeRe;
171 DenseSet<const BasicBlock *> ExecBlocks;
172 StringMap<bool> InstrumentedFiles;
173 vfs::FileSystem &VFS;
174};
175
176struct BBInfo {
177 BBInfo *Group;
178 uint32_t Index;
179 uint32_t Rank = 0;
180
181 BBInfo(unsigned Index) : Group(this), Index(Index) {}
182 std::string infoString() const {
183 return (Twine("Index=") + Twine(Index)).str();
184 }
185};
186
187struct Edge {
188 // This class implements the CFG edges. Note the CFG can be a multi-graph.
189 // So there might be multiple edges with same SrcBB and DestBB.
190 const BasicBlock *SrcBB;
191 const BasicBlock *DestBB;
192 uint64_t Weight;
193 BasicBlock *Place = nullptr;
194 uint32_t SrcNumber, DstNumber;
195 bool InMST = false;
196 bool Removed = false;
197 bool IsCritical = false;
198
199 Edge(const BasicBlock *Src, const BasicBlock *Dest, uint64_t W = 1)
200 : SrcBB(Src), DestBB(Dest), Weight(W) {}
201
202 // Return the information string of an edge.
203 std::string infoString() const {
204 return (Twine(Removed ? "-" : " ") + (InMST ? " " : "*") +
205 (IsCritical ? "c" : " ") + " W=" + Twine(Weight))
206 .str();
207 }
208};
209}
210
211static StringRef getFunctionName(const DISubprogram *SP) {
212 if (!SP->getLinkageName().empty())
213 return SP->getLinkageName();
214 return SP->getName();
215}
216
217/// Extract a filename for a DIScope.
218///
219/// Prefer relative paths in the coverage notes. Clang also may split
220/// up absolute paths into a directory and filename component. When
221/// the relative path doesn't exist, reconstruct the absolute path.
222static SmallString<128> getFilename(const DIScope *SP, vfs::FileSystem &VFS) {
223 SmallString<128> Path;
224 StringRef RelPath = SP->getFilename();
225 if (VFS.exists(Path: RelPath))
226 Path = RelPath;
227 else
228 sys::path::append(path&: Path, a: SP->getDirectory(), b: SP->getFilename());
229 return Path;
230}
231
232namespace {
233 class GCOVRecord {
234 protected:
235 GCOVProfiler *P;
236
237 GCOVRecord(GCOVProfiler *P) : P(P) {}
238
239 void write(uint32_t i) { P->write(i); }
240 void writeString(StringRef s) { P->writeString(s); }
241 void writeBytes(const char *Bytes, int Size) { P->writeBytes(Bytes, Size); }
242 };
243
244 class GCOVFunction;
245 class GCOVBlock;
246
247 // Constructed only by requesting it from a GCOVBlock, this object stores a
248 // list of line numbers and a single filename, representing lines that belong
249 // to the block.
250 class GCOVLines : public GCOVRecord {
251 public:
252 StringRef getFilename() { return Filename; }
253
254 void addLine(uint32_t Line) {
255 assert(Line != 0 && "Line zero is not a valid real line number.");
256 Lines.push_back(Elt: Line);
257 }
258
259 uint32_t length() const {
260 return 1 + wordsOfString(s: Filename) + Lines.size();
261 }
262
263 void writeOut() {
264 write(i: 0);
265 writeString(s: Filename);
266 for (uint32_t L : Lines)
267 write(i: L);
268 }
269
270 GCOVLines(GCOVProfiler *P, StringRef F)
271 : GCOVRecord(P), Filename(std::string(F)) {}
272
273 private:
274 std::string Filename;
275 SmallVector<uint32_t, 32> Lines;
276 };
277
278
279 // Represent a basic block in GCOV. Each block has a unique number in the
280 // function, number of lines belonging to each block, and a set of edges to
281 // other blocks.
282 class GCOVBlock : public GCOVRecord {
283 public:
284 GCOVLines &getFile(StringRef Filename) {
285 if (Lines.empty() || Lines.back().getFilename() != Filename)
286 Lines.emplace_back(Args&: P, Args&: Filename);
287 return Lines.back();
288 }
289
290 void addEdge(GCOVBlock &Successor, uint32_t Flags) {
291 OutEdges.emplace_back(Args: &Successor, Args&: Flags);
292 }
293
294 void writeOut() {
295 uint32_t Len = 3;
296
297 for (auto &L : Lines)
298 Len += L.length();
299
300 write(i: GCOV_TAG_LINES);
301 write(i: Len);
302 write(i: Number);
303
304 for (auto &L : Lines)
305 L.writeOut();
306 write(i: 0);
307 write(i: 0);
308 }
309
310 GCOVBlock(const GCOVBlock &RHS) : GCOVRecord(RHS), Number(RHS.Number) {
311 // Only allow copy before edges and lines have been added. After that,
312 // there are inter-block pointers (eg: edges) that won't take kindly to
313 // blocks being copied or moved around.
314 assert(Lines.empty());
315 assert(OutEdges.empty());
316 }
317
318 uint32_t Number;
319 SmallVector<std::pair<GCOVBlock *, uint32_t>, 4> OutEdges;
320
321 private:
322 friend class GCOVFunction;
323
324 GCOVBlock(GCOVProfiler *P, uint32_t Number)
325 : GCOVRecord(P), Number(Number) {}
326
327 SmallVector<GCOVLines> Lines;
328 };
329
330 // A function has a unique identifier, a checksum (we leave as zero) and a
331 // set of blocks and a map of edges between blocks. This is the only GCOV
332 // object users can construct, the blocks and lines will be rooted here.
333 class GCOVFunction : public GCOVRecord {
334 public:
335 GCOVFunction(GCOVProfiler *P, Function *F, const DISubprogram *SP,
336 unsigned EndLine, uint32_t Ident, int Version)
337 : GCOVRecord(P), SP(SP), EndLine(EndLine), Ident(Ident),
338 Version(Version), EntryBlock(P, 0), ReturnBlock(P, 1) {
339 LLVM_DEBUG(dbgs() << "Function: " << getFunctionName(SP) << "\n");
340 uint32_t i = 2;
341 for (BasicBlock &BB : *F)
342 Blocks.insert(KV: std::make_pair(x: &BB, y: GCOVBlock(P, i++)));
343
344 std::string FunctionNameAndLine;
345 raw_string_ostream FNLOS(FunctionNameAndLine);
346 FNLOS << getFunctionName(SP) << SP->getLine();
347 FuncChecksum = hash_value(arg: FunctionNameAndLine);
348 }
349
350 GCOVBlock &getBlock(const BasicBlock *BB) {
351 return Blocks.find(Key: const_cast<BasicBlock *>(BB))->second;
352 }
353
354 GCOVBlock &getEntryBlock() { return EntryBlock; }
355 GCOVBlock &getReturnBlock() {
356 return ReturnBlock;
357 }
358
359 uint32_t getFuncChecksum() const {
360 return FuncChecksum;
361 }
362
363 void writeOut(uint32_t CfgChecksum) {
364 write(i: GCOV_TAG_FUNCTION);
365 SmallString<128> Filename = getFilename(SP, VFS&: P->getVirtualFileSystem());
366 uint32_t BlockLen = 3 + wordsOfString(s: getFunctionName(SP));
367 BlockLen += 1 + wordsOfString(s: Filename) + 4;
368
369 write(i: BlockLen);
370 write(i: Ident);
371 write(i: FuncChecksum);
372 write(i: CfgChecksum);
373 writeString(s: getFunctionName(SP));
374
375 write(i: SP->isArtificial()); // artificial
376 writeString(s: Filename);
377 write(i: SP->getLine()); // start_line
378 write(i: 0); // start_column
379 // EndLine is the last line with !dbg. It is not the } line as in GCC,
380 // but good enough.
381 write(i: EndLine);
382 write(i: 0); // end_column
383
384 // Emit count of blocks.
385 write(i: GCOV_TAG_BLOCKS);
386 write(i: 1);
387 write(i: Blocks.size() + 2);
388 LLVM_DEBUG(dbgs() << (Blocks.size() + 1) << " blocks\n");
389
390 // Emit edges between blocks.
391 const uint32_t Outgoing = EntryBlock.OutEdges.size();
392 if (Outgoing) {
393 write(i: GCOV_TAG_ARCS);
394 write(i: Outgoing * 2 + 1);
395 write(i: EntryBlock.Number);
396 for (const auto &E : EntryBlock.OutEdges) {
397 write(i: E.first->Number);
398 write(i: E.second);
399 }
400 }
401 for (auto &It : Blocks) {
402 const GCOVBlock &Block = It.second;
403 if (Block.OutEdges.empty()) continue;
404
405 write(i: GCOV_TAG_ARCS);
406 write(i: Block.OutEdges.size() * 2 + 1);
407 write(i: Block.Number);
408 for (const auto &E : Block.OutEdges) {
409 write(i: E.first->Number);
410 write(i: E.second);
411 }
412 }
413
414 // Emit lines for each block.
415 for (auto &It : Blocks)
416 It.second.writeOut();
417 }
418
419 public:
420 const DISubprogram *SP;
421 unsigned EndLine;
422 uint32_t Ident;
423 uint32_t FuncChecksum;
424 int Version;
425 MapVector<BasicBlock *, GCOVBlock> Blocks;
426 GCOVBlock EntryBlock;
427 GCOVBlock ReturnBlock;
428 };
429}
430
431// RegexesStr is a string containing differents regex separated by a semi-colon.
432// For example "foo\..*$;bar\..*$".
433std::vector<Regex> GCOVProfiler::createRegexesFromString(StringRef RegexesStr) {
434 std::vector<Regex> Regexes;
435 while (!RegexesStr.empty()) {
436 std::pair<StringRef, StringRef> HeadTail = RegexesStr.split(Separator: ';');
437 if (!HeadTail.first.empty()) {
438 Regex Re(HeadTail.first);
439 std::string Err;
440 if (!Re.isValid(Error&: Err)) {
441 Ctx->emitError(ErrorStr: Twine("Regex ") + HeadTail.first +
442 " is not valid: " + Err);
443 }
444 Regexes.emplace_back(args: std::move(Re));
445 }
446 RegexesStr = HeadTail.second;
447 }
448 return Regexes;
449}
450
451bool GCOVProfiler::doesFilenameMatchARegex(StringRef Filename,
452 std::vector<Regex> &Regexes) {
453 for (Regex &Re : Regexes)
454 if (Re.match(String: Filename))
455 return true;
456 return false;
457}
458
459bool GCOVProfiler::isFunctionInstrumented(const Function &F) {
460 if (FilterRe.empty() && ExcludeRe.empty()) {
461 return true;
462 }
463 SmallString<128> Filename = getFilename(SP: F.getSubprogram(), VFS);
464 auto It = InstrumentedFiles.find(Key: Filename);
465 if (It != InstrumentedFiles.end()) {
466 return It->second;
467 }
468
469 SmallString<256> RealPath;
470 StringRef RealFilename;
471
472 // Path can be
473 // /usr/lib/gcc/x86_64-linux-gnu/8/../../../../include/c++/8/bits/*.h so for
474 // such a case we must get the real_path.
475 if (VFS.getRealPath(Path: Filename, Output&: RealPath)) {
476 // real_path can fail with path like "foo.c".
477 RealFilename = Filename;
478 } else {
479 RealFilename = RealPath;
480 }
481
482 bool ShouldInstrument;
483 if (FilterRe.empty()) {
484 ShouldInstrument = !doesFilenameMatchARegex(Filename: RealFilename, Regexes&: ExcludeRe);
485 } else if (ExcludeRe.empty()) {
486 ShouldInstrument = doesFilenameMatchARegex(Filename: RealFilename, Regexes&: FilterRe);
487 } else {
488 ShouldInstrument = doesFilenameMatchARegex(Filename: RealFilename, Regexes&: FilterRe) &&
489 !doesFilenameMatchARegex(Filename: RealFilename, Regexes&: ExcludeRe);
490 }
491 InstrumentedFiles[Filename] = ShouldInstrument;
492 return ShouldInstrument;
493}
494
495std::string GCOVProfiler::mangleName(const DICompileUnit *CU,
496 GCovFileType OutputType) {
497 bool Notes = OutputType == GCovFileType::GCNO;
498
499 if (NamedMDNode *GCov = M->getNamedMetadata(Name: "llvm.gcov")) {
500 for (int i = 0, e = GCov->getNumOperands(); i != e; ++i) {
501 MDNode *N = GCov->getOperand(i);
502 bool ThreeElement = N->getNumOperands() == 3;
503 if (!ThreeElement && N->getNumOperands() != 2)
504 continue;
505 if (dyn_cast<MDNode>(Val: N->getOperand(I: ThreeElement ? 2 : 1)) != CU)
506 continue;
507
508 if (ThreeElement) {
509 // These nodes have no mangling to apply, it's stored mangled in the
510 // bitcode.
511 MDString *NotesFile = dyn_cast<MDString>(Val: N->getOperand(I: 0));
512 MDString *DataFile = dyn_cast<MDString>(Val: N->getOperand(I: 1));
513 if (!NotesFile || !DataFile)
514 continue;
515 return std::string(Notes ? NotesFile->getString()
516 : DataFile->getString());
517 }
518
519 MDString *GCovFile = dyn_cast<MDString>(Val: N->getOperand(I: 0));
520 if (!GCovFile)
521 continue;
522
523 SmallString<128> Filename = GCovFile->getString();
524 sys::path::replace_extension(path&: Filename, extension: Notes ? "gcno" : "gcda");
525 return std::string(Filename);
526 }
527 }
528
529 SmallString<128> Filename = CU->getFilename();
530 sys::path::replace_extension(path&: Filename, extension: Notes ? "gcno" : "gcda");
531 StringRef FName = sys::path::filename(path: Filename);
532 ErrorOr<std::string> CWD = VFS.getCurrentWorkingDirectory();
533 if (!CWD)
534 return std::string(FName);
535 SmallString<128> CurPath{*CWD};
536 sys::path::append(path&: CurPath, a: FName);
537 return std::string(CurPath);
538}
539
540bool GCOVProfiler::runOnModule(
541 Module &M, function_ref<BlockFrequencyInfo *(Function &F)> GetBFI,
542 function_ref<BranchProbabilityInfo *(Function &F)> GetBPI,
543 std::function<const TargetLibraryInfo &(Function &F)> GetTLI) {
544 this->M = &M;
545 this->GetTLI = std::move(GetTLI);
546 Ctx = &M.getContext();
547
548 NamedMDNode *CUNode = M.getNamedMetadata(Name: "llvm.dbg.cu");
549 if (!CUNode || (!Options.EmitNotes && !Options.EmitData))
550 return false;
551
552 bool HasExecOrFork = AddFlushBeforeForkAndExec();
553
554 FilterRe = createRegexesFromString(RegexesStr: Options.Filter);
555 ExcludeRe = createRegexesFromString(RegexesStr: Options.Exclude);
556 emitProfileNotes(CUNode, HasExecOrFork, GetBFI, GetBPI, GetTLI: this->GetTLI);
557 return true;
558}
559
560PreservedAnalyses GCOVProfilerPass::run(Module &M,
561 ModuleAnalysisManager &AM) {
562
563 GCOVProfiler Profiler(GCOVOpts, *VFS);
564 FunctionAnalysisManager &FAM =
565 AM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
566
567 auto GetBFI = [&FAM](Function &F) {
568 return &FAM.getResult<BlockFrequencyAnalysis>(IR&: F);
569 };
570 auto GetBPI = [&FAM](Function &F) {
571 return &FAM.getResult<BranchProbabilityAnalysis>(IR&: F);
572 };
573 auto GetTLI = [&FAM](Function &F) -> const TargetLibraryInfo & {
574 return FAM.getResult<TargetLibraryAnalysis>(IR&: F);
575 };
576
577 if (!Profiler.runOnModule(M, GetBFI, GetBPI, GetTLI))
578 return PreservedAnalyses::all();
579
580 return PreservedAnalyses::none();
581}
582
583static bool functionHasLines(const Function &F, unsigned &EndLine) {
584 // Check whether this function actually has any source lines. Not only
585 // do these waste space, they also can crash gcov.
586 EndLine = 0;
587 for (const auto &BB : F) {
588 for (const auto &I : BB) {
589 const DebugLoc &Loc = I.getDebugLoc();
590 if (!Loc)
591 continue;
592
593 // Artificial lines such as calls to the global constructors.
594 if (Loc.getLine() == 0) continue;
595 EndLine = std::max(a: EndLine, b: Loc.getLine());
596
597 return true;
598 }
599 }
600 return false;
601}
602
603static bool isUsingScopeBasedEH(Function &F) {
604 if (!F.hasPersonalityFn()) return false;
605
606 EHPersonality Personality = classifyEHPersonality(Pers: F.getPersonalityFn());
607 return isScopedEHPersonality(Pers: Personality);
608}
609
610bool GCOVProfiler::AddFlushBeforeForkAndExec() {
611 const TargetLibraryInfo *TLI = nullptr;
612 SmallVector<CallInst *, 2> Forks;
613 SmallVector<CallInst *, 2> Execs;
614 for (auto &F : M->functions()) {
615 TLI = TLI == nullptr ? &GetTLI(F) : TLI;
616 for (auto &I : instructions(F)) {
617 if (CallInst *CI = dyn_cast<CallInst>(Val: &I)) {
618 if (Function *Callee = CI->getCalledFunction()) {
619 LibFunc LF;
620 if (TLI->getLibFunc(FDecl: *Callee, F&: LF)) {
621 if (LF == LibFunc_fork) {
622#if !defined(_WIN32)
623 Forks.push_back(Elt: CI);
624#endif
625 } else if (LF == LibFunc_execl || LF == LibFunc_execle ||
626 LF == LibFunc_execlp || LF == LibFunc_execv ||
627 LF == LibFunc_execvp || LF == LibFunc_execve ||
628 LF == LibFunc_execvpe || LF == LibFunc_execvP) {
629 Execs.push_back(Elt: CI);
630 }
631 }
632 }
633 }
634 }
635 }
636
637 for (auto *F : Forks) {
638 IRBuilder<> Builder(F);
639 BasicBlock *Parent = F->getParent();
640 auto NextInst = ++F->getIterator();
641
642 // We've a fork so just reset the counters in the child process
643 FunctionType *FTy = FunctionType::get(Result: Builder.getInt32Ty(), Params: {}, isVarArg: false);
644 FunctionCallee GCOVFork = M->getOrInsertFunction(
645 Name: "__gcov_fork", T: FTy,
646 AttributeList: TLI->getAttrList(C: Ctx, ArgNos: {}, /*Signed=*/true, /*Ret=*/true));
647 F->setCalledFunction(GCOVFork);
648
649 // We split just after the fork to have a counter for the lines after
650 // Anyway there's a bug:
651 // void foo() { fork(); }
652 // void bar() { foo(); blah(); }
653 // then "blah();" will be called 2 times but showed as 1
654 // because "blah()" belongs to the same block as "foo();"
655 Parent->splitBasicBlock(I: NextInst);
656
657 // back() is a br instruction with a debug location
658 // equals to the one from NextAfterFork
659 // So to avoid to have two debug locs on two blocks just change it
660 DebugLoc Loc = F->getDebugLoc();
661 Parent->back().setDebugLoc(Loc);
662 }
663
664 for (auto *E : Execs) {
665 IRBuilder<> Builder(E);
666 BasicBlock *Parent = E->getParent();
667 auto NextInst = ++E->getIterator();
668
669 // Since the process is replaced by a new one we need to write out gcdas
670 // No need to reset the counters since they'll be lost after the exec**
671 FunctionType *FTy = FunctionType::get(Result: Builder.getVoidTy(), Params: {}, isVarArg: false);
672 FunctionCallee WriteoutF =
673 M->getOrInsertFunction(Name: "llvm_writeout_files", T: FTy);
674 Builder.CreateCall(Callee: WriteoutF);
675
676 DebugLoc Loc = E->getDebugLoc();
677 Builder.SetInsertPoint(&*NextInst);
678 // If the exec** fails we must reset the counters since they've been
679 // dumped
680 FunctionCallee ResetF = M->getOrInsertFunction(Name: "llvm_reset_counters", T: FTy);
681 Builder.CreateCall(Callee: ResetF)->setDebugLoc(Loc);
682 ExecBlocks.insert(V: Parent);
683 Parent->splitBasicBlock(I: NextInst);
684 Parent->back().setDebugLoc(Loc);
685 }
686
687 return !Forks.empty() || !Execs.empty();
688}
689
690static BasicBlock *getInstrBB(CFGMST<Edge, BBInfo> &MST, Edge &E,
691 const DenseSet<const BasicBlock *> &ExecBlocks) {
692 if (E.InMST || E.Removed)
693 return nullptr;
694
695 BasicBlock *SrcBB = const_cast<BasicBlock *>(E.SrcBB);
696 BasicBlock *DestBB = const_cast<BasicBlock *>(E.DestBB);
697 // For a fake edge, instrument the real BB.
698 if (SrcBB == nullptr)
699 return DestBB;
700 if (DestBB == nullptr)
701 return SrcBB;
702
703 auto CanInstrument = [](BasicBlock *BB) -> BasicBlock * {
704 // There are basic blocks (such as catchswitch) cannot be instrumented.
705 // If the returned first insertion point is the end of BB, skip this BB.
706 if (BB->getFirstInsertionPt() == BB->end())
707 return nullptr;
708 return BB;
709 };
710
711 // Instrument the SrcBB if it has a single successor,
712 // otherwise, the DestBB if this is not a critical edge.
713 Instruction *TI = SrcBB->getTerminator();
714 if (TI->getNumSuccessors() <= 1 && !ExecBlocks.count(V: SrcBB))
715 return CanInstrument(SrcBB);
716 if (!E.IsCritical)
717 return CanInstrument(DestBB);
718
719 // Some IndirectBr critical edges cannot be split by the previous
720 // SplitIndirectBrCriticalEdges call. Bail out.
721 const unsigned SuccNum = GetSuccessorNumber(BB: SrcBB, Succ: DestBB);
722 BasicBlock *InstrBB =
723 isa<IndirectBrInst>(Val: TI) ? nullptr : SplitCriticalEdge(TI, SuccNum);
724 if (!InstrBB)
725 return nullptr;
726
727 MST.addEdge(Src: SrcBB, Dest: InstrBB, W: 0);
728 MST.addEdge(Src: InstrBB, Dest: DestBB, W: 0).InMST = true;
729 E.Removed = true;
730
731 return CanInstrument(InstrBB);
732}
733
734#ifndef NDEBUG
735static void dumpEdges(CFGMST<Edge, BBInfo> &MST, GCOVFunction &GF) {
736 size_t ID = 0;
737 for (const auto &E : make_pointee_range(MST.allEdges())) {
738 GCOVBlock &Src = E.SrcBB ? GF.getBlock(E.SrcBB) : GF.getEntryBlock();
739 GCOVBlock &Dst = E.DestBB ? GF.getBlock(E.DestBB) : GF.getReturnBlock();
740 dbgs() << " Edge " << ID++ << ": " << Src.Number << "->" << Dst.Number
741 << E.infoString() << "\n";
742 }
743}
744#endif
745
746bool GCOVProfiler::emitProfileNotes(
747 NamedMDNode *CUNode, bool HasExecOrFork,
748 function_ref<BlockFrequencyInfo *(Function &F)> GetBFI,
749 function_ref<BranchProbabilityInfo *(Function &F)> GetBPI,
750 function_ref<const TargetLibraryInfo &(Function &F)> GetTLI) {
751 {
752 uint8_t c3 = Options.Version[0];
753 uint8_t c2 = Options.Version[1];
754 uint8_t c1 = Options.Version[2];
755 Version = c3 >= 'A' ? (c3 - 'A') * 100 + (c2 - '0') * 10 + c1 - '0'
756 : (c3 - '0') * 10 + c1 - '0';
757 }
758 // Emit .gcno files that are compatible with GCC 11.1.
759 if (Version < 111) {
760 Version = 111;
761 memcpy(dest: Options.Version, src: "B11*", n: 4);
762 }
763
764 bool EmitGCDA = Options.EmitData;
765 for (unsigned i = 0, e = CUNode->getNumOperands(); i != e; ++i) {
766 // Each compile unit gets its own .gcno file. This means that whether we run
767 // this pass over the original .o's as they're produced, or run it after
768 // LTO, we'll generate the same .gcno files.
769
770 auto *CU = cast<DICompileUnit>(Val: CUNode->getOperand(i));
771
772 // Skip module skeleton (and module) CUs.
773 if (CU->getDWOId())
774 continue;
775
776 std::vector<uint8_t> EdgeDestinations;
777 SmallVector<std::pair<GlobalVariable *, MDNode *>, 8> CountersBySP;
778
779 Endian = M->getDataLayout().isLittleEndian() ? llvm::endianness::little
780 : llvm::endianness::big;
781 unsigned FunctionIdent = 0;
782 for (auto &F : M->functions()) {
783 DISubprogram *SP = F.getSubprogram();
784 unsigned EndLine;
785 if (!SP) continue;
786 if (!functionHasLines(F, EndLine) || !isFunctionInstrumented(F))
787 continue;
788 // TODO: Functions using scope-based EH are currently not supported.
789 if (isUsingScopeBasedEH(F)) continue;
790 if (F.hasFnAttribute(Kind: llvm::Attribute::NoProfile))
791 continue;
792 if (F.hasFnAttribute(Kind: llvm::Attribute::SkipProfile))
793 continue;
794
795 // Add the function line number to the lines of the entry block
796 // to have a counter for the function definition.
797 uint32_t Line = SP->getLine();
798 auto Filename = getFilename(SP, VFS);
799
800 BranchProbabilityInfo *BPI = GetBPI(F);
801 BlockFrequencyInfo *BFI = GetBFI(F);
802
803 // Split indirectbr critical edges here before computing the MST rather
804 // than later in getInstrBB() to avoid invalidating it.
805 SplitIndirectBrCriticalEdges(F, /*IgnoreBlocksWithoutPHI=*/false, BPI,
806 BFI);
807
808 CFGMST<Edge, BBInfo> MST(F, /*InstrumentFuncEntry=*/false,
809 /*InstrumentLoopEntries=*/false, BPI, BFI);
810
811 // getInstrBB can split basic blocks and push elements to AllEdges.
812 for (size_t I : llvm::seq<size_t>(Begin: 0, End: MST.numEdges())) {
813 auto &E = *MST.allEdges()[I];
814 // For now, disable spanning tree optimization when fork or exec* is
815 // used.
816 if (HasExecOrFork)
817 E.InMST = false;
818 E.Place = getInstrBB(MST, E, ExecBlocks);
819 }
820 // Basic blocks in F are finalized at this point.
821 BasicBlock &EntryBlock = F.getEntryBlock();
822 Funcs.push_back(Elt: std::make_unique<GCOVFunction>(args: this, args: &F, args&: SP, args&: EndLine,
823 args: FunctionIdent++, args&: Version));
824 GCOVFunction &Func = *Funcs.back();
825
826 // Some non-tree edges are IndirectBr which cannot be split. Ignore them
827 // as well.
828 llvm::erase_if(C&: MST.allEdges(), P: [](std::unique_ptr<Edge> &E) {
829 return E->Removed || (!E->InMST && !E->Place);
830 });
831 const size_t Measured =
832 std::stable_partition(
833 first: MST.allEdges().begin(), last: MST.allEdges().end(),
834 pred: [](std::unique_ptr<Edge> &E) { return E->Place; }) -
835 MST.allEdges().begin();
836 for (size_t I : llvm::seq<size_t>(Begin: 0, End: Measured)) {
837 Edge &E = *MST.allEdges()[I];
838 GCOVBlock &Src =
839 E.SrcBB ? Func.getBlock(BB: E.SrcBB) : Func.getEntryBlock();
840 GCOVBlock &Dst =
841 E.DestBB ? Func.getBlock(BB: E.DestBB) : Func.getReturnBlock();
842 E.SrcNumber = Src.Number;
843 E.DstNumber = Dst.Number;
844 }
845 std::stable_sort(
846 first: MST.allEdges().begin(), last: MST.allEdges().begin() + Measured,
847 comp: [](const std::unique_ptr<Edge> &L, const std::unique_ptr<Edge> &R) {
848 return L->SrcNumber != R->SrcNumber ? L->SrcNumber < R->SrcNumber
849 : L->DstNumber < R->DstNumber;
850 });
851
852 for (const Edge &E : make_pointee_range(Range&: MST.allEdges())) {
853 GCOVBlock &Src =
854 E.SrcBB ? Func.getBlock(BB: E.SrcBB) : Func.getEntryBlock();
855 GCOVBlock &Dst =
856 E.DestBB ? Func.getBlock(BB: E.DestBB) : Func.getReturnBlock();
857 Src.addEdge(Successor&: Dst, Flags: E.Place ? 0 : uint32_t(GCOV_ARC_ON_TREE));
858 }
859
860 // Artificial functions such as global initializers
861 if (!SP->isArtificial())
862 Func.getBlock(BB: &EntryBlock).getFile(Filename).addLine(Line);
863
864 LLVM_DEBUG(dumpEdges(MST, Func));
865
866 for (auto &GB : Func.Blocks) {
867 const BasicBlock &BB = *GB.first;
868 auto &Block = GB.second;
869 for (auto Succ : Block.OutEdges) {
870 uint32_t Idx = Succ.first->Number;
871 do EdgeDestinations.push_back(x: Idx & 255);
872 while ((Idx >>= 8) > 0);
873 }
874
875 for (const auto &I : BB) {
876 const DebugLoc &Loc = I.getDebugLoc();
877 if (!Loc)
878 continue;
879
880 // Artificial lines such as calls to the global constructors.
881 if (Loc.getLine() == 0 || Loc.isImplicitCode())
882 continue;
883
884 if (Line == Loc.getLine()) continue;
885 Line = Loc.getLine();
886 MDNode *Scope = Loc.getScope();
887 if (SP != getDISubprogram(Scope))
888 continue;
889
890 GCOVLines &Lines = Block.getFile(Filename: getFilename(SP: Loc->getScope(), VFS));
891 Lines.addLine(Line: Loc.getLine());
892 }
893 Line = 0;
894 }
895 if (EmitGCDA) {
896 DISubprogram *SP = F.getSubprogram();
897 ArrayType *CounterTy = ArrayType::get(ElementType: Type::getInt64Ty(C&: *Ctx), NumElements: Measured);
898 GlobalVariable *Counters = new GlobalVariable(
899 *M, CounterTy, false, GlobalValue::InternalLinkage,
900 Constant::getNullValue(Ty: CounterTy), "__llvm_gcov_ctr");
901 const llvm::Triple &Triple = M->getTargetTriple();
902 if (Triple.getObjectFormat() == llvm::Triple::XCOFF)
903 Counters->setSection("__llvm_gcov_ctr_section");
904 CountersBySP.emplace_back(Args&: Counters, Args&: SP);
905
906 for (size_t I : llvm::seq<size_t>(Begin: 0, End: Measured)) {
907 const Edge &E = *MST.allEdges()[I];
908 IRBuilder<> Builder(E.Place, E.Place->getFirstInsertionPt());
909 Value *V = Builder.CreateConstInBoundsGEP2_64(
910 Ty: Counters->getValueType(), Ptr: Counters, Idx0: 0, Idx1: I);
911 // Disable sanitizers to decrease size bloat. We don't expect
912 // sanitizers to catch interesting issues.
913 Instruction *Inst;
914 if (Options.Atomic) {
915 Inst = Builder.CreateAtomicRMW(Op: AtomicRMWInst::Add, Ptr: V,
916 Val: Builder.getInt64(C: 1), Align: MaybeAlign(),
917 Ordering: AtomicOrdering::Monotonic);
918 } else {
919 LoadInst *OldCount =
920 Builder.CreateLoad(Ty: Builder.getInt64Ty(), Ptr: V, Name: "gcov_ctr");
921 OldCount->setNoSanitizeMetadata();
922 Value *NewCount = Builder.CreateAdd(LHS: OldCount, RHS: Builder.getInt64(C: 1));
923 Inst = Builder.CreateStore(Val: NewCount, Ptr: V);
924 }
925 Inst->setNoSanitizeMetadata();
926 }
927 }
928 }
929
930 char Tmp[4];
931 JamCRC JC;
932 JC.update(Data: EdgeDestinations);
933 uint32_t Stamp = JC.getCRC();
934 FileChecksums.push_back(Elt: Stamp);
935
936 if (Options.EmitNotes) {
937 std::error_code EC;
938 raw_fd_ostream out(mangleName(CU, OutputType: GCovFileType::GCNO), EC,
939 sys::fs::OF_None);
940 if (EC) {
941 Ctx->emitError(
942 ErrorStr: Twine("failed to open coverage notes file for writing: ") +
943 EC.message());
944 continue;
945 }
946 os = &out;
947 if (Endian == llvm::endianness::big) {
948 out.write(Ptr: "gcno", Size: 4);
949 out.write(Ptr: Options.Version, Size: 4);
950 } else {
951 out.write(Ptr: "oncg", Size: 4);
952 std::reverse_copy(first: Options.Version, last: Options.Version + 4, result: Tmp);
953 out.write(Ptr: Tmp, Size: 4);
954 }
955 write(i: Stamp);
956 writeString(s: "."); // unuseful current_working_directory
957 write(i: 0); // unuseful has_unexecuted_blocks
958
959 for (auto &Func : Funcs)
960 Func->writeOut(CfgChecksum: Stamp);
961
962 write(i: 0);
963 write(i: 0);
964 out.close();
965 }
966
967 if (EmitGCDA) {
968 const llvm::Triple &Triple = M->getTargetTriple();
969 if (Triple.getObjectFormat() == llvm::Triple::XCOFF)
970 emitModuleInitFunctionPtrs(CountersBySP);
971 else
972 emitGlobalConstructor(CountersBySP);
973 EmitGCDA = false;
974 }
975 }
976 return true;
977}
978
979Function *GCOVProfiler::createInternalFunction(FunctionType *FTy,
980 StringRef Name,
981 StringRef MangledType /*=""*/) {
982 Function *F = Function::createWithDefaultAttr(
983 Ty: FTy, Linkage: GlobalValue::InternalLinkage, AddrSpace: 0, N: Name, M);
984 F->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
985 F->addFnAttr(Kind: Attribute::NoUnwind);
986 if (Options.NoRedZone)
987 F->addFnAttr(Kind: Attribute::NoRedZone);
988 if (!MangledType.empty())
989 setKCFIType(M&: *M, F&: *F, MangledType);
990 return F;
991}
992
993void GCOVProfiler::emitGlobalConstructor(
994 SmallVectorImpl<std::pair<GlobalVariable *, MDNode *>> &CountersBySP) {
995 Function *WriteoutF = insertCounterWriteout(CountersBySP);
996 Function *ResetF = insertReset(CountersBySP);
997
998 // Create a small bit of code that registers the "__llvm_gcov_writeout" to
999 // be executed at exit and the "__llvm_gcov_reset" function to be executed
1000 // when "__gcov_flush" is called.
1001 FunctionType *FTy = FunctionType::get(Result: Type::getVoidTy(C&: *Ctx), isVarArg: false);
1002 Function *F = createInternalFunction(FTy, Name: "__llvm_gcov_init", MangledType: "_ZTSFvvE");
1003 F->addFnAttr(Kind: Attribute::NoInline);
1004
1005 BasicBlock *BB = BasicBlock::Create(Context&: *Ctx, Name: "entry", Parent: F);
1006 IRBuilder<> Builder(BB);
1007
1008 FTy = FunctionType::get(Result: Type::getVoidTy(C&: *Ctx), isVarArg: false);
1009 auto *PFTy = PointerType::get(C&: *Ctx, AddressSpace: 0);
1010 FTy = FunctionType::get(Result: Builder.getVoidTy(), Params: {PFTy, PFTy}, isVarArg: false);
1011
1012 // Initialize the environment and register the local writeout, flush and
1013 // reset functions.
1014 FunctionCallee GCOVInit = M->getOrInsertFunction(Name: "llvm_gcov_init", T: FTy);
1015 Builder.CreateCall(Callee: GCOVInit, Args: {WriteoutF, ResetF});
1016 Builder.CreateRetVoid();
1017
1018 appendToGlobalCtors(M&: *M, F, Priority: 0);
1019}
1020
1021void GCOVProfiler::emitModuleInitFunctionPtrs(
1022 SmallVectorImpl<std::pair<GlobalVariable *, MDNode *>> &CountersBySP) {
1023 Function *WriteoutF = insertCounterWriteout(CountersBySP);
1024 Function *ResetF = insertReset(CountersBySP);
1025
1026 // Instead of creating a function call and add it to the constructors list,
1027 // create a global variable in the __llvm_covinit section so the functions
1028 // can be registered by a constructor in the runtime.
1029
1030 auto &Ctx = M->getContext();
1031
1032 Type *InitFuncDataTy[] = {
1033#define COVINIT_FUNC(Type, LLVMType, Name, Init) LLVMType,
1034#include "llvm/ProfileData/InstrProfData.inc"
1035 };
1036
1037 auto STy = StructType::get(Context&: Ctx, Elements: ArrayRef(InitFuncDataTy));
1038
1039 Constant *InitFuncPtrs[] = {
1040#define COVINIT_FUNC(Type, LLVMType, Name, Init) Init,
1041#include "llvm/ProfileData/InstrProfData.inc"
1042 };
1043
1044 auto *CovInitGV =
1045 new GlobalVariable(*M, STy, false, GlobalValue::PrivateLinkage, nullptr,
1046 "__llvm_covinit_functions");
1047 CovInitGV->setInitializer(ConstantStruct::get(T: STy, V: InitFuncPtrs));
1048 CovInitGV->setVisibility(GlobalValue::VisibilityTypes::DefaultVisibility);
1049 CovInitGV->setSection(getInstrProfSectionName(
1050 IPSK: IPSK_covinit, OF: M->getTargetTriple().getObjectFormat()));
1051 CovInitGV->setAlignment(Align(INSTR_PROF_DATA_ALIGNMENT));
1052 CovInitGV->setConstant(true);
1053}
1054
1055FunctionCallee GCOVProfiler::getStartFileFunc(const TargetLibraryInfo *TLI) {
1056 Type *Args[] = {
1057 PointerType::getUnqual(C&: *Ctx), // const char *orig_filename
1058 Type::getInt32Ty(C&: *Ctx), // uint32_t version
1059 Type::getInt32Ty(C&: *Ctx), // uint32_t checksum
1060 };
1061 FunctionType *FTy = FunctionType::get(Result: Type::getVoidTy(C&: *Ctx), Params: Args, isVarArg: false);
1062 return M->getOrInsertFunction(Name: "llvm_gcda_start_file", T: FTy,
1063 AttributeList: TLI->getAttrList(C: Ctx, ArgNos: {1, 2}, /*Signed=*/false));
1064}
1065
1066FunctionCallee GCOVProfiler::getEmitFunctionFunc(const TargetLibraryInfo *TLI) {
1067 Type *Args[] = {
1068 Type::getInt32Ty(C&: *Ctx), // uint32_t ident
1069 Type::getInt32Ty(C&: *Ctx), // uint32_t func_checksum
1070 Type::getInt32Ty(C&: *Ctx), // uint32_t cfg_checksum
1071 };
1072 FunctionType *FTy = FunctionType::get(Result: Type::getVoidTy(C&: *Ctx), Params: Args, isVarArg: false);
1073 return M->getOrInsertFunction(Name: "llvm_gcda_emit_function", T: FTy,
1074 AttributeList: TLI->getAttrList(C: Ctx, ArgNos: {0, 1, 2}, /*Signed=*/false));
1075}
1076
1077FunctionCallee GCOVProfiler::getEmitArcsFunc(const TargetLibraryInfo *TLI) {
1078 Type *Args[] = {
1079 Type::getInt32Ty(C&: *Ctx), // uint32_t num_counters
1080 PointerType::getUnqual(C&: *Ctx), // uint64_t *counters
1081 };
1082 FunctionType *FTy = FunctionType::get(Result: Type::getVoidTy(C&: *Ctx), Params: Args, isVarArg: false);
1083 return M->getOrInsertFunction(Name: "llvm_gcda_emit_arcs", T: FTy,
1084 AttributeList: TLI->getAttrList(C: Ctx, ArgNos: {0}, /*Signed=*/false));
1085}
1086
1087FunctionCallee GCOVProfiler::getSummaryInfoFunc() {
1088 FunctionType *FTy = FunctionType::get(Result: Type::getVoidTy(C&: *Ctx), isVarArg: false);
1089 return M->getOrInsertFunction(Name: "llvm_gcda_summary_info", T: FTy);
1090}
1091
1092FunctionCallee GCOVProfiler::getEndFileFunc() {
1093 FunctionType *FTy = FunctionType::get(Result: Type::getVoidTy(C&: *Ctx), isVarArg: false);
1094 return M->getOrInsertFunction(Name: "llvm_gcda_end_file", T: FTy);
1095}
1096
1097Function *GCOVProfiler::insertCounterWriteout(
1098 ArrayRef<std::pair<GlobalVariable *, MDNode *> > CountersBySP) {
1099 FunctionType *WriteoutFTy = FunctionType::get(Result: Type::getVoidTy(C&: *Ctx), isVarArg: false);
1100 Function *WriteoutF = M->getFunction(Name: "__llvm_gcov_writeout");
1101 if (!WriteoutF)
1102 WriteoutF =
1103 createInternalFunction(FTy: WriteoutFTy, Name: "__llvm_gcov_writeout", MangledType: "_ZTSFvvE");
1104 WriteoutF->addFnAttr(Kind: Attribute::NoInline);
1105
1106 BasicBlock *BB = BasicBlock::Create(Context&: *Ctx, Name: "entry", Parent: WriteoutF);
1107 IRBuilder<> Builder(BB);
1108
1109 auto *TLI = &GetTLI(*WriteoutF);
1110
1111 FunctionCallee StartFile = getStartFileFunc(TLI);
1112 FunctionCallee EmitFunction = getEmitFunctionFunc(TLI);
1113 FunctionCallee EmitArcs = getEmitArcsFunc(TLI);
1114 FunctionCallee SummaryInfo = getSummaryInfoFunc();
1115 FunctionCallee EndFile = getEndFileFunc();
1116
1117 NamedMDNode *CUNodes = M->getNamedMetadata(Name: "llvm.dbg.cu");
1118 if (!CUNodes) {
1119 Builder.CreateRetVoid();
1120 return WriteoutF;
1121 }
1122
1123 // Collect the relevant data into a large constant data structure that we can
1124 // walk to write out everything.
1125 StructType *StartFileCallArgsTy = StructType::create(
1126 Elements: {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getInt32Ty()},
1127 Name: "start_file_args_ty");
1128 StructType *EmitFunctionCallArgsTy = StructType::create(
1129 Elements: {Builder.getInt32Ty(), Builder.getInt32Ty(), Builder.getInt32Ty()},
1130 Name: "emit_function_args_ty");
1131 auto *PtrTy = Builder.getPtrTy();
1132 StructType *EmitArcsCallArgsTy =
1133 StructType::create(Elements: {Builder.getInt32Ty(), PtrTy}, Name: "emit_arcs_args_ty");
1134 StructType *FileInfoTy = StructType::create(
1135 Elements: {StartFileCallArgsTy, Builder.getInt32Ty(), PtrTy, PtrTy}, Name: "file_info");
1136
1137 Constant *Zero32 = Builder.getInt32(C: 0);
1138 // Build an explicit array of two zeros for use in ConstantExpr GEP building.
1139 Constant *TwoZero32s[] = {Zero32, Zero32};
1140
1141 SmallVector<Constant *, 8> FileInfos;
1142 for (int i : llvm::seq<int>(Begin: 0, End: CUNodes->getNumOperands())) {
1143 auto *CU = cast<DICompileUnit>(Val: CUNodes->getOperand(i));
1144
1145 // Skip module skeleton (and module) CUs.
1146 if (CU->getDWOId())
1147 continue;
1148
1149 std::string FilenameGcda = mangleName(CU, OutputType: GCovFileType::GCDA);
1150 uint32_t CfgChecksum = FileChecksums.empty() ? 0 : FileChecksums[i];
1151 auto *StartFileCallArgs = ConstantStruct::get(
1152 T: StartFileCallArgsTy,
1153 V: {Builder.CreateGlobalString(Str: FilenameGcda),
1154 Builder.getInt32(C: endian::read32be(P: Options.Version)),
1155 Builder.getInt32(C: CfgChecksum)});
1156
1157 SmallVector<Constant *, 8> EmitFunctionCallArgsArray;
1158 SmallVector<Constant *, 8> EmitArcsCallArgsArray;
1159 for (int j : llvm::seq<int>(Begin: 0, End: CountersBySP.size())) {
1160 uint32_t FuncChecksum = Funcs.empty() ? 0 : Funcs[j]->getFuncChecksum();
1161 EmitFunctionCallArgsArray.push_back(Elt: ConstantStruct::get(
1162 T: EmitFunctionCallArgsTy,
1163 V: {Builder.getInt32(C: j),
1164 Builder.getInt32(C: FuncChecksum),
1165 Builder.getInt32(C: CfgChecksum)}));
1166
1167 GlobalVariable *GV = CountersBySP[j].first;
1168 unsigned Arcs = cast<ArrayType>(Val: GV->getValueType())->getNumElements();
1169 EmitArcsCallArgsArray.push_back(Elt: ConstantStruct::get(
1170 T: EmitArcsCallArgsTy,
1171 V: {Builder.getInt32(C: Arcs), ConstantExpr::getInBoundsGetElementPtr(
1172 Ty: GV->getValueType(), C: GV, IdxList: TwoZero32s)}));
1173 }
1174 // Create global arrays for the two emit calls.
1175 int CountersSize = CountersBySP.size();
1176 assert(CountersSize == (int)EmitFunctionCallArgsArray.size() &&
1177 "Mismatched array size!");
1178 assert(CountersSize == (int)EmitArcsCallArgsArray.size() &&
1179 "Mismatched array size!");
1180 auto *EmitFunctionCallArgsArrayTy =
1181 ArrayType::get(ElementType: EmitFunctionCallArgsTy, NumElements: CountersSize);
1182 auto *EmitFunctionCallArgsArrayGV = new GlobalVariable(
1183 *M, EmitFunctionCallArgsArrayTy, /*isConstant*/ true,
1184 GlobalValue::InternalLinkage,
1185 ConstantArray::get(T: EmitFunctionCallArgsArrayTy,
1186 V: EmitFunctionCallArgsArray),
1187 Twine("__llvm_internal_gcov_emit_function_args.") + Twine(i));
1188 auto *EmitArcsCallArgsArrayTy =
1189 ArrayType::get(ElementType: EmitArcsCallArgsTy, NumElements: CountersSize);
1190 EmitFunctionCallArgsArrayGV->setUnnamedAddr(
1191 GlobalValue::UnnamedAddr::Global);
1192 auto *EmitArcsCallArgsArrayGV = new GlobalVariable(
1193 *M, EmitArcsCallArgsArrayTy, /*isConstant*/ true,
1194 GlobalValue::InternalLinkage,
1195 ConstantArray::get(T: EmitArcsCallArgsArrayTy, V: EmitArcsCallArgsArray),
1196 Twine("__llvm_internal_gcov_emit_arcs_args.") + Twine(i));
1197 EmitArcsCallArgsArrayGV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
1198
1199 FileInfos.push_back(Elt: ConstantStruct::get(
1200 T: FileInfoTy,
1201 V: {StartFileCallArgs, Builder.getInt32(C: CountersSize),
1202 ConstantExpr::getInBoundsGetElementPtr(Ty: EmitFunctionCallArgsArrayTy,
1203 C: EmitFunctionCallArgsArrayGV,
1204 IdxList: TwoZero32s),
1205 ConstantExpr::getInBoundsGetElementPtr(
1206 Ty: EmitArcsCallArgsArrayTy, C: EmitArcsCallArgsArrayGV, IdxList: TwoZero32s)}));
1207 }
1208
1209 // If we didn't find anything to actually emit, bail on out.
1210 if (FileInfos.empty()) {
1211 Builder.CreateRetVoid();
1212 return WriteoutF;
1213 }
1214
1215 // To simplify code, we cap the number of file infos we write out to fit
1216 // easily in a 32-bit signed integer. This gives consistent behavior between
1217 // 32-bit and 64-bit systems without requiring (potentially very slow) 64-bit
1218 // operations on 32-bit systems. It also seems unreasonable to try to handle
1219 // more than 2 billion files.
1220 if ((int64_t)FileInfos.size() > (int64_t)INT_MAX)
1221 FileInfos.resize(INT_MAX);
1222
1223 // Create a global for the entire data structure so we can walk it more
1224 // easily.
1225 auto *FileInfoArrayTy = ArrayType::get(ElementType: FileInfoTy, NumElements: FileInfos.size());
1226 auto *FileInfoArrayGV = new GlobalVariable(
1227 *M, FileInfoArrayTy, /*isConstant*/ true, GlobalValue::InternalLinkage,
1228 ConstantArray::get(T: FileInfoArrayTy, V: FileInfos),
1229 "__llvm_internal_gcov_emit_file_info");
1230 FileInfoArrayGV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
1231
1232 // Create the CFG for walking this data structure.
1233 auto *FileLoopHeader =
1234 BasicBlock::Create(Context&: *Ctx, Name: "file.loop.header", Parent: WriteoutF);
1235 auto *CounterLoopHeader =
1236 BasicBlock::Create(Context&: *Ctx, Name: "counter.loop.header", Parent: WriteoutF);
1237 auto *FileLoopLatch = BasicBlock::Create(Context&: *Ctx, Name: "file.loop.latch", Parent: WriteoutF);
1238 auto *ExitBB = BasicBlock::Create(Context&: *Ctx, Name: "exit", Parent: WriteoutF);
1239
1240 // We always have at least one file, so just branch to the header.
1241 Builder.CreateBr(Dest: FileLoopHeader);
1242
1243 // The index into the files structure is our loop induction variable.
1244 Builder.SetInsertPoint(FileLoopHeader);
1245 PHINode *IV = Builder.CreatePHI(Ty: Builder.getInt32Ty(), /*NumReservedValues*/ 2,
1246 Name: "file_idx");
1247 IV->addIncoming(V: Builder.getInt32(C: 0), BB);
1248 auto *FileInfoPtr = Builder.CreateInBoundsGEP(
1249 Ty: FileInfoArrayTy, Ptr: FileInfoArrayGV, IdxList: {Builder.getInt32(C: 0), IV});
1250 auto *StartFileCallArgsPtr =
1251 Builder.CreateStructGEP(Ty: FileInfoTy, Ptr: FileInfoPtr, Idx: 0, Name: "start_file_args");
1252 auto *StartFileCall = Builder.CreateCall(
1253 Callee: StartFile,
1254 Args: {Builder.CreateLoad(Ty: StartFileCallArgsTy->getElementType(N: 0),
1255 Ptr: Builder.CreateStructGEP(Ty: StartFileCallArgsTy,
1256 Ptr: StartFileCallArgsPtr, Idx: 0),
1257 Name: "filename"),
1258 Builder.CreateLoad(Ty: StartFileCallArgsTy->getElementType(N: 1),
1259 Ptr: Builder.CreateStructGEP(Ty: StartFileCallArgsTy,
1260 Ptr: StartFileCallArgsPtr, Idx: 1),
1261 Name: "version"),
1262 Builder.CreateLoad(Ty: StartFileCallArgsTy->getElementType(N: 2),
1263 Ptr: Builder.CreateStructGEP(Ty: StartFileCallArgsTy,
1264 Ptr: StartFileCallArgsPtr, Idx: 2),
1265 Name: "stamp")});
1266 if (auto AK = TLI->getExtAttrForI32Param(Signed: false))
1267 StartFileCall->addParamAttr(ArgNo: 2, Kind: AK);
1268 auto *NumCounters = Builder.CreateLoad(
1269 Ty: FileInfoTy->getElementType(N: 1),
1270 Ptr: Builder.CreateStructGEP(Ty: FileInfoTy, Ptr: FileInfoPtr, Idx: 1), Name: "num_ctrs");
1271 auto *EmitFunctionCallArgsArray =
1272 Builder.CreateLoad(Ty: FileInfoTy->getElementType(N: 2),
1273 Ptr: Builder.CreateStructGEP(Ty: FileInfoTy, Ptr: FileInfoPtr, Idx: 2),
1274 Name: "emit_function_args");
1275 auto *EmitArcsCallArgsArray = Builder.CreateLoad(
1276 Ty: FileInfoTy->getElementType(N: 3),
1277 Ptr: Builder.CreateStructGEP(Ty: FileInfoTy, Ptr: FileInfoPtr, Idx: 3), Name: "emit_arcs_args");
1278 auto *EnterCounterLoopCond =
1279 Builder.CreateICmpSLT(LHS: Builder.getInt32(C: 0), RHS: NumCounters);
1280 Builder.CreateCondBr(Cond: EnterCounterLoopCond, True: CounterLoopHeader, False: FileLoopLatch);
1281
1282 Builder.SetInsertPoint(CounterLoopHeader);
1283 auto *JV = Builder.CreatePHI(Ty: Builder.getInt32Ty(), /*NumReservedValues*/ 2,
1284 Name: "ctr_idx");
1285 JV->addIncoming(V: Builder.getInt32(C: 0), BB: FileLoopHeader);
1286 auto *EmitFunctionCallArgsPtr = Builder.CreateInBoundsGEP(
1287 Ty: EmitFunctionCallArgsTy, Ptr: EmitFunctionCallArgsArray, IdxList: JV);
1288 auto *EmitFunctionCall = Builder.CreateCall(
1289 Callee: EmitFunction,
1290 Args: {Builder.CreateLoad(Ty: EmitFunctionCallArgsTy->getElementType(N: 0),
1291 Ptr: Builder.CreateStructGEP(Ty: EmitFunctionCallArgsTy,
1292 Ptr: EmitFunctionCallArgsPtr, Idx: 0),
1293 Name: "ident"),
1294 Builder.CreateLoad(Ty: EmitFunctionCallArgsTy->getElementType(N: 1),
1295 Ptr: Builder.CreateStructGEP(Ty: EmitFunctionCallArgsTy,
1296 Ptr: EmitFunctionCallArgsPtr, Idx: 1),
1297 Name: "func_checkssum"),
1298 Builder.CreateLoad(Ty: EmitFunctionCallArgsTy->getElementType(N: 2),
1299 Ptr: Builder.CreateStructGEP(Ty: EmitFunctionCallArgsTy,
1300 Ptr: EmitFunctionCallArgsPtr, Idx: 2),
1301 Name: "cfg_checksum")});
1302 if (auto AK = TLI->getExtAttrForI32Param(Signed: false)) {
1303 EmitFunctionCall->addParamAttr(ArgNo: 0, Kind: AK);
1304 EmitFunctionCall->addParamAttr(ArgNo: 1, Kind: AK);
1305 EmitFunctionCall->addParamAttr(ArgNo: 2, Kind: AK);
1306 }
1307 auto *EmitArcsCallArgsPtr =
1308 Builder.CreateInBoundsGEP(Ty: EmitArcsCallArgsTy, Ptr: EmitArcsCallArgsArray, IdxList: JV);
1309 auto *EmitArcsCall = Builder.CreateCall(
1310 Callee: EmitArcs,
1311 Args: {Builder.CreateLoad(
1312 Ty: EmitArcsCallArgsTy->getElementType(N: 0),
1313 Ptr: Builder.CreateStructGEP(Ty: EmitArcsCallArgsTy, Ptr: EmitArcsCallArgsPtr, Idx: 0),
1314 Name: "num_counters"),
1315 Builder.CreateLoad(
1316 Ty: EmitArcsCallArgsTy->getElementType(N: 1),
1317 Ptr: Builder.CreateStructGEP(Ty: EmitArcsCallArgsTy, Ptr: EmitArcsCallArgsPtr, Idx: 1),
1318 Name: "counters")});
1319 if (auto AK = TLI->getExtAttrForI32Param(Signed: false))
1320 EmitArcsCall->addParamAttr(ArgNo: 0, Kind: AK);
1321 auto *NextJV = Builder.CreateAdd(LHS: JV, RHS: Builder.getInt32(C: 1));
1322 auto *CounterLoopCond = Builder.CreateICmpSLT(LHS: NextJV, RHS: NumCounters);
1323 Builder.CreateCondBr(Cond: CounterLoopCond, True: CounterLoopHeader, False: FileLoopLatch);
1324 JV->addIncoming(V: NextJV, BB: CounterLoopHeader);
1325
1326 Builder.SetInsertPoint(FileLoopLatch);
1327 Builder.CreateCall(Callee: SummaryInfo, Args: {});
1328 Builder.CreateCall(Callee: EndFile, Args: {});
1329 auto *NextIV = Builder.CreateAdd(LHS: IV, RHS: Builder.getInt32(C: 1), Name: "next_file_idx");
1330 auto *FileLoopCond =
1331 Builder.CreateICmpSLT(LHS: NextIV, RHS: Builder.getInt32(C: FileInfos.size()));
1332 Builder.CreateCondBr(Cond: FileLoopCond, True: FileLoopHeader, False: ExitBB);
1333 IV->addIncoming(V: NextIV, BB: FileLoopLatch);
1334
1335 Builder.SetInsertPoint(ExitBB);
1336 Builder.CreateRetVoid();
1337
1338 return WriteoutF;
1339}
1340
1341Function *GCOVProfiler::insertReset(
1342 ArrayRef<std::pair<GlobalVariable *, MDNode *>> CountersBySP) {
1343 FunctionType *FTy = FunctionType::get(Result: Type::getVoidTy(C&: *Ctx), isVarArg: false);
1344 Function *ResetF = M->getFunction(Name: "__llvm_gcov_reset");
1345 if (!ResetF)
1346 ResetF = createInternalFunction(FTy, Name: "__llvm_gcov_reset", MangledType: "_ZTSFvvE");
1347 ResetF->addFnAttr(Kind: Attribute::NoInline);
1348
1349 BasicBlock *Entry = BasicBlock::Create(Context&: *Ctx, Name: "entry", Parent: ResetF);
1350 IRBuilder<> Builder(Entry);
1351 LLVMContext &C = Entry->getContext();
1352
1353 // Zero out the counters.
1354 for (const auto &I : CountersBySP) {
1355 GlobalVariable *GV = I.first;
1356 auto *GVTy = cast<ArrayType>(Val: GV->getValueType());
1357 Builder.CreateMemSet(Ptr: GV, Val: Constant::getNullValue(Ty: Type::getInt8Ty(C)),
1358 Size: GVTy->getNumElements() *
1359 GVTy->getElementType()->getScalarSizeInBits() / 8,
1360 Align: GV->getAlign());
1361 }
1362
1363 Type *RetTy = ResetF->getReturnType();
1364 if (RetTy->isVoidTy())
1365 Builder.CreateRetVoid();
1366 else if (RetTy->isIntegerTy())
1367 // Used if __llvm_gcov_reset was implicitly declared.
1368 Builder.CreateRet(V: ConstantInt::get(Ty: RetTy, V: 0));
1369 else
1370 report_fatal_error(reason: "invalid return type for __llvm_gcov_reset");
1371
1372 return ResetF;
1373}
1374