1//===-- ProfiledBinary.cpp - Binary decoder ---------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "ProfiledBinary.h"
10#include "ErrorHandling.h"
11#include "MissingFrameInferrer.h"
12#include "Options.h"
13#include "ProfileGenerator.h"
14#include "llvm/ADT/StringExtras.h"
15#include "llvm/BinaryFormat/Magic.h"
16#include "llvm/DebugInfo/PDB/IPDBSession.h"
17#include "llvm/DebugInfo/PDB/PDB.h"
18#include "llvm/DebugInfo/PDB/PDBSymbolFunc.h"
19#include "llvm/MC/TargetRegistry.h"
20#include "llvm/Object/COFF.h"
21#include "llvm/Support/CommandLine.h"
22#include "llvm/Support/Debug.h"
23#include "llvm/Support/Format.h"
24#include "llvm/TargetParser/Triple.h"
25#include <optional>
26
27#define DEBUG_TYPE "load-binary"
28
29namespace llvm {
30
31using namespace object;
32
33cl::opt<bool> ShowDisassemblyOnly("show-disassembly-only",
34 cl::desc("Print disassembled code."),
35 cl::cat(ProfGenCategory));
36
37cl::opt<bool> ShowSourceLocations("show-source-locations",
38 cl::desc("Print source locations."),
39 cl::cat(ProfGenCategory));
40
41cl::opt<bool> LoadFunctionFromSymbol(
42 "load-function-from-symbol", cl::init(Val: true),
43 cl::desc("Gather additional binary function info from symbols (e.g. "
44 "symtab) in case dwarf info is incomplete."),
45 cl::cat(ProfGenCategory));
46
47static cl::opt<bool>
48 ShowCanonicalFnName("show-canonical-fname",
49 cl::desc("Print canonical function name."),
50 cl::cat(ProfGenCategory));
51
52static cl::opt<bool> ShowPseudoProbe(
53 "show-pseudo-probe",
54 cl::desc("Print pseudo probe section and disassembled info."),
55 cl::cat(ProfGenCategory));
56
57static cl::opt<bool> UseDwarfCorrelation(
58 "use-dwarf-correlation",
59 cl::desc("Use dwarf for profile correlation even when binary contains "
60 "pseudo probe."),
61 cl::cat(ProfGenCategory));
62
63static cl::opt<std::string>
64 DWPPath("dwp", cl::init(Val: ""),
65 cl::desc("Path of .dwp file. When not specified, it will be "
66 "<binary>.dwp in the same directory as the main binary."),
67 cl::cat(ProfGenCategory));
68
69static cl::list<std::string> DisassembleFunctions(
70 "disassemble-functions", cl::CommaSeparated,
71 cl::desc("List of functions to print disassembly for. Accept mangled "
72 "names only. Only work with show-disassembly-only"),
73 cl::cat(ProfGenCategory));
74
75static cl::opt<bool>
76 KernelBinary("kernel",
77 cl::desc("Generate the profile for Linux kernel binary."),
78 cl::cat(ProfGenCategory));
79
80static cl::opt<bool>
81 WarnNotSymbolized("warn-not-symbolized", cl::init(Val: false),
82 cl::desc("Generate warnings for unsymbolized addresses"),
83 cl::cat(ProfGenCategory));
84
85namespace sampleprof {
86
87// Internal suffixes which are not reflected in the source code.
88static constexpr StringRef CanonicalSuffixes[] = {
89 // Internal suffixes from CoroSplit pass
90 ".cleanup", ".destroy", ".resume",
91 // Internal suffixes from Bolt
92 ".cold", ".warm",
93 // Compiler/LTO internal
94 ".llvm.", ".part.", ".isra.", ".constprop.", ".lto_priv."};
95static const StringRef CoroSuffixes[] = {".cleanup", ".destroy", ".resume"};
96
97static const Target *getTarget(const ObjectFile *Obj) {
98 Triple TheTriple = Obj->makeTriple();
99 std::string Error;
100 std::string ArchName;
101 const Target *TheTarget =
102 TargetRegistry::lookupTarget(ArchName, TheTriple, Error);
103 if (!TheTarget)
104 exitWithError(Message: Error, Whence: Obj->getFileName());
105 return TheTarget;
106}
107
108void BinarySizeContextTracker::addInstructionForContext(
109 const SampleContextFrameVector &Context, uint32_t InstrSize) {
110 ContextTrieNode *CurNode = &RootContext;
111 bool IsLeaf = true;
112 for (const auto &Callsite : reverse(C: Context)) {
113 FunctionId CallerName = Callsite.Func;
114 LineLocation CallsiteLoc = IsLeaf ? LineLocation(0, 0) : Callsite.Location;
115 CurNode = CurNode->getOrCreateChildContext(CallSite: CallsiteLoc, ChildName: CallerName);
116 IsLeaf = false;
117 }
118
119 CurNode->addFunctionSize(FSize: InstrSize);
120}
121
122uint32_t
123BinarySizeContextTracker::getFuncSizeForContext(const ContextTrieNode *Node) {
124 ContextTrieNode *CurrNode = &RootContext;
125 ContextTrieNode *PrevNode = nullptr;
126
127 std::optional<uint32_t> Size;
128
129 // Start from top-level context-less function, traverse down the reverse
130 // context trie to find the best/longest match for given context, then
131 // retrieve the size.
132 LineLocation CallSiteLoc(0, 0);
133 while (CurrNode && Node->getParentContext() != nullptr) {
134 PrevNode = CurrNode;
135 CurrNode = CurrNode->getChildContext(CallSite: CallSiteLoc, ChildName: Node->getFuncName());
136 if (CurrNode && CurrNode->getFunctionSize())
137 Size = *CurrNode->getFunctionSize();
138 CallSiteLoc = Node->getCallSiteLoc();
139 Node = Node->getParentContext();
140 }
141
142 // If we traversed all nodes along the path of the context and haven't
143 // found a size yet, pivot to look for size from sibling nodes, i.e size
144 // of inlinee under different context.
145 if (!Size) {
146 if (!CurrNode)
147 CurrNode = PrevNode;
148 while (!Size && CurrNode && !CurrNode->getAllChildContext().empty()) {
149 CurrNode = &CurrNode->getAllChildContext().begin()->second;
150 if (CurrNode->getFunctionSize())
151 Size = *CurrNode->getFunctionSize();
152 }
153 }
154
155 assert(Size && "We should at least find one context size.");
156 return *Size;
157}
158
159void BinarySizeContextTracker::trackInlineesOptimizedAway(
160 MCPseudoProbeDecoder &ProbeDecoder) {
161 ProbeFrameStack ProbeContext;
162 for (const auto &Child : ProbeDecoder.getDummyInlineRoot().getChildren())
163 trackInlineesOptimizedAway(ProbeDecoder, ProbeNode: Child, Context&: ProbeContext);
164}
165
166void BinarySizeContextTracker::trackInlineesOptimizedAway(
167 MCPseudoProbeDecoder &ProbeDecoder,
168 const MCDecodedPseudoProbeInlineTree &ProbeNode,
169 ProbeFrameStack &ProbeContext) {
170 StringRef FuncName =
171 ProbeDecoder.getFuncDescForGUID(GUID: ProbeNode.Guid)->FuncName;
172 ProbeContext.emplace_back(Args&: FuncName, Args: 0);
173
174 // This ProbeContext has a probe, so it has code before inlining and
175 // optimization. Make sure we mark its size as known.
176 if (!ProbeNode.getProbes().empty()) {
177 ContextTrieNode *SizeContext = &RootContext;
178 for (auto &ProbeFrame : reverse(C&: ProbeContext)) {
179 StringRef CallerName = ProbeFrame.first;
180 LineLocation CallsiteLoc(ProbeFrame.second, 0);
181 SizeContext = SizeContext->getOrCreateChildContext(
182 CallSite: CallsiteLoc, ChildName: FunctionId(CallerName));
183 }
184 // Add 0 size to make known.
185 SizeContext->addFunctionSize(FSize: 0);
186 }
187
188 // DFS down the probe inline tree
189 for (const auto &ChildNode : ProbeNode.getChildren()) {
190 InlineSite Location = ChildNode.getInlineSite();
191 ProbeContext.back().second = std::get<1>(t&: Location);
192 trackInlineesOptimizedAway(ProbeDecoder, ProbeNode: ChildNode, ProbeContext);
193 }
194
195 ProbeContext.pop_back();
196}
197
198ProfiledBinary::ProfiledBinary(const cl::LibraryOptionsContext &LibraryOptions,
199 const StringRef ExeBinPath,
200 const StringRef DebugBinPath)
201 : LibraryOptions(LibraryOptions), Path(ExeBinPath),
202 DebugBinaryPath(DebugBinPath), SymbolizerOpts(getSymbolizerOpts()),
203 ProEpilogTracker(this),
204 Symbolizer(std::make_unique<symbolize::LLVMSymbolizer>(args&: SymbolizerOpts)),
205 TrackFuncContextSize(EnableCSPreInliner && UseContextCostForPreInliner) {
206 // Point to executable binary if debug info binary is not specified.
207 SymbolizerPath = DebugBinPath.empty() ? ExeBinPath : DebugBinPath;
208 if (InferMissingFrames)
209 MissingContextInferrer = std::make_unique<MissingFrameInferrer>(args: this);
210}
211
212ProfiledBinary::~ProfiledBinary() = default;
213
214void ProfiledBinary::warnNoFuncEntry() {
215 uint64_t NoFuncEntryNum = 0;
216 for (auto &F : BinaryFunctions) {
217 if (F.second.Ranges.empty())
218 continue;
219 bool hasFuncEntry = false;
220 for (auto &R : F.second.Ranges) {
221 if (FuncRange *FR = findFuncRangeForStartAddr(Address: R.first)) {
222 if (FR->IsFuncEntry) {
223 hasFuncEntry = true;
224 break;
225 }
226 }
227 }
228
229 if (!hasFuncEntry) {
230 NoFuncEntryNum++;
231 if (ShowDetailedWarning)
232 WithColor::warning()
233 << "Failed to determine function entry for " << F.first()
234 << " due to inconsistent name from symbol table and dwarf info.\n";
235 }
236 }
237 emitWarningSummary(Num: NoFuncEntryNum, Total: BinaryFunctions.size(),
238 Msg: "of functions failed to determine function entry due to "
239 "inconsistent name from symbol table and dwarf info.");
240}
241
242void ProfiledBinary::load(StringRef TripleStr) {
243 // Attempt to open the binary.
244 OBinary = unwrapOrError(EO: createBinary(Path), Args&: Path);
245 Binary &ExeBinary = *OBinary.getBinary();
246
247 IsCOFF = isa<COFFObjectFile>(Val: &ExeBinary);
248 if (!isa<ELFObjectFileBase>(Val: &ExeBinary) && !IsCOFF)
249 exitWithError(Message: "not a valid ELF/COFF image", Whence: Path);
250
251 auto *Obj = cast<ObjectFile>(Val: &ExeBinary);
252 if (!TripleStr.empty())
253 TheTriple = Triple(TripleStr);
254 else
255 TheTriple = Obj->makeTriple();
256
257 LLVM_DEBUG(dbgs() << "Loading " << Path << "\n");
258
259 // Mark the binary as a kernel image;
260 IsKernel = KernelBinary;
261
262 // Find the preferred load address for text sections.
263 setPreferredTextSegmentAddresses(Obj);
264
265 // For shared libraries, read build ID to filter perfscript addresses
266 // in [buildid:]addr format. Main executables (including PIE) use empty
267 // FilterBuildID since their addresses have no buildid prefix.
268 // Both PIE executables and shared libraries are ET_DYN, but only PIE
269 // executables have a PT_INTERP program header.
270 file_magic Magic;
271 if (auto EC = identify_magic(path: Path, result&: Magic);
272 !EC && Magic == file_magic::elf_shared_object && !HasInterp) {
273 auto BID = object::getBuildID(Obj);
274 if (!BID.empty())
275 FilterBuildID = llvm::toHex(Input: BID, /*LowerCase=*/true);
276 }
277
278 // Load debug info of subprograms from DWARF section.
279 // If path of debug info binary is specified, use the debug info from it,
280 // otherwise use the debug info from the executable binary.
281 OwningBinary<Binary> DebugBinary;
282 ObjectFile *PseudoProbeObj = nullptr;
283 if (!DebugBinaryPath.empty()) {
284 DebugBinary = unwrapOrError(EO: createBinary(Path: DebugBinaryPath), Args&: DebugBinaryPath);
285 ObjectFile *DebugObj = cast<ObjectFile>(Val: DebugBinary.getBinary());
286 loadSymbolsFromDWARF(Obj&: *DebugObj);
287 if (checkPseudoProbe(Obj: DebugObj, ObjPath: DebugBinaryPath))
288 PseudoProbeObj = DebugObj;
289 } else {
290 loadSymbolsFromDWARF(Obj&: *Obj);
291 }
292
293 // Prefer loading pseudo probe from binary.
294 if (checkPseudoProbe(Obj, ObjPath: Path))
295 PseudoProbeObj = Obj;
296
297 DisassembleFunctionSet.insert_range(R&: DisassembleFunctions);
298
299 if (usePseudoProbes())
300 populateSymbolAddressList(O: Obj);
301
302 if (ShowDisassemblyOnly && PseudoProbeObj)
303 decodePseudoProbe(Obj: PseudoProbeObj);
304
305 if (LoadFunctionFromSymbol && usePseudoProbes())
306 loadSymbolsFromSymtab(O: Obj);
307
308 // Disassemble the text sections.
309 disassemble(O: Obj);
310
311 // Use function start and return address to infer prolog and epilog
312 ProEpilogTracker.inferPrologAddresses(FuncStartAddressMap&: StartAddrToFuncRangeMap);
313 ProEpilogTracker.inferEpilogAddresses(RetAddrs&: RetAddressSet);
314
315 warnNoFuncEntry();
316
317 // TODO: decode other sections.
318}
319
320bool ProfiledBinary::inlineContextEqual(uint64_t Address1, uint64_t Address2) {
321 const SampleContextFrameVector &Context1 =
322 getCachedFrameLocationStack(Address: Address1);
323 const SampleContextFrameVector &Context2 =
324 getCachedFrameLocationStack(Address: Address2);
325 if (Context1.size() != Context2.size())
326 return false;
327 if (Context1.empty())
328 return false;
329 // The leaf frame contains location within the leaf, and it
330 // needs to be remove that as it's not part of the calling context
331 return std::equal(first1: Context1.begin(), last1: Context1.begin() + Context1.size() - 1,
332 first2: Context2.begin(), last2: Context2.begin() + Context2.size() - 1);
333}
334
335SampleContextFrameVector
336ProfiledBinary::getExpandedContext(const SmallVectorImpl<uint64_t> &Stack,
337 bool &WasLeafInlined) {
338 SampleContextFrameVector ContextVec;
339 if (Stack.empty())
340 return ContextVec;
341 // Process from frame root to leaf
342 for (auto Address : Stack) {
343 const SampleContextFrameVector &ExpandedContext =
344 getCachedFrameLocationStack(Address);
345 // An instruction without a valid debug line will be ignored by sample
346 // processing
347 if (ExpandedContext.empty())
348 return SampleContextFrameVector();
349 // Set WasLeafInlined to the size of inlined frame count for the last
350 // address which is leaf
351 WasLeafInlined = (ExpandedContext.size() > 1);
352 ContextVec.append(RHS: ExpandedContext);
353 }
354
355 // Replace with decoded base discriminator
356 for (auto &Frame : ContextVec) {
357 Frame.Location.Discriminator = ProfileGeneratorBase::getBaseDiscriminator(
358 Discriminator: Frame.Location.Discriminator, UseFSD: UseFSDiscriminator);
359 }
360
361 assert(ContextVec.size() && "Context length should be at least 1");
362
363 // Compress the context string except for the leaf frame
364 auto LeafFrame = ContextVec.back();
365 LeafFrame.Location = LineLocation(0, 0);
366 ContextVec.pop_back();
367 CSProfileGenerator::compressRecursionContext(Context&: ContextVec);
368 CSProfileGenerator::trimContext(S&: ContextVec);
369 ContextVec.push_back(Elt: LeafFrame);
370 return ContextVec;
371}
372
373template <class ELFT>
374void ProfiledBinary::setPreferredTextSegmentAddresses(const ELFFile<ELFT> &Obj,
375 StringRef FileName) {
376 const auto &PhdrRange = unwrapOrError(Obj.program_headers(), FileName);
377 bool SeenFirstLoadableSegment = false;
378 for (const typename ELFT::Phdr &Phdr : PhdrRange) {
379 if (Phdr.p_type == ELF::PT_INTERP)
380 HasInterp = true;
381 if (Phdr.p_type == ELF::PT_LOAD) {
382 if (!SeenFirstLoadableSegment) {
383 // Derive the preferred address corresponding to file offset zero
384 // without assuming a page size.
385 FirstLoadableAddress = Phdr.p_vaddr - Phdr.p_offset;
386 SeenFirstLoadableSegment = true;
387 }
388 if (Phdr.p_flags & ELF::PF_X) {
389 PreferredTextSegmentAddresses.push_back(Phdr.p_vaddr);
390 TextSegmentOffsets.push_back(Phdr.p_offset);
391 } else {
392 PhdrInfo Info;
393 Info.FileOffset = Phdr.p_offset;
394 Info.FileSz = Phdr.p_filesz;
395 Info.VirtualAddr = Phdr.p_vaddr;
396 NonTextPhdrInfo.push_back(Elt: Info);
397 }
398 }
399 }
400
401 if (PreferredTextSegmentAddresses.empty())
402 exitWithError(Message: "no executable segment found", Whence: FileName);
403}
404
405uint64_t ProfiledBinary::CanonicalizeNonTextAddress(uint64_t Address) {
406 uint64_t FileOffset = 0;
407 auto MMapIter = NonTextMMapEvents.lower_bound(x: Address);
408 if (MMapIter == NonTextMMapEvents.end())
409 return Address; // No non-text mmap event found, return the address as is.
410
411 const auto &MMapEvent = MMapIter->second;
412
413 // If the address is within the non-text mmap event, calculate its file
414 // offset in the binary.
415 if (MMapEvent.Address <= Address &&
416 Address < MMapEvent.Address + MMapEvent.Size)
417 FileOffset = Address - MMapEvent.Address + MMapEvent.Offset;
418
419 // If the address is not within the non-text mmap event, return the address
420 // as is.
421 if (FileOffset == 0)
422 return Address;
423
424 for (const auto &PhdrInfo : NonTextPhdrInfo) {
425 // Find the program section that contains the file offset and map the
426 // file offset to the virtual address.
427 if (PhdrInfo.FileOffset <= FileOffset &&
428 FileOffset < PhdrInfo.FileOffset + PhdrInfo.FileSz)
429 return PhdrInfo.VirtualAddr + (FileOffset - PhdrInfo.FileOffset);
430 }
431
432 return Address;
433}
434
435void ProfiledBinary::setPreferredTextSegmentAddresses(const COFFObjectFile *Obj,
436 StringRef FileName) {
437 uint64_t ImageBase = Obj->getImageBase();
438 if (!ImageBase)
439 exitWithError(Message: "Not a COFF image", Whence: FileName);
440
441 PreferredTextSegmentAddresses.push_back(x: ImageBase);
442 FirstLoadableAddress = ImageBase;
443
444 for (SectionRef Section : Obj->sections()) {
445 const coff_section *Sec = Obj->getCOFFSection(Section);
446 if (Sec->Characteristics & COFF::IMAGE_SCN_CNT_CODE)
447 TextSegmentOffsets.push_back(x: Sec->VirtualAddress);
448 }
449}
450
451void ProfiledBinary::setPreferredTextSegmentAddresses(const ObjectFile *Obj) {
452 if (const auto *ELFObj = dyn_cast<ELF32LEObjectFile>(Val: Obj))
453 setPreferredTextSegmentAddresses(Obj: ELFObj->getELFFile(), FileName: Obj->getFileName());
454 else if (const auto *ELFObj = dyn_cast<ELF32BEObjectFile>(Val: Obj))
455 setPreferredTextSegmentAddresses(Obj: ELFObj->getELFFile(), FileName: Obj->getFileName());
456 else if (const auto *ELFObj = dyn_cast<ELF64LEObjectFile>(Val: Obj))
457 setPreferredTextSegmentAddresses(Obj: ELFObj->getELFFile(), FileName: Obj->getFileName());
458 else if (const auto *ELFObj = dyn_cast<ELF64BEObjectFile>(Val: Obj))
459 setPreferredTextSegmentAddresses(Obj: ELFObj->getELFFile(), FileName: Obj->getFileName());
460 else if (const auto *COFFObj = dyn_cast<COFFObjectFile>(Val: Obj))
461 setPreferredTextSegmentAddresses(Obj: COFFObj, FileName: Obj->getFileName());
462 else
463 llvm_unreachable("invalid object format");
464}
465
466bool ProfiledBinary::checkPseudoProbe(const ObjectFile *Obj,
467 StringRef ObjPath) {
468 if (UseDwarfCorrelation)
469 return false;
470
471 bool HasProbeDescSection = false;
472 bool HasPseudoProbeSection = false;
473
474 StringRef FileName = Obj->getFileName();
475 for (section_iterator SI = Obj->section_begin(), SE = Obj->section_end();
476 SI != SE; ++SI) {
477 const SectionRef &Section = *SI;
478 StringRef SectionName = unwrapOrError(EO: Section.getName(), Args&: FileName);
479 if (SectionName == ".pseudo_probe_desc") {
480 HasProbeDescSection = true;
481 } else if (SectionName == ".pseudo_probe") {
482 HasPseudoProbeSection = true;
483 }
484 }
485
486 if (HasProbeDescSection && HasPseudoProbeSection) {
487 PseudoProbeBinPath = ObjPath;
488 return true;
489 }
490
491 return false;
492}
493
494void ProfiledBinary::decodePseudoProbe(const ObjectFile *Obj) {
495 if (!usePseudoProbes())
496 return;
497
498 LLVM_DEBUG(dbgs() << "Decoding pseudo probe in " << Obj->getFileName()
499 << "\n");
500
501 MCPseudoProbeDecoder::Uint64Set GuidFilter;
502 MCPseudoProbeDecoder::Uint64Map FuncStartAddresses;
503 if (ShowDisassemblyOnly) {
504 if (DisassembleFunctionSet.empty()) {
505 FuncStartAddresses = SymbolStartAddrs;
506 } else {
507 for (auto &F : DisassembleFunctionSet) {
508 auto GUID = Function::getGUIDAssumingExternalLinkage(GlobalName: F.first());
509 if (auto StartAddr = SymbolStartAddrs.lookup(Val: GUID)) {
510 FuncStartAddresses[GUID] = StartAddr;
511 if (FuncRange *Range = findFuncRangeForStartAddr(Address: StartAddr))
512 GuidFilter.insert(
513 V: Function::getGUIDAssumingExternalLinkage(GlobalName: Range->getFuncName()));
514 else
515 GuidFilter.insert(V: GUID);
516 }
517 }
518 }
519 } else {
520 for (auto *F : ProfiledFunctions) {
521 GuidFilter.insert(V: Function::getGUIDAssumingExternalLinkage(GlobalName: F->FuncName));
522 // DWARF name might be broken when a DWARF32 .debug_str.dwo section
523 // execeeds 4GB. We expect symbol table to contain the correct function
524 // names which matches the pseudo probe. Adding back all the GUIDs if
525 // possible.
526 auto AltGUIDs = AlternativeFunctionGUIDs.equal_range(x: F);
527 for (const auto &[_, Func] : make_range(p: AltGUIDs))
528 GuidFilter.insert(V: Func);
529 for (auto &Range : F->Ranges) {
530 auto GUIDs = StartAddrToSymMap.equal_range(x: Range.first);
531 for (const auto &[StartAddr, Func] : make_range(p: GUIDs))
532 FuncStartAddresses[Func] = StartAddr;
533 }
534 }
535 }
536
537 StringRef FileName = Obj->getFileName();
538 for (section_iterator SI = Obj->section_begin(), SE = Obj->section_end();
539 SI != SE; ++SI) {
540 const SectionRef &Section = *SI;
541 StringRef SectionName = unwrapOrError(EO: Section.getName(), Args&: FileName);
542
543 if (SectionName == ".pseudo_probe_desc") {
544 StringRef Contents = unwrapOrError(EO: Section.getContents(), Args&: FileName);
545 if (!ProbeDecoder.buildGUID2FuncDescMap(
546 Start: reinterpret_cast<const uint8_t *>(Contents.data()),
547 Size: Contents.size(), /*IsMMapped=*/false, VerboseWarnings: ShowDetailedWarning))
548 exitWithError(
549 Message: "Pseudo Probe decoder fail in .pseudo_probe_desc section");
550 } else if (SectionName == ".pseudo_probe") {
551 StringRef Contents = unwrapOrError(EO: Section.getContents(), Args&: FileName);
552 if (!ProbeDecoder.buildAddress2ProbeMap(
553 Start: reinterpret_cast<const uint8_t *>(Contents.data()),
554 Size: Contents.size(), GuildFilter: GuidFilter, FuncStartAddrs: FuncStartAddresses))
555 exitWithError(Message: "Pseudo Probe decoder fail in .pseudo_probe section");
556 }
557 }
558
559 // Build TopLevelProbeFrameMap to track size for optimized inlinees when probe
560 // is available
561 if (TrackFuncContextSize) {
562 for (auto &Child : ProbeDecoder.getDummyInlineRoot().getChildren()) {
563 auto *Frame = &Child;
564 StringRef FuncName =
565 ProbeDecoder.getFuncDescForGUID(GUID: Frame->Guid)->FuncName;
566 TopLevelProbeFrameMap[FuncName] = Frame;
567 }
568 }
569
570 if (ShowPseudoProbe)
571 ProbeDecoder.printGUID2FuncDescMap(OS&: outs());
572}
573
574void ProfiledBinary::decodePseudoProbe() {
575 OwningBinary<Binary> OBinary =
576 unwrapOrError(EO: createBinary(Path: PseudoProbeBinPath), Args&: PseudoProbeBinPath);
577 auto *Obj = cast<ObjectFile>(Val: OBinary.getBinary());
578 decodePseudoProbe(Obj);
579}
580
581void ProfiledBinary::setIsFuncEntry(FuncRange *FuncRange,
582 StringRef RangeSymName) {
583 // Skip external function symbol.
584 if (!FuncRange)
585 return;
586
587 // Set IsFuncEntry to ture if there is only one range in the function or the
588 // RangeSymName from ELF is equal to its DWARF-based function name.
589 if (FuncRange->Func->Ranges.size() == 1 ||
590 (!FuncRange->IsFuncEntry &&
591 (FuncRange->getFuncName() == RangeSymName ||
592 FuncRange->Func->NameStatus != DwarfNameStatus::Matched)))
593 FuncRange->IsFuncEntry = true;
594}
595
596bool ProfiledBinary::dissassembleSymbol(std::size_t SI, ArrayRef<uint8_t> Bytes,
597 SectionSymbolsTy &Symbols,
598 const SectionRef &Section) {
599 std::size_t SE = Symbols.size();
600 uint64_t SectionAddress = Section.getAddress();
601 uint64_t SectSize = Section.getSize();
602 uint64_t StartAddress = Symbols[SI].Addr;
603 uint64_t NextStartAddress =
604 (SI + 1 < SE) ? Symbols[SI + 1].Addr : SectionAddress + SectSize;
605 FuncRange *FRange = findFuncRange(Address: StartAddress);
606 setIsFuncEntry(FuncRange: FRange, RangeSymName: FunctionSamples::getCanonicalFnName(FnName: Symbols[SI].Name));
607 StringRef SymbolName =
608 ShowCanonicalFnName
609 ? FunctionSamples::getCanonicalFnName(FnName: Symbols[SI].Name)
610 : Symbols[SI].Name;
611 bool ShowDisassembly =
612 ShowDisassemblyOnly && (DisassembleFunctionSet.empty() ||
613 DisassembleFunctionSet.count(Key: SymbolName));
614 if (ShowDisassembly)
615 outs() << '<' << SymbolName << ">:\n";
616
617 uint64_t Address = StartAddress;
618 // Size of a consecutive invalid instruction range starting from Address -1
619 // backwards.
620 uint64_t InvalidInstLength = 0;
621 while (Address < NextStartAddress) {
622 MCInst Inst;
623 uint64_t Size;
624 // Disassemble an instruction.
625 bool Disassembled = DisAsm->getInstruction(
626 Instr&: Inst, Size, Bytes: Bytes.slice(N: Address - SectionAddress), Address, CStream&: nulls());
627 if (Size == 0)
628 Size = 1;
629
630 if (ShowDisassembly) {
631 if (ShowPseudoProbe) {
632 ProbeDecoder.printProbeForAddress(OS&: outs(), Address);
633 }
634 outs() << format(Fmt: "%8" PRIx64 ":", Vals: Address);
635 size_t Start = outs().tell();
636 if (Disassembled)
637 IPrinter->printInst(MI: &Inst, Address: Address + Size, Annot: "", STI: *STI, OS&: outs());
638 else
639 outs() << "\t<unknown>";
640 if (ShowSourceLocations) {
641 unsigned Cur = outs().tell() - Start;
642 if (Cur < 40)
643 outs().indent(NumSpaces: 40 - Cur);
644 InstructionPointer IP(this, Address);
645 outs() << getReversedLocWithContext(
646 Context: symbolize(IP, UseCanonicalFnName: ShowCanonicalFnName, UseProbeDiscriminator: ShowPseudoProbe));
647 }
648 outs() << "\n";
649 }
650
651 if (Disassembled) {
652 const MCInstrDesc &MCDesc = MII->get(Opcode: Inst.getOpcode());
653
654 // Record instruction size.
655 AddressToInstSizeMap[Address] = Size;
656
657 // Populate address maps.
658 CodeAddressVec.push_back(x: Address);
659 if (MCDesc.isCall()) {
660 CallAddressSet.insert(V: Address);
661 UncondBranchAddrSet.insert(x: Address);
662 // Record the instruction after call as the branch target of a ret
663 BranchTargetAddressSet.insert(V: Address + Size);
664 } else if (MCDesc.isReturn()) {
665 RetAddressSet.insert(V: Address);
666 UncondBranchAddrSet.insert(x: Address);
667 } else if (MCDesc.isBranch()) {
668 if (MCDesc.isUnconditionalBranch())
669 UncondBranchAddrSet.insert(x: Address);
670 BranchAddressSet.insert(V: Address);
671 }
672
673 if (MCDesc.isIndirectBranch()) {
674 IndirectBranchAddressSet.insert(V: Address);
675 }
676
677 // Record branch target addresses for branches and calls.
678 if (MCDesc.isCall() || MCDesc.isBranch()) {
679 uint64_t Target = 0;
680 if (MIA->evaluateBranch(Inst, Addr: Address, Size, Target))
681 BranchTargetAddressSet.insert(V: Target);
682 }
683
684 // Record potential call targets for tail frame inference later-on.
685 if (InferMissingFrames && FRange) {
686 uint64_t Target = 0;
687 [[maybe_unused]] bool Err =
688 MIA->evaluateBranch(Inst, Addr: Address, Size, Target);
689 if (MCDesc.isCall()) {
690 // Indirect call targets are unknown at this point. Recording the
691 // unknown target (zero) for further LBR-based refinement.
692 MissingContextInferrer->CallEdges[Address].insert(V: Target);
693 } else if (MCDesc.isUnconditionalBranch()) {
694 assert(Err &&
695 "target should be known for unconditional direct branch");
696 // Any inter-function unconditional jump is considered tail call at
697 // this point. This is not 100% accurate and could further be
698 // optimized based on some source annotation.
699 FuncRange *ToFRange = findFuncRange(Address: Target);
700 if (ToFRange && ToFRange->Func != FRange->Func)
701 MissingContextInferrer->TailCallEdges[Address].insert(V: Target);
702 LLVM_DEBUG({
703 dbgs() << "Direct Tail call: " << format("%8" PRIx64 ":", Address);
704 IPrinter->printInst(&Inst, Address + Size, "", *STI.get(), dbgs());
705 dbgs() << "\n";
706 });
707 } else if (MCDesc.isIndirectBranch() && MCDesc.isBarrier()) {
708 // This is an indirect branch but not necessarily an indirect tail
709 // call. The isBarrier check is to filter out conditional branch.
710 // Similar with indirect call targets, recording the unknown target
711 // (zero) for further LBR-based refinement.
712 MissingContextInferrer->TailCallEdges[Address].insert(V: Target);
713 LLVM_DEBUG({
714 dbgs() << "Indirect Tail call: "
715 << format("%8" PRIx64 ":", Address);
716 IPrinter->printInst(&Inst, Address + Size, "", *STI.get(), dbgs());
717 dbgs() << "\n";
718 });
719 }
720 }
721
722 if (InvalidInstLength) {
723 AddrsWithInvalidInstruction.insert(
724 V: {Address - InvalidInstLength, Address - 1});
725 InvalidInstLength = 0;
726 }
727 } else {
728 InvalidInstLength += Size;
729 }
730
731 Address += Size;
732 }
733
734 if (InvalidInstLength)
735 AddrsWithInvalidInstruction.insert(
736 V: {Address - InvalidInstLength, Address - 1});
737
738 if (ShowDisassembly)
739 outs() << "\n";
740
741 return true;
742}
743
744void ProfiledBinary::setUpDisassembler(const ObjectFile *Obj) {
745 const Target *TheTarget = getTarget(Obj);
746 StringRef FileName = Obj->getFileName();
747
748 MRI.reset(p: TheTarget->createMCRegInfo(TT: TheTriple));
749 if (!MRI)
750 exitWithError(Message: "no register info for target " + TheTriple.str(), Whence: FileName);
751
752 MCTargetOptions MCOptions;
753 MCOptions.LibraryOptions = &LibraryOptions;
754 AsmInfo.reset(p: TheTarget->createMCAsmInfo(MRI: *MRI, TheTriple, Options: MCOptions));
755 if (!AsmInfo)
756 exitWithError(Message: "no assembly info for target " + TheTriple.str(), Whence: FileName);
757
758 Expected<SubtargetFeatures> Features = Obj->getFeatures();
759 if (!Features)
760 exitWithError(E: Features.takeError(), Whence: FileName);
761 // AArch64 object files do not generally carry complete ISA feature metadata,
762 // so the subtarget would default to the baseline (Armv8.0-A) feature set.
763 // That disassembler cannot decode feature-gated instructions (LSE atomics,
764 // RCPC loads, SVE, ...) that are pervasive in modern AArch64 binaries; they
765 // would be miscounted as "invalid instructions" and, worse, their addresses
766 // would be absent from the code/branch maps used for sample attribution.
767 // Enable all instructions so the disassembler recognizes whatever the
768 // compiler emitted, matching llvm-objdump's default for AArch64.
769 if (TheTriple.isAArch64())
770 Features->AddFeature(String: "+all");
771 STI.reset(
772 p: TheTarget->createMCSubtargetInfo(TheTriple, CPU: "", Features: Features->getString()));
773 if (!STI)
774 exitWithError(Message: "no subtarget info for target " + TheTriple.str(), Whence: FileName);
775
776 MII.reset(p: TheTarget->createMCInstrInfo());
777 if (!MII)
778 exitWithError(Message: "no instruction info for target " + TheTriple.str(),
779 Whence: FileName);
780
781 MCContext Ctx(TheTriple, *AsmInfo, *MRI, *STI);
782 std::unique_ptr<MCObjectFileInfo> MOFI(
783 TheTarget->createMCObjectFileInfo(Ctx, /*PIC=*/false));
784 Ctx.setObjectFileInfo(MOFI.get());
785 DisAsm.reset(p: TheTarget->createMCDisassembler(STI: *STI, Ctx));
786 if (!DisAsm)
787 exitWithError(Message: "no disassembler for target " + TheTriple.str(), Whence: FileName);
788
789 MIA.reset(p: TheTarget->createMCInstrAnalysis(Info: MII.get()));
790
791 int AsmPrinterVariant = AsmInfo->getAssemblerDialect();
792 IPrinter.reset(p: TheTarget->createMCInstPrinter(T: TheTriple, SyntaxVariant: AsmPrinterVariant,
793 MAI: *AsmInfo, MII: *MII, MRI: *MRI));
794 IPrinter->setPrintBranchImmAsAddress(true);
795}
796
797void ProfiledBinary::disassemble(const ObjectFile *Obj) {
798 // Set up disassembler and related components.
799 setUpDisassembler(Obj);
800
801 // Create a mapping from virtual address to symbol name. The symbols in text
802 // sections are the candidates to dissassemble.
803 std::map<SectionRef, SectionSymbolsTy> AllSymbols;
804 StringRef FileName = Obj->getFileName();
805 for (const SymbolRef &Symbol : Obj->symbols()) {
806 const uint64_t Addr = unwrapOrError(EO: Symbol.getAddress(), Args&: FileName);
807 const StringRef Name = unwrapOrError(EO: Symbol.getName(), Args&: FileName);
808 section_iterator SecI = unwrapOrError(EO: Symbol.getSection(), Args&: FileName);
809 if (SecI != Obj->section_end())
810 AllSymbols[*SecI].push_back(x: SymbolInfoTy(Addr, Name, ELF::STT_NOTYPE));
811 }
812
813 // Sort all the symbols. Use a stable sort to stabilize the output.
814 for (std::pair<const SectionRef, SectionSymbolsTy> &SecSyms : AllSymbols)
815 stable_sort(Range&: SecSyms.second);
816
817 assert((DisassembleFunctionSet.empty() || ShowDisassemblyOnly) &&
818 "Functions to disassemble should be only specified together with "
819 "--show-disassembly-only");
820
821 if (ShowDisassemblyOnly)
822 outs() << "\nDisassembly of " << FileName << ":\n";
823
824 // Dissassemble a text section.
825 for (section_iterator SI = Obj->section_begin(), SE = Obj->section_end();
826 SI != SE; ++SI) {
827 const SectionRef &Section = *SI;
828 if (!Section.isText())
829 continue;
830
831 uint64_t ImageLoadAddr = getPreferredBaseAddress();
832 uint64_t SectionAddress = Section.getAddress() - ImageLoadAddr;
833 uint64_t SectSize = Section.getSize();
834 if (!SectSize)
835 continue;
836
837 // Register the text section.
838 TextSections.insert(x: {SectionAddress, SectSize});
839
840 StringRef SectionName = unwrapOrError(EO: Section.getName(), Args&: FileName);
841
842 if (ShowDisassemblyOnly) {
843 outs() << "\nDisassembly of section " << SectionName;
844 outs() << " [" << format(Fmt: "0x%" PRIx64, Vals: Section.getAddress()) << ", "
845 << format(Fmt: "0x%" PRIx64, Vals: Section.getAddress() + SectSize)
846 << "]:\n\n";
847 }
848
849 if (isa<ELFObjectFileBase>(Val: Obj) && SectionName == ".plt")
850 continue;
851
852 // Get the section data.
853 ArrayRef<uint8_t> Bytes =
854 arrayRefFromStringRef(Input: unwrapOrError(EO: Section.getContents(), Args&: FileName));
855
856 // Get the list of all the symbols in this section.
857 SectionSymbolsTy &Symbols = AllSymbols[Section];
858
859 // Disassemble symbol by symbol.
860 for (std::size_t SI = 0, SE = Symbols.size(); SI != SE; ++SI) {
861 if (!dissassembleSymbol(SI, Bytes, Symbols, Section))
862 exitWithError(Message: "disassembling error", Whence: FileName);
863 }
864 }
865
866 if (!AddrsWithInvalidInstruction.empty()) {
867 if (ShowDetailedWarning) {
868 for (auto &Addr : AddrsWithInvalidInstruction) {
869 WithColor::warning()
870 << "Invalid instructions at " << format(Fmt: "%8" PRIx64, Vals: Addr.first)
871 << " - " << format(Fmt: "%8" PRIx64, Vals: Addr.second) << "\n";
872 }
873 }
874 WithColor::warning() << "Found " << AddrsWithInvalidInstruction.size()
875 << " invalid instructions\n";
876 AddrsWithInvalidInstruction.clear();
877 }
878
879 // Dissassemble rodata section to check if FS discriminator symbol exists.
880 checkUseFSDiscriminator(Obj, AllSymbols);
881}
882
883void ProfiledBinary::checkUseFSDiscriminator(
884 const ObjectFile *Obj, std::map<SectionRef, SectionSymbolsTy> &AllSymbols) {
885 const char *FSDiscriminatorVar = "__llvm_fs_discriminator__";
886 for (section_iterator SI = Obj->section_begin(), SE = Obj->section_end();
887 SI != SE; ++SI) {
888 const SectionRef &Section = *SI;
889 if (!Section.isData() || Section.getSize() == 0)
890 continue;
891 SectionSymbolsTy &Symbols = AllSymbols[Section];
892
893 for (std::size_t SI = 0, SE = Symbols.size(); SI != SE; ++SI) {
894 if (Symbols[SI].Name == FSDiscriminatorVar) {
895 UseFSDiscriminator = true;
896 return;
897 }
898 }
899 }
900}
901
902void ProfiledBinary::populateSymbolAddressList(const ObjectFile *Obj) {
903 // Create a mapping from virtual address to symbol GUID and the other way
904 // around.
905 StringRef FileName = Obj->getFileName();
906 for (const SymbolRef &Symbol : Obj->symbols()) {
907 const uint64_t Addr = unwrapOrError(EO: Symbol.getAddress(), Args&: FileName);
908 const StringRef Name = unwrapOrError(EO: Symbol.getName(), Args&: FileName);
909 uint64_t GUID = Function::getGUIDAssumingExternalLinkage(GlobalName: Name);
910 SymbolStartAddrs[GUID] = Addr;
911 StartAddrToSymMap.emplace(args: Addr, args&: GUID);
912 }
913}
914
915void ProfiledBinary::loadSymbolsFromSymtab(const ObjectFile *Obj) {
916 // Load binary functions from symbol table when Debug info is incomplete.
917 StringRef FileName = Obj->getFileName();
918
919 // COFF symtab does not have size field. Try to load size from PDB instead.
920 std::unique_ptr<pdb::IPDBSession> PDBSession;
921 if (auto *COFFObj = dyn_cast<COFFObjectFile>(Val: Obj)) {
922 if (auto E = pdb::loadDataForEXE(Type: pdb::PDB_ReaderType::Native, Path: FileName,
923 Session&: PDBSession)) {
924 StringRef PdbPath;
925 const codeview::DebugInfo *PdbInfo;
926 if (auto Err = COFFObj->getDebugPDBInfo(Info&: PdbInfo, PDBFileName&: PdbPath))
927 consumeError(Err: std::move(Err));
928
929 auto Style = PdbPath.starts_with(Prefix: "/") ? sys::path::Style::posix
930 : sys::path::Style::windows;
931 WithColor::warning() << "Cannot load PDB file "
932 << sys::path::filename(path: PdbPath, style: Style) << " for "
933 << FileName << ": " << E << "\n";
934 consumeError(Err: std::move(E));
935 } else {
936 PDBSession->setLoadAddress(FirstLoadableAddress);
937 }
938 }
939
940 for (const SymbolRef &Symbol : Obj->symbols()) {
941 const SymbolRef::Type Type = unwrapOrError(EO: Symbol.getType(), Args&: FileName);
942 const uint64_t StartAddr = unwrapOrError(EO: Symbol.getAddress(), Args&: FileName);
943 const StringRef Name = unwrapOrError(EO: Symbol.getName(), Args&: FileName);
944 uint64_t Size = 0;
945 if (isa<ELFObjectFileBase>(Val: Obj)) {
946 ELFSymbolRef ElfSymbol(Symbol);
947 Size = ElfSymbol.getSize();
948 } else if (PDBSession) {
949 if (std::unique_ptr<pdb::PDBSymbol> Sym = PDBSession->findSymbolByAddress(
950 Address: StartAddr, Type: pdb::PDB_SymType::Function)) {
951 auto FuncSym = cast<pdb::PDBSymbolFunc>(Val: std::move(Sym));
952 if (StartAddr == FuncSym->getVirtualAddress())
953 Size = FuncSym->getLength();
954 }
955 }
956
957 if (Size == 0 || Type != SymbolRef::ST_Function)
958 continue;
959
960 const uint64_t EndAddr = StartAddr + Size;
961 const StringRef SymName =
962 FunctionSamples::getCanonicalFnName(FnName: Name, Suffixes: CanonicalSuffixes);
963 assert(StartAddr < EndAddr && StartAddr >= getPreferredBaseAddress() &&
964 "Function range is invalid.");
965
966 auto Range = findFuncRange(Address: StartAddr);
967 if (!Range) {
968 assert(findFuncRange(EndAddr - 1) == nullptr &&
969 "Function range overlaps with existing functions.");
970 // Function from symbol table not found previously in DWARF, store ranges.
971 auto Ret = BinaryFunctions.try_emplace(Key: SymName);
972 auto &Func = Ret.first->second;
973 if (Ret.second) {
974 Func.FuncName = Ret.first->first();
975 HashBinaryFunctions[Function::getGUIDAssumingExternalLinkage(GlobalName: SymName)] =
976 &Func;
977 }
978
979 Func.NameStatus = DwarfNameStatus::Missing;
980 Func.Ranges.emplace_back(args: StartAddr, args: EndAddr);
981
982 auto R = StartAddrToFuncRangeMap.emplace(args: StartAddr, args: FuncRange());
983 FuncRange &FRange = R.first->second;
984
985 FRange.Func = &Func;
986 FRange.StartAddress = StartAddr;
987 FRange.EndAddress = EndAddr;
988
989 } else if (SymName != Range->getFuncName()) {
990 // Function range already found from DWARF or symtab, but the symbol name
991 // from symbol table is inconsistent with the existing name associated
992 // with the range. Log this discrepancy and the alternative function GUID.
993 if (ShowDetailedWarning)
994 WithColor::warning()
995 << "Conflicting name for symbol " << Name << " with range ("
996 << format(Fmt: "%8" PRIx64, Vals: StartAddr) << ", "
997 << format(Fmt: "%8" PRIx64, Vals: EndAddr) << ")"
998 << ", but the existing symbol " << Range->getFuncName()
999 << " indicates an overlapping range ("
1000 << format(Fmt: "%8" PRIx64, Vals: Range->StartAddress) << ", "
1001 << format(Fmt: "%8" PRIx64, Vals: Range->EndAddress) << ")\n";
1002
1003 assert(StartAddr == Range->StartAddress && EndAddr == Range->EndAddress &&
1004 "Mismatched function range");
1005
1006 Range->Func->NameStatus = DwarfNameStatus::Mismatch;
1007 AlternativeFunctionGUIDs.emplace(
1008 args&: Range->Func, args: Function::getGUIDAssumingExternalLinkage(GlobalName: SymName));
1009
1010 } else if (StartAddr != Range->StartAddress &&
1011 EndAddr != Range->EndAddress) {
1012 // Function already found in DWARF or symtab, but the address range from
1013 // symbol table conflicts/overlaps with the existing one.
1014 WithColor::warning() << "Conflicting range for symbol " << Name
1015 << " with range (" << format(Fmt: "%8" PRIx64, Vals: StartAddr)
1016 << ", " << format(Fmt: "%8" PRIx64, Vals: EndAddr) << ")"
1017 << ", but the existing symbol "
1018 << Range->getFuncName()
1019 << " indicates another range ("
1020 << format(Fmt: "%8" PRIx64, Vals: Range->StartAddress) << ", "
1021 << format(Fmt: "%8" PRIx64, Vals: Range->EndAddress) << ")\n";
1022 }
1023 }
1024}
1025
1026void ProfiledBinary::loadSymbolsFromDWARFUnit(DWARFUnit &CompilationUnit) {
1027 for (const auto &DieInfo : CompilationUnit.dies()) {
1028 llvm::DWARFDie Die(&CompilationUnit, &DieInfo);
1029
1030 if (!Die.isSubprogramDIE())
1031 continue;
1032 auto Name = Die.getName(Kind: llvm::DINameKind::LinkageName);
1033 if (!Name)
1034 Name = Die.getName(Kind: llvm::DINameKind::ShortName);
1035 if (!Name)
1036 continue;
1037
1038 auto CanonName = FunctionSamples::getCanonicalCoroFnName(FnName: Name);
1039 auto RangesOrError = Die.getAddressRanges();
1040 if (!RangesOrError)
1041 continue;
1042 const DWARFAddressRangesVector &Ranges = RangesOrError.get();
1043
1044 if (Ranges.empty())
1045 continue;
1046
1047 // Different DWARF symbols can have same function name, search or create
1048 // BinaryFunction indexed by the name.
1049 auto Ret = BinaryFunctions.try_emplace(Key: CanonName);
1050 auto &Func = Ret.first->second;
1051 if (Ret.second)
1052 Func.FuncName = Ret.first->first();
1053
1054 for (const auto &Range : Ranges) {
1055 uint64_t StartAddress = Range.LowPC;
1056 uint64_t EndAddress = Range.HighPC;
1057
1058 if (EndAddress <= StartAddress ||
1059 StartAddress < getPreferredBaseAddress())
1060 continue;
1061
1062 // We may want to know all ranges for one function. Here group the
1063 // ranges and store them into BinaryFunction.
1064 Func.Ranges.emplace_back(args&: StartAddress, args&: EndAddress);
1065
1066 auto R = StartAddrToFuncRangeMap.emplace(args&: StartAddress, args: FuncRange());
1067 if (R.second) {
1068 FuncRange &FRange = R.first->second;
1069 FRange.Func = &Func;
1070 FRange.StartAddress = StartAddress;
1071 FRange.EndAddress = EndAddress;
1072 } else {
1073 AddrsWithMultipleSymbols.insert(V: StartAddress);
1074 if (ShowDetailedWarning)
1075 WithColor::warning()
1076 << "Duplicated symbol start address at "
1077 << format(Fmt: "%8" PRIx64, Vals: StartAddress) << " "
1078 << R.first->second.getFuncName() << " and " << Name << "\n";
1079 }
1080 }
1081 }
1082}
1083
1084void ProfiledBinary::loadSymbolsFromDWARF(ObjectFile &Obj) {
1085 auto DebugContext = llvm::DWARFContext::create(
1086 Obj, RelocAction: DWARFContext::ProcessDebugRelocations::Process, L: nullptr, DWPName: DWPPath);
1087 if (!DebugContext)
1088 exitWithError(Message: "Error creating the debug info context", Whence: Path);
1089
1090 for (const auto &CompilationUnit : DebugContext->compile_units())
1091 loadSymbolsFromDWARFUnit(CompilationUnit&: *CompilationUnit);
1092
1093 // Handles DWO sections that can either be in .o, .dwo or .dwp files.
1094 uint32_t NumOfDWOMissing = 0;
1095 for (const auto &CompilationUnit : DebugContext->compile_units()) {
1096 DWARFUnit *const DwarfUnit = CompilationUnit.get();
1097 if (DwarfUnit->getDWOId()) {
1098 DWARFUnit *DWOCU = DwarfUnit->getNonSkeletonUnitDIE(ExtractUnitDIEOnly: false).getDwarfUnit();
1099 if (!DWOCU->isDWOUnit()) {
1100 NumOfDWOMissing++;
1101 if (ShowDetailedWarning) {
1102 std::string DWOName = dwarf::toString(
1103 V: DwarfUnit->getUnitDIE().find(
1104 Attrs: {dwarf::DW_AT_dwo_name, dwarf::DW_AT_GNU_dwo_name}),
1105 Default: "");
1106 WithColor::warning() << "DWO debug information for " << DWOName
1107 << " was not loaded.\n";
1108 }
1109 continue;
1110 }
1111 loadSymbolsFromDWARFUnit(CompilationUnit&: *DWOCU);
1112 }
1113 }
1114
1115 if (NumOfDWOMissing)
1116 WithColor::warning()
1117 << " DWO debug information was not loaded for " << NumOfDWOMissing
1118 << " modules. Please check the .o, .dwo or .dwp path.\n";
1119 if (BinaryFunctions.empty())
1120 WithColor::warning() << "Loading of DWARF info completed, but no binary "
1121 "functions have been retrieved.\n";
1122 // Populate the hash binary function map for MD5 function name lookup. This
1123 // is done after BinaryFunctions are finalized.
1124 for (auto &BinaryFunction : BinaryFunctions) {
1125 HashBinaryFunctions[MD5Hash(Str: BinaryFunction.first())] =
1126 &BinaryFunction.second;
1127 }
1128
1129 if (!AddrsWithMultipleSymbols.empty()) {
1130 WithColor::warning() << "Found " << AddrsWithMultipleSymbols.size()
1131 << " start addresses with multiple symbols\n";
1132 AddrsWithMultipleSymbols.clear();
1133 }
1134}
1135
1136void ProfiledBinary::populateSymbolListFromDWARF(
1137 ProfileSymbolList &SymbolList) {
1138 for (auto &I : StartAddrToFuncRangeMap)
1139 SymbolList.add(Name: I.second.getFuncName());
1140}
1141
1142symbolize::LLVMSymbolizer::Options ProfiledBinary::getSymbolizerOpts() const {
1143 symbolize::LLVMSymbolizer::Options SymbolizerOpts;
1144 SymbolizerOpts.PrintFunctions =
1145 DILineInfoSpecifier::FunctionNameKind::LinkageName;
1146 SymbolizerOpts.Demangle = false;
1147 SymbolizerOpts.DefaultArch = TheTriple.getArchName().str();
1148 SymbolizerOpts.UseSymbolTable = false;
1149 SymbolizerOpts.RelativeAddresses = false;
1150 SymbolizerOpts.DWPName = DWPPath;
1151 return SymbolizerOpts;
1152}
1153
1154SampleContextFrameVector ProfiledBinary::symbolize(const InstructionPointer &IP,
1155 bool UseCanonicalFnName,
1156 bool UseProbeDiscriminator) {
1157 assert(this == IP.Binary &&
1158 "Binary should only symbolize its own instruction");
1159 DIInliningInfo InlineStack =
1160 unwrapOrError(EO: Symbolizer->symbolizeInlinedCode(
1161 ModuleName: SymbolizerPath.str(), ModuleOffset: getSectionedAddress(Address: IP.Address)),
1162 Args&: SymbolizerPath);
1163
1164 SampleContextFrameVector CallStack;
1165 for (int32_t I = InlineStack.getNumberOfFrames() - 1; I >= 0; I--) {
1166 const auto &CallerFrame = InlineStack.getFrame(Index: I);
1167 if (CallerFrame.FunctionName.empty() ||
1168 (CallerFrame.FunctionName == "<invalid>"))
1169 break;
1170
1171 StringRef FunctionName(CallerFrame.FunctionName);
1172 if (UseCanonicalFnName)
1173 FunctionName =
1174 FunctionSamples::getCanonicalFnName(FnName: FunctionName, Suffixes: CanonicalSuffixes);
1175 else
1176 FunctionName =
1177 FunctionSamples::getCanonicalFnName(FnName: FunctionName, Suffixes: CoroSuffixes);
1178
1179 uint32_t Discriminator = CallerFrame.Discriminator;
1180 uint32_t LineOffset = (CallerFrame.Line - CallerFrame.StartLine) & 0xffff;
1181 if (UseProbeDiscriminator) {
1182 LineOffset =
1183 PseudoProbeDwarfDiscriminator::extractProbeIndex(Value: Discriminator);
1184 Discriminator = 0;
1185 }
1186
1187 LineLocation Line(LineOffset, Discriminator);
1188 auto It = NameStrings.insert(key: FunctionName);
1189 CallStack.emplace_back(Args: FunctionId(It.first->getKey()), Args&: Line);
1190 }
1191
1192 if (WarnNotSymbolized && CallStack.empty()) {
1193 uint64_t VAddr = IP.Address + BaseAddress - getPreferredBaseAddress();
1194 if (isVaddrMMapped(VAddr))
1195 WithColor::warning() << "Failed to symbolize address "
1196 << format(Fmt: "%8" PRIx64, Vals: IP.Address)
1197 << " (vaddr=" << format(Fmt: "%8" PRIx64, Vals: VAddr) << ")\n";
1198 }
1199
1200 return CallStack;
1201}
1202
1203StringRef ProfiledBinary::symbolizeDataAddress(uint64_t Address) {
1204 DIGlobal DataDIGlobal =
1205 unwrapOrError(EO: Symbolizer->symbolizeData(ModuleName: SymbolizerPath.str(),
1206 ModuleOffset: getSectionedAddress(Address)),
1207 Args&: SymbolizerPath);
1208 return NameStrings.insert(key: DataDIGlobal.Name).first->getKey();
1209}
1210
1211void ProfiledBinary::computeInlinedContextSizeForRange(uint64_t RangeBegin,
1212 uint64_t RangeEnd) {
1213 InstructionPointer IP(this, RangeBegin, true);
1214
1215 if (IP.Address != RangeBegin)
1216 WithColor::warning() << "Invalid start instruction at "
1217 << format(Fmt: "%8" PRIx64, Vals: RangeBegin) << "\n";
1218
1219 if (IP.Address >= RangeEnd)
1220 return;
1221
1222 do {
1223 const SampleContextFrameVector SymbolizedCallStack =
1224 getFrameLocationStack(Address: IP.Address, UseProbeDiscriminator: usePseudoProbes());
1225 uint64_t Size = AddressToInstSizeMap[IP.Address];
1226 // Record instruction size for the corresponding context
1227 FuncSizeTracker.addInstructionForContext(Context: SymbolizedCallStack, InstrSize: Size);
1228
1229 } while (IP.advance() && IP.Address < RangeEnd);
1230}
1231
1232void ProfiledBinary::computeInlinedContextSizeForFunc(
1233 const BinaryFunction *Func) {
1234 // Note that a function can be spilt into multiple ranges, so compute for all
1235 // ranges of the function.
1236 for (const auto &Range : Func->Ranges)
1237 computeInlinedContextSizeForRange(RangeBegin: Range.first, RangeEnd: Range.second);
1238
1239 // Track optimized-away inlinee for probed binary. A function inlined and then
1240 // optimized away should still have their probes left over in places.
1241 if (usePseudoProbes()) {
1242 auto I = TopLevelProbeFrameMap.find(Key: Func->FuncName);
1243 if (I != TopLevelProbeFrameMap.end()) {
1244 BinarySizeContextTracker::ProbeFrameStack ProbeContext;
1245 FuncSizeTracker.trackInlineesOptimizedAway(ProbeDecoder, ProbeNode: *I->second,
1246 ProbeContext);
1247 }
1248 }
1249}
1250
1251void ProfiledBinary::loadSymbolsFromPseudoProbe() {
1252 if (!usePseudoProbes())
1253 return;
1254
1255 const AddressProbesMap &Address2ProbesMap = getAddress2ProbesMap();
1256 for (auto *Func : ProfiledFunctions) {
1257 if (Func->NameStatus != DwarfNameStatus::Mismatch)
1258 continue;
1259 for (auto &[StartAddr, EndAddr] : Func->Ranges) {
1260 auto Range = findFuncRangeForStartAddr(Address: StartAddr);
1261 if (!Range->IsFuncEntry)
1262 continue;
1263 const auto &Probe = Address2ProbesMap.find(From: StartAddr, To: EndAddr);
1264 if (Probe.begin() != Probe.end()) {
1265 const MCDecodedPseudoProbeInlineTree *InlineTreeNode =
1266 Probe.begin()->get().getInlineTreeNode();
1267 while (!InlineTreeNode->isTopLevelFunc())
1268 InlineTreeNode = static_cast<MCDecodedPseudoProbeInlineTree *>(
1269 InlineTreeNode->Parent);
1270
1271 assert(llvm::any_of(InlineTreeNode->getProbes(),
1272 [Start = StartAddr, End = EndAddr](const auto &P) {
1273 return P.getAddress() >= Start &&
1274 P.getAddress() < End;
1275 }) &&
1276 "Top level pseudo probe does not match function range");
1277
1278 const auto *ProbeDesc = getFuncDescForGUID(GUID: InlineTreeNode->Guid);
1279 auto Ret = PseudoProbeNames.try_emplace(Key: Func, Args: ProbeDesc->FuncName);
1280 if (!Ret.second && Ret.first->second != ProbeDesc->FuncName &&
1281 ShowDetailedWarning)
1282 WithColor::warning()
1283 << "Mismatched pseudo probe names in function " << Func->FuncName
1284 << " at range: (" << format(Fmt: "%8" PRIx64, Vals: StartAddr) << ", "
1285 << format(Fmt: "%8" PRIx64, Vals: EndAddr) << "). "
1286 << "The previously found pseudo probe name is "
1287 << Ret.first->second << " but it conflicts with name "
1288 << ProbeDesc->FuncName
1289 << " This likely indicates a DWARF error that produces "
1290 "conflicting symbols at the same starting address.\n";
1291 }
1292 }
1293 }
1294}
1295
1296StringRef ProfiledBinary::findPseudoProbeName(const BinaryFunction *Func) {
1297 auto ProbeName = PseudoProbeNames.find(Val: Func);
1298 if (ProbeName == PseudoProbeNames.end())
1299 return StringRef();
1300 return ProbeName->second;
1301}
1302
1303void ProfiledBinary::inferMissingFrames(
1304 const SmallVectorImpl<uint64_t> &Context,
1305 SmallVectorImpl<uint64_t> &NewContext) {
1306 MissingContextInferrer->inferMissingFrames(Context, NewContext);
1307}
1308
1309InstructionPointer::InstructionPointer(const ProfiledBinary *Binary,
1310 uint64_t Address, bool RoundToNext)
1311 : Binary(Binary), Address(Address) {
1312 Index = Binary->getIndexForAddr(Address);
1313 if (RoundToNext) {
1314 // we might get address which is not the code
1315 // it should round to the next valid address
1316 if (Index >= Binary->getCodeAddrVecSize())
1317 this->Address = UINT64_MAX;
1318 else
1319 this->Address = Binary->getAddressforIndex(Index);
1320 }
1321}
1322
1323bool InstructionPointer::advance() {
1324 Index++;
1325 if (Index >= Binary->getCodeAddrVecSize()) {
1326 Address = UINT64_MAX;
1327 return false;
1328 }
1329 Address = Binary->getAddressforIndex(Index);
1330 return true;
1331}
1332
1333bool InstructionPointer::backward() {
1334 if (Index == 0) {
1335 Address = 0;
1336 return false;
1337 }
1338 Index--;
1339 Address = Binary->getAddressforIndex(Index);
1340 return true;
1341}
1342
1343void InstructionPointer::update(uint64_t Addr) {
1344 Address = Addr;
1345 Index = Binary->getIndexForAddr(Address);
1346}
1347
1348} // end namespace sampleprof
1349} // end namespace llvm
1350