1//===-- ProfiledBinary.cpp - Binary decoder ---------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "ProfiledBinary.h"
10#include "ErrorHandling.h"
11#include "MissingFrameInferrer.h"
12#include "Options.h"
13#include "ProfileGenerator.h"
14#include "llvm/ADT/StringExtras.h"
15#include "llvm/BinaryFormat/Magic.h"
16#include "llvm/DebugInfo/PDB/IPDBSession.h"
17#include "llvm/DebugInfo/PDB/PDB.h"
18#include "llvm/DebugInfo/PDB/PDBSymbolFunc.h"
19#include "llvm/DebugInfo/Symbolize/SymbolizableModule.h"
20#include "llvm/Demangle/Demangle.h"
21#include "llvm/IR/DebugInfoMetadata.h"
22#include "llvm/MC/TargetRegistry.h"
23#include "llvm/Object/COFF.h"
24#include "llvm/Support/CommandLine.h"
25#include "llvm/Support/Debug.h"
26#include "llvm/Support/Format.h"
27#include "llvm/Support/TargetSelect.h"
28#include "llvm/TargetParser/Triple.h"
29#include <optional>
30
31#define DEBUG_TYPE "load-binary"
32
33namespace llvm {
34
35using namespace object;
36
37cl::opt<bool> ShowDisassemblyOnly("show-disassembly-only",
38 cl::desc("Print disassembled code."),
39 cl::cat(ProfGenCategory));
40
41cl::opt<bool> ShowSourceLocations("show-source-locations",
42 cl::desc("Print source locations."),
43 cl::cat(ProfGenCategory));
44
45cl::opt<bool> LoadFunctionFromSymbol(
46 "load-function-from-symbol", cl::init(Val: true),
47 cl::desc("Gather additional binary function info from symbols (e.g. "
48 "symtab) in case dwarf info is incomplete."),
49 cl::cat(ProfGenCategory));
50
51static cl::opt<bool>
52 ShowCanonicalFnName("show-canonical-fname",
53 cl::desc("Print canonical function name."),
54 cl::cat(ProfGenCategory));
55
56static cl::opt<bool> ShowPseudoProbe(
57 "show-pseudo-probe",
58 cl::desc("Print pseudo probe section and disassembled info."),
59 cl::cat(ProfGenCategory));
60
61static cl::opt<bool> UseDwarfCorrelation(
62 "use-dwarf-correlation",
63 cl::desc("Use dwarf for profile correlation even when binary contains "
64 "pseudo probe."),
65 cl::cat(ProfGenCategory));
66
67static cl::opt<std::string>
68 DWPPath("dwp", cl::init(Val: ""),
69 cl::desc("Path of .dwp file. When not specified, it will be "
70 "<binary>.dwp in the same directory as the main binary."),
71 cl::cat(ProfGenCategory));
72
73static cl::list<std::string> DisassembleFunctions(
74 "disassemble-functions", cl::CommaSeparated,
75 cl::desc("List of functions to print disassembly for. Accept demangled "
76 "names only. Only work with show-disassembly-only"),
77 cl::cat(ProfGenCategory));
78
79static cl::opt<bool>
80 KernelBinary("kernel",
81 cl::desc("Generate the profile for Linux kernel binary."),
82 cl::cat(ProfGenCategory));
83
84namespace sampleprof {
85
86static const Target *getTarget(const ObjectFile *Obj) {
87 Triple TheTriple = Obj->makeTriple();
88 std::string Error;
89 std::string ArchName;
90 const Target *TheTarget =
91 TargetRegistry::lookupTarget(ArchName, TheTriple, Error);
92 if (!TheTarget)
93 exitWithError(Message: Error, Whence: Obj->getFileName());
94 return TheTarget;
95}
96
97void BinarySizeContextTracker::addInstructionForContext(
98 const SampleContextFrameVector &Context, uint32_t InstrSize) {
99 ContextTrieNode *CurNode = &RootContext;
100 bool IsLeaf = true;
101 for (const auto &Callsite : reverse(C: Context)) {
102 FunctionId CallerName = Callsite.Func;
103 LineLocation CallsiteLoc = IsLeaf ? LineLocation(0, 0) : Callsite.Location;
104 CurNode = CurNode->getOrCreateChildContext(CallSite: CallsiteLoc, ChildName: CallerName);
105 IsLeaf = false;
106 }
107
108 CurNode->addFunctionSize(FSize: InstrSize);
109}
110
111uint32_t
112BinarySizeContextTracker::getFuncSizeForContext(const ContextTrieNode *Node) {
113 ContextTrieNode *CurrNode = &RootContext;
114 ContextTrieNode *PrevNode = nullptr;
115
116 std::optional<uint32_t> Size;
117
118 // Start from top-level context-less function, traverse down the reverse
119 // context trie to find the best/longest match for given context, then
120 // retrieve the size.
121 LineLocation CallSiteLoc(0, 0);
122 while (CurrNode && Node->getParentContext() != nullptr) {
123 PrevNode = CurrNode;
124 CurrNode = CurrNode->getChildContext(CallSite: CallSiteLoc, ChildName: Node->getFuncName());
125 if (CurrNode && CurrNode->getFunctionSize())
126 Size = *CurrNode->getFunctionSize();
127 CallSiteLoc = Node->getCallSiteLoc();
128 Node = Node->getParentContext();
129 }
130
131 // If we traversed all nodes along the path of the context and haven't
132 // found a size yet, pivot to look for size from sibling nodes, i.e size
133 // of inlinee under different context.
134 if (!Size) {
135 if (!CurrNode)
136 CurrNode = PrevNode;
137 while (!Size && CurrNode && !CurrNode->getAllChildContext().empty()) {
138 CurrNode = &CurrNode->getAllChildContext().begin()->second;
139 if (CurrNode->getFunctionSize())
140 Size = *CurrNode->getFunctionSize();
141 }
142 }
143
144 assert(Size && "We should at least find one context size.");
145 return *Size;
146}
147
148void BinarySizeContextTracker::trackInlineesOptimizedAway(
149 MCPseudoProbeDecoder &ProbeDecoder) {
150 ProbeFrameStack ProbeContext;
151 for (const auto &Child : ProbeDecoder.getDummyInlineRoot().getChildren())
152 trackInlineesOptimizedAway(ProbeDecoder, ProbeNode: Child, Context&: ProbeContext);
153}
154
155void BinarySizeContextTracker::trackInlineesOptimizedAway(
156 MCPseudoProbeDecoder &ProbeDecoder,
157 const MCDecodedPseudoProbeInlineTree &ProbeNode,
158 ProbeFrameStack &ProbeContext) {
159 StringRef FuncName =
160 ProbeDecoder.getFuncDescForGUID(GUID: ProbeNode.Guid)->FuncName;
161 ProbeContext.emplace_back(Args&: FuncName, Args: 0);
162
163 // This ProbeContext has a probe, so it has code before inlining and
164 // optimization. Make sure we mark its size as known.
165 if (!ProbeNode.getProbes().empty()) {
166 ContextTrieNode *SizeContext = &RootContext;
167 for (auto &ProbeFrame : reverse(C&: ProbeContext)) {
168 StringRef CallerName = ProbeFrame.first;
169 LineLocation CallsiteLoc(ProbeFrame.second, 0);
170 SizeContext =
171 SizeContext->getOrCreateChildContext(CallSite: CallsiteLoc,
172 ChildName: FunctionId(CallerName));
173 }
174 // Add 0 size to make known.
175 SizeContext->addFunctionSize(FSize: 0);
176 }
177
178 // DFS down the probe inline tree
179 for (const auto &ChildNode : ProbeNode.getChildren()) {
180 InlineSite Location = ChildNode.getInlineSite();
181 ProbeContext.back().second = std::get<1>(t&: Location);
182 trackInlineesOptimizedAway(ProbeDecoder, ProbeNode: ChildNode, ProbeContext);
183 }
184
185 ProbeContext.pop_back();
186}
187
188ProfiledBinary::ProfiledBinary(const StringRef ExeBinPath,
189 const StringRef DebugBinPath)
190 : Path(ExeBinPath), DebugBinaryPath(DebugBinPath),
191 SymbolizerOpts(getSymbolizerOpts()), ProEpilogTracker(this),
192 Symbolizer(std::make_unique<symbolize::LLVMSymbolizer>(args&: SymbolizerOpts)),
193 TrackFuncContextSize(EnableCSPreInliner && UseContextCostForPreInliner) {
194 // Point to executable binary if debug info binary is not specified.
195 SymbolizerPath = DebugBinPath.empty() ? ExeBinPath : DebugBinPath;
196 if (InferMissingFrames)
197 MissingContextInferrer = std::make_unique<MissingFrameInferrer>(args: this);
198}
199
200ProfiledBinary::~ProfiledBinary() = default;
201
202void ProfiledBinary::warnNoFuncEntry() {
203 uint64_t NoFuncEntryNum = 0;
204 for (auto &F : BinaryFunctions) {
205 if (F.second.Ranges.empty())
206 continue;
207 bool hasFuncEntry = false;
208 for (auto &R : F.second.Ranges) {
209 if (FuncRange *FR = findFuncRangeForStartAddr(Address: R.first)) {
210 if (FR->IsFuncEntry) {
211 hasFuncEntry = true;
212 break;
213 }
214 }
215 }
216
217 if (!hasFuncEntry) {
218 NoFuncEntryNum++;
219 if (ShowDetailedWarning)
220 WithColor::warning()
221 << "Failed to determine function entry for " << F.first()
222 << " due to inconsistent name from symbol table and dwarf info.\n";
223 }
224 }
225 emitWarningSummary(Num: NoFuncEntryNum, Total: BinaryFunctions.size(),
226 Msg: "of functions failed to determine function entry due to "
227 "inconsistent name from symbol table and dwarf info.");
228}
229
230void ProfiledBinary::load(StringRef TripleStr) {
231 // Attempt to open the binary.
232 OBinary = unwrapOrError(EO: createBinary(Path), Args&: Path);
233 Binary &ExeBinary = *OBinary.getBinary();
234
235 IsCOFF = isa<COFFObjectFile>(Val: &ExeBinary);
236 if (!isa<ELFObjectFileBase>(Val: &ExeBinary) && !IsCOFF)
237 exitWithError(Message: "not a valid ELF/COFF image", Whence: Path);
238
239 auto *Obj = cast<ObjectFile>(Val: &ExeBinary);
240 if (!TripleStr.empty())
241 TheTriple = Triple(TripleStr);
242 else
243 TheTriple = Obj->makeTriple();
244
245 LLVM_DEBUG(dbgs() << "Loading " << Path << "\n");
246
247 // Mark the binary as a kernel image;
248 IsKernel = KernelBinary;
249
250 // Find the preferred load address for text sections.
251 setPreferredTextSegmentAddresses(Obj);
252
253 // For shared libraries, read build ID to filter perfscript addresses
254 // in [buildid:]addr format. Main executables (including PIE) use empty
255 // FilterBuildID since their addresses have no buildid prefix.
256 // Both PIE executables and shared libraries are ET_DYN, but only PIE
257 // executables have a PT_INTERP program header.
258 file_magic Magic;
259 if (auto EC = identify_magic(path: Path, result&: Magic);
260 !EC && Magic == file_magic::elf_shared_object && !HasInterp) {
261 auto BID = object::getBuildID(Obj);
262 if (!BID.empty())
263 FilterBuildID = llvm::toHex(Input: BID, /*LowerCase=*/true);
264 }
265
266 // Load debug info of subprograms from DWARF section.
267 // If path of debug info binary is specified, use the debug info from it,
268 // otherwise use the debug info from the executable binary.
269 OwningBinary<Binary> DebugBinary;
270 ObjectFile *PseudoProbeObj = nullptr;
271 if (!DebugBinaryPath.empty()) {
272 DebugBinary = unwrapOrError(EO: createBinary(Path: DebugBinaryPath), Args&: DebugBinaryPath);
273 ObjectFile *DebugObj = cast<ObjectFile>(Val: DebugBinary.getBinary());
274 loadSymbolsFromDWARF(Obj&: *DebugObj);
275 if (checkPseudoProbe(Obj: DebugObj, ObjPath: DebugBinaryPath))
276 PseudoProbeObj = DebugObj;
277 } else {
278 loadSymbolsFromDWARF(Obj&: *Obj);
279 }
280
281 // Prefer loading pseudo probe from binary.
282 if (checkPseudoProbe(Obj, ObjPath: Path))
283 PseudoProbeObj = Obj;
284
285 DisassembleFunctionSet.insert_range(R&: DisassembleFunctions);
286
287 if (usePseudoProbes())
288 populateSymbolAddressList(O: Obj);
289
290 if (ShowDisassemblyOnly && PseudoProbeObj)
291 decodePseudoProbe(Obj: PseudoProbeObj);
292
293 if (LoadFunctionFromSymbol && usePseudoProbes())
294 loadSymbolsFromSymtab(O: Obj);
295
296 // Disassemble the text sections.
297 disassemble(O: Obj);
298
299 // Use function start and return address to infer prolog and epilog
300 ProEpilogTracker.inferPrologAddresses(FuncStartAddressMap&: StartAddrToFuncRangeMap);
301 ProEpilogTracker.inferEpilogAddresses(RetAddrs&: RetAddressSet);
302
303 warnNoFuncEntry();
304
305 // TODO: decode other sections.
306}
307
308bool ProfiledBinary::inlineContextEqual(uint64_t Address1, uint64_t Address2) {
309 const SampleContextFrameVector &Context1 =
310 getCachedFrameLocationStack(Address: Address1);
311 const SampleContextFrameVector &Context2 =
312 getCachedFrameLocationStack(Address: Address2);
313 if (Context1.size() != Context2.size())
314 return false;
315 if (Context1.empty())
316 return false;
317 // The leaf frame contains location within the leaf, and it
318 // needs to be remove that as it's not part of the calling context
319 return std::equal(first1: Context1.begin(), last1: Context1.begin() + Context1.size() - 1,
320 first2: Context2.begin(), last2: Context2.begin() + Context2.size() - 1);
321}
322
323SampleContextFrameVector
324ProfiledBinary::getExpandedContext(const SmallVectorImpl<uint64_t> &Stack,
325 bool &WasLeafInlined) {
326 SampleContextFrameVector ContextVec;
327 if (Stack.empty())
328 return ContextVec;
329 // Process from frame root to leaf
330 for (auto Address : Stack) {
331 const SampleContextFrameVector &ExpandedContext =
332 getCachedFrameLocationStack(Address);
333 // An instruction without a valid debug line will be ignored by sample
334 // processing
335 if (ExpandedContext.empty())
336 return SampleContextFrameVector();
337 // Set WasLeafInlined to the size of inlined frame count for the last
338 // address which is leaf
339 WasLeafInlined = (ExpandedContext.size() > 1);
340 ContextVec.append(RHS: ExpandedContext);
341 }
342
343 // Replace with decoded base discriminator
344 for (auto &Frame : ContextVec) {
345 Frame.Location.Discriminator = ProfileGeneratorBase::getBaseDiscriminator(
346 Discriminator: Frame.Location.Discriminator, UseFSD: UseFSDiscriminator);
347 }
348
349 assert(ContextVec.size() && "Context length should be at least 1");
350
351 // Compress the context string except for the leaf frame
352 auto LeafFrame = ContextVec.back();
353 LeafFrame.Location = LineLocation(0, 0);
354 ContextVec.pop_back();
355 CSProfileGenerator::compressRecursionContext(Context&: ContextVec);
356 CSProfileGenerator::trimContext(S&: ContextVec);
357 ContextVec.push_back(Elt: LeafFrame);
358 return ContextVec;
359}
360
361template <class ELFT>
362void ProfiledBinary::setPreferredTextSegmentAddresses(const ELFFile<ELFT> &Obj,
363 StringRef FileName) {
364 const auto &PhdrRange = unwrapOrError(Obj.program_headers(), FileName);
365 // FIXME: This should be the page size of the system running profiling.
366 // However such info isn't available at post-processing time, assuming
367 // 4K page now. Note that we don't use EXEC_PAGESIZE from <linux/param.h>
368 // because we may build the tools on non-linux.
369 uint64_t PageSize = 0x1000;
370 bool SeenFirstLoadableSegment = false;
371 for (const typename ELFT::Phdr &Phdr : PhdrRange) {
372 if (Phdr.p_type == ELF::PT_INTERP)
373 HasInterp = true;
374 if (Phdr.p_type == ELF::PT_LOAD) {
375 if (!SeenFirstLoadableSegment) {
376 FirstLoadableAddress = Phdr.p_vaddr & ~(PageSize - 1U);
377 SeenFirstLoadableSegment = true;
378 }
379 if (Phdr.p_flags & ELF::PF_X) {
380 // Segments will always be loaded at a page boundary.
381 PreferredTextSegmentAddresses.push_back(Phdr.p_vaddr &
382 ~(PageSize - 1U));
383 TextSegmentOffsets.push_back(Phdr.p_offset & ~(PageSize - 1U));
384 } else {
385 PhdrInfo Info;
386 Info.FileOffset = Phdr.p_offset;
387 Info.FileSz = Phdr.p_filesz;
388 Info.VirtualAddr = Phdr.p_vaddr;
389 NonTextPhdrInfo.push_back(Elt: Info);
390 }
391 }
392 }
393
394 if (PreferredTextSegmentAddresses.empty())
395 exitWithError(Message: "no executable segment found", Whence: FileName);
396}
397
398uint64_t ProfiledBinary::CanonicalizeNonTextAddress(uint64_t Address) {
399 uint64_t FileOffset = 0;
400 auto MMapIter = NonTextMMapEvents.lower_bound(x: Address);
401 if (MMapIter == NonTextMMapEvents.end())
402 return Address; // No non-text mmap event found, return the address as is.
403
404 const auto &MMapEvent = MMapIter->second;
405
406 // If the address is within the non-text mmap event, calculate its file
407 // offset in the binary.
408 if (MMapEvent.Address <= Address &&
409 Address < MMapEvent.Address + MMapEvent.Size)
410 FileOffset = Address - MMapEvent.Address + MMapEvent.Offset;
411
412 // If the address is not within the non-text mmap event, return the address
413 // as is.
414 if (FileOffset == 0)
415 return Address;
416
417 for (const auto &PhdrInfo : NonTextPhdrInfo) {
418 // Find the program section that contains the file offset and map the
419 // file offset to the virtual address.
420 if (PhdrInfo.FileOffset <= FileOffset &&
421 FileOffset < PhdrInfo.FileOffset + PhdrInfo.FileSz)
422 return PhdrInfo.VirtualAddr + (FileOffset - PhdrInfo.FileOffset);
423 }
424
425 return Address;
426}
427
428void ProfiledBinary::setPreferredTextSegmentAddresses(const COFFObjectFile *Obj,
429 StringRef FileName) {
430 uint64_t ImageBase = Obj->getImageBase();
431 if (!ImageBase)
432 exitWithError(Message: "Not a COFF image", Whence: FileName);
433
434 PreferredTextSegmentAddresses.push_back(x: ImageBase);
435 FirstLoadableAddress = ImageBase;
436
437 for (SectionRef Section : Obj->sections()) {
438 const coff_section *Sec = Obj->getCOFFSection(Section);
439 if (Sec->Characteristics & COFF::IMAGE_SCN_CNT_CODE)
440 TextSegmentOffsets.push_back(x: Sec->VirtualAddress);
441 }
442}
443
444void ProfiledBinary::setPreferredTextSegmentAddresses(const ObjectFile *Obj) {
445 if (const auto *ELFObj = dyn_cast<ELF32LEObjectFile>(Val: Obj))
446 setPreferredTextSegmentAddresses(Obj: ELFObj->getELFFile(), FileName: Obj->getFileName());
447 else if (const auto *ELFObj = dyn_cast<ELF32BEObjectFile>(Val: Obj))
448 setPreferredTextSegmentAddresses(Obj: ELFObj->getELFFile(), FileName: Obj->getFileName());
449 else if (const auto *ELFObj = dyn_cast<ELF64LEObjectFile>(Val: Obj))
450 setPreferredTextSegmentAddresses(Obj: ELFObj->getELFFile(), FileName: Obj->getFileName());
451 else if (const auto *ELFObj = dyn_cast<ELF64BEObjectFile>(Val: Obj))
452 setPreferredTextSegmentAddresses(Obj: ELFObj->getELFFile(), FileName: Obj->getFileName());
453 else if (const auto *COFFObj = dyn_cast<COFFObjectFile>(Val: Obj))
454 setPreferredTextSegmentAddresses(Obj: COFFObj, FileName: Obj->getFileName());
455 else
456 llvm_unreachable("invalid object format");
457}
458
459bool ProfiledBinary::checkPseudoProbe(const ObjectFile *Obj,
460 StringRef ObjPath) {
461 if (UseDwarfCorrelation)
462 return false;
463
464 bool HasProbeDescSection = false;
465 bool HasPseudoProbeSection = false;
466
467 StringRef FileName = Obj->getFileName();
468 for (section_iterator SI = Obj->section_begin(), SE = Obj->section_end();
469 SI != SE; ++SI) {
470 const SectionRef &Section = *SI;
471 StringRef SectionName = unwrapOrError(EO: Section.getName(), Args&: FileName);
472 if (SectionName == ".pseudo_probe_desc") {
473 HasProbeDescSection = true;
474 } else if (SectionName == ".pseudo_probe") {
475 HasPseudoProbeSection = true;
476 }
477 }
478
479 if (HasProbeDescSection && HasPseudoProbeSection) {
480 PseudoProbeBinPath = ObjPath;
481 return true;
482 }
483
484 return false;
485}
486
487void ProfiledBinary::decodePseudoProbe(const ObjectFile *Obj) {
488 if (!usePseudoProbes())
489 return;
490
491 LLVM_DEBUG(dbgs() << "Decoding pseudo probe in " << Obj->getFileName()
492 << "\n");
493
494 MCPseudoProbeDecoder::Uint64Set GuidFilter;
495 MCPseudoProbeDecoder::Uint64Map FuncStartAddresses;
496 if (ShowDisassemblyOnly) {
497 if (DisassembleFunctionSet.empty()) {
498 FuncStartAddresses = SymbolStartAddrs;
499 } else {
500 for (auto &F : DisassembleFunctionSet) {
501 auto GUID = Function::getGUIDAssumingExternalLinkage(GlobalName: F.first());
502 if (auto StartAddr = SymbolStartAddrs.lookup(Val: GUID)) {
503 FuncStartAddresses[GUID] = StartAddr;
504 FuncRange &Range = StartAddrToFuncRangeMap[StartAddr];
505 GuidFilter.insert(
506 V: Function::getGUIDAssumingExternalLinkage(GlobalName: Range.getFuncName()));
507 }
508 }
509 }
510 } else {
511 for (auto *F : ProfiledFunctions) {
512 GuidFilter.insert(V: Function::getGUIDAssumingExternalLinkage(GlobalName: F->FuncName));
513 // DWARF name might be broken when a DWARF32 .debug_str.dwo section
514 // execeeds 4GB. We expect symbol table to contain the correct function
515 // names which matches the pseudo probe. Adding back all the GUIDs if
516 // possible.
517 auto AltGUIDs = AlternativeFunctionGUIDs.equal_range(x: F);
518 for (const auto &[_, Func] : make_range(p: AltGUIDs))
519 GuidFilter.insert(V: Func);
520 for (auto &Range : F->Ranges) {
521 auto GUIDs = StartAddrToSymMap.equal_range(x: Range.first);
522 for (const auto &[StartAddr, Func] : make_range(p: GUIDs))
523 FuncStartAddresses[Func] = StartAddr;
524 }
525 }
526 }
527
528 StringRef FileName = Obj->getFileName();
529 for (section_iterator SI = Obj->section_begin(), SE = Obj->section_end();
530 SI != SE; ++SI) {
531 const SectionRef &Section = *SI;
532 StringRef SectionName = unwrapOrError(EO: Section.getName(), Args&: FileName);
533
534 if (SectionName == ".pseudo_probe_desc") {
535 StringRef Contents = unwrapOrError(EO: Section.getContents(), Args&: FileName);
536 if (!ProbeDecoder.buildGUID2FuncDescMap(
537 Start: reinterpret_cast<const uint8_t *>(Contents.data()),
538 Size: Contents.size(), /*IsMMapped=*/false, VerboseWarnings: ShowDetailedWarning))
539 exitWithError(
540 Message: "Pseudo Probe decoder fail in .pseudo_probe_desc section");
541 } else if (SectionName == ".pseudo_probe") {
542 StringRef Contents = unwrapOrError(EO: Section.getContents(), Args&: FileName);
543 if (!ProbeDecoder.buildAddress2ProbeMap(
544 Start: reinterpret_cast<const uint8_t *>(Contents.data()),
545 Size: Contents.size(), GuildFilter: GuidFilter, FuncStartAddrs: FuncStartAddresses))
546 exitWithError(Message: "Pseudo Probe decoder fail in .pseudo_probe section");
547 }
548 }
549
550 // Build TopLevelProbeFrameMap to track size for optimized inlinees when probe
551 // is available
552 if (TrackFuncContextSize) {
553 for (auto &Child : ProbeDecoder.getDummyInlineRoot().getChildren()) {
554 auto *Frame = &Child;
555 StringRef FuncName =
556 ProbeDecoder.getFuncDescForGUID(GUID: Frame->Guid)->FuncName;
557 TopLevelProbeFrameMap[FuncName] = Frame;
558 }
559 }
560
561 if (ShowPseudoProbe)
562 ProbeDecoder.printGUID2FuncDescMap(OS&: outs());
563}
564
565void ProfiledBinary::decodePseudoProbe() {
566 OwningBinary<Binary> OBinary =
567 unwrapOrError(EO: createBinary(Path: PseudoProbeBinPath), Args&: PseudoProbeBinPath);
568 auto *Obj = cast<ObjectFile>(Val: OBinary.getBinary());
569 decodePseudoProbe(Obj);
570}
571
572void ProfiledBinary::setIsFuncEntry(FuncRange *FuncRange,
573 StringRef RangeSymName) {
574 // Skip external function symbol.
575 if (!FuncRange)
576 return;
577
578 // Set IsFuncEntry to ture if there is only one range in the function or the
579 // RangeSymName from ELF is equal to its DWARF-based function name.
580 if (FuncRange->Func->Ranges.size() == 1 ||
581 (!FuncRange->IsFuncEntry &&
582 (FuncRange->getFuncName() == RangeSymName ||
583 FuncRange->Func->NameStatus != DwarfNameStatus::Matched)))
584 FuncRange->IsFuncEntry = true;
585}
586
587bool ProfiledBinary::dissassembleSymbol(std::size_t SI, ArrayRef<uint8_t> Bytes,
588 SectionSymbolsTy &Symbols,
589 const SectionRef &Section) {
590 std::size_t SE = Symbols.size();
591 uint64_t SectionAddress = Section.getAddress();
592 uint64_t SectSize = Section.getSize();
593 uint64_t StartAddress = Symbols[SI].Addr;
594 uint64_t NextStartAddress =
595 (SI + 1 < SE) ? Symbols[SI + 1].Addr : SectionAddress + SectSize;
596 FuncRange *FRange = findFuncRange(Address: StartAddress);
597 setIsFuncEntry(FuncRange: FRange, RangeSymName: FunctionSamples::getCanonicalFnName(FnName: Symbols[SI].Name));
598 StringRef SymbolName =
599 ShowCanonicalFnName
600 ? FunctionSamples::getCanonicalFnName(FnName: Symbols[SI].Name)
601 : Symbols[SI].Name;
602 bool ShowDisassembly =
603 ShowDisassemblyOnly && (DisassembleFunctionSet.empty() ||
604 DisassembleFunctionSet.count(Key: SymbolName));
605 if (ShowDisassembly)
606 outs() << '<' << SymbolName << ">:\n";
607
608 uint64_t Address = StartAddress;
609 // Size of a consecutive invalid instruction range starting from Address -1
610 // backwards.
611 uint64_t InvalidInstLength = 0;
612 while (Address < NextStartAddress) {
613 MCInst Inst;
614 uint64_t Size;
615 // Disassemble an instruction.
616 bool Disassembled = DisAsm->getInstruction(
617 Instr&: Inst, Size, Bytes: Bytes.slice(N: Address - SectionAddress), Address, CStream&: nulls());
618 if (Size == 0)
619 Size = 1;
620
621 if (ShowDisassembly) {
622 if (ShowPseudoProbe) {
623 ProbeDecoder.printProbeForAddress(OS&: outs(), Address);
624 }
625 outs() << format(Fmt: "%8" PRIx64 ":", Vals: Address);
626 size_t Start = outs().tell();
627 if (Disassembled)
628 IPrinter->printInst(MI: &Inst, Address: Address + Size, Annot: "", STI: *STI, OS&: outs());
629 else
630 outs() << "\t<unknown>";
631 if (ShowSourceLocations) {
632 unsigned Cur = outs().tell() - Start;
633 if (Cur < 40)
634 outs().indent(NumSpaces: 40 - Cur);
635 InstructionPointer IP(this, Address);
636 outs() << getReversedLocWithContext(
637 Context: symbolize(IP, UseCanonicalFnName: ShowCanonicalFnName, UseProbeDiscriminator: ShowPseudoProbe));
638 }
639 outs() << "\n";
640 }
641
642 if (Disassembled) {
643 const MCInstrDesc &MCDesc = MII->get(Opcode: Inst.getOpcode());
644
645 // Record instruction size.
646 AddressToInstSizeMap[Address] = Size;
647
648 // Populate address maps.
649 CodeAddressVec.push_back(x: Address);
650 if (MCDesc.isCall()) {
651 CallAddressSet.insert(V: Address);
652 UncondBranchAddrSet.insert(x: Address);
653 // Record the instruction after call as the branch target of a ret
654 BranchTargetAddressSet.insert(V: Address + Size);
655 } else if (MCDesc.isReturn()) {
656 RetAddressSet.insert(V: Address);
657 UncondBranchAddrSet.insert(x: Address);
658 } else if (MCDesc.isBranch()) {
659 if (MCDesc.isUnconditionalBranch())
660 UncondBranchAddrSet.insert(x: Address);
661 BranchAddressSet.insert(V: Address);
662 }
663
664 if (MCDesc.isIndirectBranch()) {
665 IndirectBranchAddressSet.insert(V: Address);
666 }
667
668 // Record branch target addresses for branches and calls.
669 if (MCDesc.isCall() || MCDesc.isBranch()) {
670 uint64_t Target = 0;
671 if (MIA->evaluateBranch(Inst, Addr: Address, Size, Target))
672 BranchTargetAddressSet.insert(V: Target);
673 }
674
675 // Record potential call targets for tail frame inference later-on.
676 if (InferMissingFrames && FRange) {
677 uint64_t Target = 0;
678 [[maybe_unused]] bool Err =
679 MIA->evaluateBranch(Inst, Addr: Address, Size, Target);
680 if (MCDesc.isCall()) {
681 // Indirect call targets are unknown at this point. Recording the
682 // unknown target (zero) for further LBR-based refinement.
683 MissingContextInferrer->CallEdges[Address].insert(V: Target);
684 } else if (MCDesc.isUnconditionalBranch()) {
685 assert(Err &&
686 "target should be known for unconditional direct branch");
687 // Any inter-function unconditional jump is considered tail call at
688 // this point. This is not 100% accurate and could further be
689 // optimized based on some source annotation.
690 FuncRange *ToFRange = findFuncRange(Address: Target);
691 if (ToFRange && ToFRange->Func != FRange->Func)
692 MissingContextInferrer->TailCallEdges[Address].insert(V: Target);
693 LLVM_DEBUG({
694 dbgs() << "Direct Tail call: " << format("%8" PRIx64 ":", Address);
695 IPrinter->printInst(&Inst, Address + Size, "", *STI.get(), dbgs());
696 dbgs() << "\n";
697 });
698 } else if (MCDesc.isIndirectBranch() && MCDesc.isBarrier()) {
699 // This is an indirect branch but not necessarily an indirect tail
700 // call. The isBarrier check is to filter out conditional branch.
701 // Similar with indirect call targets, recording the unknown target
702 // (zero) for further LBR-based refinement.
703 MissingContextInferrer->TailCallEdges[Address].insert(V: Target);
704 LLVM_DEBUG({
705 dbgs() << "Indirect Tail call: "
706 << format("%8" PRIx64 ":", Address);
707 IPrinter->printInst(&Inst, Address + Size, "", *STI.get(), dbgs());
708 dbgs() << "\n";
709 });
710 }
711 }
712
713 if (InvalidInstLength) {
714 AddrsWithInvalidInstruction.insert(
715 V: {Address - InvalidInstLength, Address - 1});
716 InvalidInstLength = 0;
717 }
718 } else {
719 InvalidInstLength += Size;
720 }
721
722 Address += Size;
723 }
724
725 if (InvalidInstLength)
726 AddrsWithInvalidInstruction.insert(
727 V: {Address - InvalidInstLength, Address - 1});
728
729 if (ShowDisassembly)
730 outs() << "\n";
731
732 return true;
733}
734
735void ProfiledBinary::setUpDisassembler(const ObjectFile *Obj) {
736 const Target *TheTarget = getTarget(Obj);
737 StringRef FileName = Obj->getFileName();
738
739 MRI.reset(p: TheTarget->createMCRegInfo(TT: TheTriple));
740 if (!MRI)
741 exitWithError(Message: "no register info for target " + TheTriple.str(), Whence: FileName);
742
743 MCTargetOptions MCOptions;
744 AsmInfo.reset(p: TheTarget->createMCAsmInfo(MRI: *MRI, TheTriple, Options: MCOptions));
745 if (!AsmInfo)
746 exitWithError(Message: "no assembly info for target " + TheTriple.str(), Whence: FileName);
747
748 Expected<SubtargetFeatures> Features = Obj->getFeatures();
749 if (!Features)
750 exitWithError(E: Features.takeError(), Whence: FileName);
751 // AArch64 object files do not generally carry complete ISA feature metadata,
752 // so the subtarget would default to the baseline (Armv8.0-A) feature set.
753 // That disassembler cannot decode feature-gated instructions (LSE atomics,
754 // RCPC loads, SVE, ...) that are pervasive in modern AArch64 binaries; they
755 // would be miscounted as "invalid instructions" and, worse, their addresses
756 // would be absent from the code/branch maps used for sample attribution.
757 // Enable all instructions so the disassembler recognizes whatever the
758 // compiler emitted, matching llvm-objdump's default for AArch64.
759 if (TheTriple.isAArch64())
760 Features->AddFeature(String: "+all");
761 STI.reset(
762 p: TheTarget->createMCSubtargetInfo(TheTriple, CPU: "", Features: Features->getString()));
763 if (!STI)
764 exitWithError(Message: "no subtarget info for target " + TheTriple.str(), Whence: FileName);
765
766 MII.reset(p: TheTarget->createMCInstrInfo());
767 if (!MII)
768 exitWithError(Message: "no instruction info for target " + TheTriple.str(),
769 Whence: FileName);
770
771 MCContext Ctx(TheTriple, *AsmInfo, *MRI, *STI);
772 std::unique_ptr<MCObjectFileInfo> MOFI(
773 TheTarget->createMCObjectFileInfo(Ctx, /*PIC=*/false));
774 Ctx.setObjectFileInfo(MOFI.get());
775 DisAsm.reset(p: TheTarget->createMCDisassembler(STI: *STI, Ctx));
776 if (!DisAsm)
777 exitWithError(Message: "no disassembler for target " + TheTriple.str(), Whence: FileName);
778
779 MIA.reset(p: TheTarget->createMCInstrAnalysis(Info: MII.get()));
780
781 int AsmPrinterVariant = AsmInfo->getAssemblerDialect();
782 IPrinter.reset(p: TheTarget->createMCInstPrinter(T: TheTriple, SyntaxVariant: AsmPrinterVariant,
783 MAI: *AsmInfo, MII: *MII, MRI: *MRI));
784 IPrinter->setPrintBranchImmAsAddress(true);
785}
786
787void ProfiledBinary::disassemble(const ObjectFile *Obj) {
788 // Set up disassembler and related components.
789 setUpDisassembler(Obj);
790
791 // Create a mapping from virtual address to symbol name. The symbols in text
792 // sections are the candidates to dissassemble.
793 std::map<SectionRef, SectionSymbolsTy> AllSymbols;
794 StringRef FileName = Obj->getFileName();
795 for (const SymbolRef &Symbol : Obj->symbols()) {
796 const uint64_t Addr = unwrapOrError(EO: Symbol.getAddress(), Args&: FileName);
797 const StringRef Name = unwrapOrError(EO: Symbol.getName(), Args&: FileName);
798 section_iterator SecI = unwrapOrError(EO: Symbol.getSection(), Args&: FileName);
799 if (SecI != Obj->section_end())
800 AllSymbols[*SecI].push_back(x: SymbolInfoTy(Addr, Name, ELF::STT_NOTYPE));
801 }
802
803 // Sort all the symbols. Use a stable sort to stabilize the output.
804 for (std::pair<const SectionRef, SectionSymbolsTy> &SecSyms : AllSymbols)
805 stable_sort(Range&: SecSyms.second);
806
807 assert((DisassembleFunctionSet.empty() || ShowDisassemblyOnly) &&
808 "Functions to disassemble should be only specified together with "
809 "--show-disassembly-only");
810
811 if (ShowDisassemblyOnly)
812 outs() << "\nDisassembly of " << FileName << ":\n";
813
814 // Dissassemble a text section.
815 for (section_iterator SI = Obj->section_begin(), SE = Obj->section_end();
816 SI != SE; ++SI) {
817 const SectionRef &Section = *SI;
818 if (!Section.isText())
819 continue;
820
821 uint64_t ImageLoadAddr = getPreferredBaseAddress();
822 uint64_t SectionAddress = Section.getAddress() - ImageLoadAddr;
823 uint64_t SectSize = Section.getSize();
824 if (!SectSize)
825 continue;
826
827 // Register the text section.
828 TextSections.insert(x: {SectionAddress, SectSize});
829
830 StringRef SectionName = unwrapOrError(EO: Section.getName(), Args&: FileName);
831
832 if (ShowDisassemblyOnly) {
833 outs() << "\nDisassembly of section " << SectionName;
834 outs() << " [" << format(Fmt: "0x%" PRIx64, Vals: Section.getAddress()) << ", "
835 << format(Fmt: "0x%" PRIx64, Vals: Section.getAddress() + SectSize)
836 << "]:\n\n";
837 }
838
839 if (isa<ELFObjectFileBase>(Val: Obj) && SectionName == ".plt")
840 continue;
841
842 // Get the section data.
843 ArrayRef<uint8_t> Bytes =
844 arrayRefFromStringRef(Input: unwrapOrError(EO: Section.getContents(), Args&: FileName));
845
846 // Get the list of all the symbols in this section.
847 SectionSymbolsTy &Symbols = AllSymbols[Section];
848
849 // Disassemble symbol by symbol.
850 for (std::size_t SI = 0, SE = Symbols.size(); SI != SE; ++SI) {
851 if (!dissassembleSymbol(SI, Bytes, Symbols, Section))
852 exitWithError(Message: "disassembling error", Whence: FileName);
853 }
854 }
855
856 if (!AddrsWithInvalidInstruction.empty()) {
857 if (ShowDetailedWarning) {
858 for (auto &Addr : AddrsWithInvalidInstruction) {
859 WithColor::warning()
860 << "Invalid instructions at " << format(Fmt: "%8" PRIx64, Vals: Addr.first)
861 << " - " << format(Fmt: "%8" PRIx64, Vals: Addr.second) << "\n";
862 }
863 }
864 WithColor::warning() << "Found " << AddrsWithInvalidInstruction.size()
865 << " invalid instructions\n";
866 AddrsWithInvalidInstruction.clear();
867 }
868
869 // Dissassemble rodata section to check if FS discriminator symbol exists.
870 checkUseFSDiscriminator(Obj, AllSymbols);
871}
872
873void ProfiledBinary::checkUseFSDiscriminator(
874 const ObjectFile *Obj, std::map<SectionRef, SectionSymbolsTy> &AllSymbols) {
875 const char *FSDiscriminatorVar = "__llvm_fs_discriminator__";
876 for (section_iterator SI = Obj->section_begin(), SE = Obj->section_end();
877 SI != SE; ++SI) {
878 const SectionRef &Section = *SI;
879 if (!Section.isData() || Section.getSize() == 0)
880 continue;
881 SectionSymbolsTy &Symbols = AllSymbols[Section];
882
883 for (std::size_t SI = 0, SE = Symbols.size(); SI != SE; ++SI) {
884 if (Symbols[SI].Name == FSDiscriminatorVar) {
885 UseFSDiscriminator = true;
886 return;
887 }
888 }
889 }
890}
891
892void ProfiledBinary::populateSymbolAddressList(const ObjectFile *Obj) {
893 // Create a mapping from virtual address to symbol GUID and the other way
894 // around.
895 StringRef FileName = Obj->getFileName();
896 for (const SymbolRef &Symbol : Obj->symbols()) {
897 const uint64_t Addr = unwrapOrError(EO: Symbol.getAddress(), Args&: FileName);
898 const StringRef Name = unwrapOrError(EO: Symbol.getName(), Args&: FileName);
899 uint64_t GUID = Function::getGUIDAssumingExternalLinkage(GlobalName: Name);
900 SymbolStartAddrs[GUID] = Addr;
901 StartAddrToSymMap.emplace(args: Addr, args&: GUID);
902 }
903}
904
905void ProfiledBinary::loadSymbolsFromSymtab(const ObjectFile *Obj) {
906 // Load binary functions from symbol table when Debug info is incomplete.
907 // Strip the internal suffixes which are not reflected in the DWARF info.
908 const SmallVector<StringRef, 10> Suffixes(
909 {// Internal suffixes from CoroSplit pass
910 ".cleanup", ".destroy", ".resume",
911 // Internal suffixes from Bolt
912 ".cold", ".warm",
913 // Compiler/LTO internal
914 ".llvm.", ".part.", ".isra.", ".constprop.", ".lto_priv."});
915 StringRef FileName = Obj->getFileName();
916
917 // COFF symtab does not have size field. Try to load size from PDB instead.
918 std::unique_ptr<pdb::IPDBSession> PDBSession;
919 if (auto *COFFObj = dyn_cast<COFFObjectFile>(Val: Obj)) {
920 if (auto E = pdb::loadDataForEXE(Type: pdb::PDB_ReaderType::Native, Path: FileName,
921 Session&: PDBSession)) {
922 StringRef PdbPath;
923 const codeview::DebugInfo *PdbInfo;
924 if (auto Err = COFFObj->getDebugPDBInfo(Info&: PdbInfo, PDBFileName&: PdbPath))
925 consumeError(Err: std::move(Err));
926
927 auto Style = PdbPath.starts_with(Prefix: "/") ? sys::path::Style::posix
928 : sys::path::Style::windows;
929 WithColor::warning() << "Cannot load PDB file "
930 << sys::path::filename(path: PdbPath, style: Style) << " for "
931 << FileName << ": " << E << "\n";
932 consumeError(Err: std::move(E));
933 } else {
934 PDBSession->setLoadAddress(FirstLoadableAddress);
935 }
936 }
937
938 for (const SymbolRef &Symbol : Obj->symbols()) {
939 const SymbolRef::Type Type = unwrapOrError(EO: Symbol.getType(), Args&: FileName);
940 const uint64_t StartAddr = unwrapOrError(EO: Symbol.getAddress(), Args&: FileName);
941 const StringRef Name = unwrapOrError(EO: Symbol.getName(), Args&: FileName);
942 uint64_t Size = 0;
943 if (isa<ELFObjectFileBase>(Val: Obj)) {
944 ELFSymbolRef ElfSymbol(Symbol);
945 Size = ElfSymbol.getSize();
946 } else if (PDBSession) {
947 if (std::unique_ptr<pdb::PDBSymbol> Sym = PDBSession->findSymbolByAddress(
948 Address: StartAddr, Type: pdb::PDB_SymType::Function)) {
949 auto FuncSym = cast<pdb::PDBSymbolFunc>(Val: std::move(Sym));
950 if (StartAddr == FuncSym->getVirtualAddress())
951 Size = FuncSym->getLength();
952 }
953 }
954
955 if (Size == 0 || Type != SymbolRef::ST_Function)
956 continue;
957
958 const uint64_t EndAddr = StartAddr + Size;
959 const StringRef SymName =
960 FunctionSamples::getCanonicalFnName(FnName: Name, Suffixes);
961 assert(StartAddr < EndAddr && StartAddr >= getPreferredBaseAddress() &&
962 "Function range is invalid.");
963
964 auto Range = findFuncRange(Address: StartAddr);
965 if (!Range) {
966 assert(findFuncRange(EndAddr - 1) == nullptr &&
967 "Function range overlaps with existing functions.");
968 // Function from symbol table not found previously in DWARF, store ranges.
969 auto Ret = BinaryFunctions.try_emplace(Key: SymName);
970 auto &Func = Ret.first->second;
971 if (Ret.second) {
972 Func.FuncName = Ret.first->first();
973 HashBinaryFunctions[Function::getGUIDAssumingExternalLinkage(GlobalName: SymName)] =
974 &Func;
975 }
976
977 Func.NameStatus = DwarfNameStatus::Missing;
978 Func.Ranges.emplace_back(args: StartAddr, args: EndAddr);
979
980 auto R = StartAddrToFuncRangeMap.emplace(args: StartAddr, args: FuncRange());
981 FuncRange &FRange = R.first->second;
982
983 FRange.Func = &Func;
984 FRange.StartAddress = StartAddr;
985 FRange.EndAddress = EndAddr;
986
987 } else if (SymName != Range->getFuncName()) {
988 // Function range already found from DWARF or symtab, but the symbol name
989 // from symbol table is inconsistent with the existing name associated
990 // with the range. Log this discrepancy and the alternative function GUID.
991 if (ShowDetailedWarning)
992 WithColor::warning()
993 << "Conflicting name for symbol " << Name << " with range ("
994 << format(Fmt: "%8" PRIx64, Vals: StartAddr) << ", "
995 << format(Fmt: "%8" PRIx64, Vals: EndAddr) << ")"
996 << ", but the existing symbol " << Range->getFuncName()
997 << " indicates an overlapping range ("
998 << format(Fmt: "%8" PRIx64, Vals: Range->StartAddress) << ", "
999 << format(Fmt: "%8" PRIx64, Vals: Range->EndAddress) << ")\n";
1000
1001 assert(StartAddr == Range->StartAddress && EndAddr == Range->EndAddress &&
1002 "Mismatched function range");
1003
1004 Range->Func->NameStatus = DwarfNameStatus::Mismatch;
1005 AlternativeFunctionGUIDs.emplace(
1006 args&: Range->Func, args: Function::getGUIDAssumingExternalLinkage(GlobalName: SymName));
1007
1008 } else if (StartAddr != Range->StartAddress &&
1009 EndAddr != Range->EndAddress) {
1010 // Function already found in DWARF or symtab, but the address range from
1011 // symbol table conflicts/overlaps with the existing one.
1012 WithColor::warning() << "Conflicting range for symbol " << Name
1013 << " with range (" << format(Fmt: "%8" PRIx64, Vals: StartAddr)
1014 << ", " << format(Fmt: "%8" PRIx64, Vals: EndAddr) << ")"
1015 << ", but the existing symbol "
1016 << Range->getFuncName()
1017 << " indicates another range ("
1018 << format(Fmt: "%8" PRIx64, Vals: Range->StartAddress) << ", "
1019 << format(Fmt: "%8" PRIx64, Vals: Range->EndAddress) << ")\n";
1020 }
1021 }
1022}
1023
1024void ProfiledBinary::loadSymbolsFromDWARFUnit(DWARFUnit &CompilationUnit) {
1025 for (const auto &DieInfo : CompilationUnit.dies()) {
1026 llvm::DWARFDie Die(&CompilationUnit, &DieInfo);
1027
1028 if (!Die.isSubprogramDIE())
1029 continue;
1030 auto Name = Die.getName(Kind: llvm::DINameKind::LinkageName);
1031 if (!Name)
1032 Name = Die.getName(Kind: llvm::DINameKind::ShortName);
1033 if (!Name)
1034 continue;
1035
1036 auto RangesOrError = Die.getAddressRanges();
1037 if (!RangesOrError)
1038 continue;
1039 const DWARFAddressRangesVector &Ranges = RangesOrError.get();
1040
1041 if (Ranges.empty())
1042 continue;
1043
1044 // Different DWARF symbols can have same function name, search or create
1045 // BinaryFunction indexed by the name.
1046 auto Ret = BinaryFunctions.try_emplace(Key: Name);
1047 auto &Func = Ret.first->second;
1048 if (Ret.second)
1049 Func.FuncName = Ret.first->first();
1050
1051 for (const auto &Range : Ranges) {
1052 uint64_t StartAddress = Range.LowPC;
1053 uint64_t EndAddress = Range.HighPC;
1054
1055 if (EndAddress <= StartAddress ||
1056 StartAddress < getPreferredBaseAddress())
1057 continue;
1058
1059 // We may want to know all ranges for one function. Here group the
1060 // ranges and store them into BinaryFunction.
1061 Func.Ranges.emplace_back(args&: StartAddress, args&: EndAddress);
1062
1063 auto R = StartAddrToFuncRangeMap.emplace(args&: StartAddress, args: FuncRange());
1064 if (R.second) {
1065 FuncRange &FRange = R.first->second;
1066 FRange.Func = &Func;
1067 FRange.StartAddress = StartAddress;
1068 FRange.EndAddress = EndAddress;
1069 } else {
1070 AddrsWithMultipleSymbols.insert(V: StartAddress);
1071 if (ShowDetailedWarning)
1072 WithColor::warning()
1073 << "Duplicated symbol start address at "
1074 << format(Fmt: "%8" PRIx64, Vals: StartAddress) << " "
1075 << R.first->second.getFuncName() << " and " << Name << "\n";
1076 }
1077 }
1078 }
1079}
1080
1081void ProfiledBinary::loadSymbolsFromDWARF(ObjectFile &Obj) {
1082 auto DebugContext = llvm::DWARFContext::create(
1083 Obj, RelocAction: DWARFContext::ProcessDebugRelocations::Process, L: nullptr, DWPName: DWPPath);
1084 if (!DebugContext)
1085 exitWithError(Message: "Error creating the debug info context", Whence: Path);
1086
1087 for (const auto &CompilationUnit : DebugContext->compile_units())
1088 loadSymbolsFromDWARFUnit(CompilationUnit&: *CompilationUnit);
1089
1090 // Handles DWO sections that can either be in .o, .dwo or .dwp files.
1091 uint32_t NumOfDWOMissing = 0;
1092 for (const auto &CompilationUnit : DebugContext->compile_units()) {
1093 DWARFUnit *const DwarfUnit = CompilationUnit.get();
1094 if (DwarfUnit->getDWOId()) {
1095 DWARFUnit *DWOCU = DwarfUnit->getNonSkeletonUnitDIE(ExtractUnitDIEOnly: false).getDwarfUnit();
1096 if (!DWOCU->isDWOUnit()) {
1097 NumOfDWOMissing++;
1098 if (ShowDetailedWarning) {
1099 std::string DWOName = dwarf::toString(
1100 V: DwarfUnit->getUnitDIE().find(
1101 Attrs: {dwarf::DW_AT_dwo_name, dwarf::DW_AT_GNU_dwo_name}),
1102 Default: "");
1103 WithColor::warning() << "DWO debug information for " << DWOName
1104 << " was not loaded.\n";
1105 }
1106 continue;
1107 }
1108 loadSymbolsFromDWARFUnit(CompilationUnit&: *DWOCU);
1109 }
1110 }
1111
1112 if (NumOfDWOMissing)
1113 WithColor::warning()
1114 << " DWO debug information was not loaded for " << NumOfDWOMissing
1115 << " modules. Please check the .o, .dwo or .dwp path.\n";
1116 if (BinaryFunctions.empty())
1117 WithColor::warning() << "Loading of DWARF info completed, but no binary "
1118 "functions have been retrieved.\n";
1119 // Populate the hash binary function map for MD5 function name lookup. This
1120 // is done after BinaryFunctions are finalized.
1121 for (auto &BinaryFunction : BinaryFunctions) {
1122 HashBinaryFunctions[MD5Hash(Str: BinaryFunction.first())] =
1123 &BinaryFunction.second;
1124 }
1125
1126 if (!AddrsWithMultipleSymbols.empty()) {
1127 WithColor::warning() << "Found " << AddrsWithMultipleSymbols.size()
1128 << " start addresses with multiple symbols\n";
1129 AddrsWithMultipleSymbols.clear();
1130 }
1131}
1132
1133void ProfiledBinary::populateSymbolListFromDWARF(
1134 ProfileSymbolList &SymbolList) {
1135 for (auto &I : StartAddrToFuncRangeMap)
1136 SymbolList.add(Name: I.second.getFuncName());
1137}
1138
1139symbolize::LLVMSymbolizer::Options ProfiledBinary::getSymbolizerOpts() const {
1140 symbolize::LLVMSymbolizer::Options SymbolizerOpts;
1141 SymbolizerOpts.PrintFunctions =
1142 DILineInfoSpecifier::FunctionNameKind::LinkageName;
1143 SymbolizerOpts.Demangle = false;
1144 SymbolizerOpts.DefaultArch = TheTriple.getArchName().str();
1145 SymbolizerOpts.UseSymbolTable = false;
1146 SymbolizerOpts.RelativeAddresses = false;
1147 SymbolizerOpts.DWPName = DWPPath;
1148 return SymbolizerOpts;
1149}
1150
1151SampleContextFrameVector ProfiledBinary::symbolize(const InstructionPointer &IP,
1152 bool UseCanonicalFnName,
1153 bool UseProbeDiscriminator) {
1154 assert(this == IP.Binary &&
1155 "Binary should only symbolize its own instruction");
1156 DIInliningInfo InlineStack =
1157 unwrapOrError(EO: Symbolizer->symbolizeInlinedCode(
1158 ModuleName: SymbolizerPath.str(), ModuleOffset: getSectionedAddress(Address: IP.Address)),
1159 Args&: SymbolizerPath);
1160
1161 SampleContextFrameVector CallStack;
1162 for (int32_t I = InlineStack.getNumberOfFrames() - 1; I >= 0; I--) {
1163 const auto &CallerFrame = InlineStack.getFrame(Index: I);
1164 if (CallerFrame.FunctionName.empty() ||
1165 (CallerFrame.FunctionName == "<invalid>"))
1166 break;
1167
1168 StringRef FunctionName(CallerFrame.FunctionName);
1169 if (UseCanonicalFnName)
1170 FunctionName = FunctionSamples::getCanonicalFnName(FnName: FunctionName);
1171
1172 uint32_t Discriminator = CallerFrame.Discriminator;
1173 uint32_t LineOffset = (CallerFrame.Line - CallerFrame.StartLine) & 0xffff;
1174 if (UseProbeDiscriminator) {
1175 LineOffset =
1176 PseudoProbeDwarfDiscriminator::extractProbeIndex(Value: Discriminator);
1177 Discriminator = 0;
1178 }
1179
1180 LineLocation Line(LineOffset, Discriminator);
1181 auto It = NameStrings.insert(key: FunctionName);
1182 CallStack.emplace_back(Args: FunctionId(It.first->getKey()), Args&: Line);
1183 }
1184
1185 return CallStack;
1186}
1187
1188StringRef ProfiledBinary::symbolizeDataAddress(uint64_t Address) {
1189 DIGlobal DataDIGlobal =
1190 unwrapOrError(EO: Symbolizer->symbolizeData(ModuleName: SymbolizerPath.str(),
1191 ModuleOffset: getSectionedAddress(Address)),
1192 Args&: SymbolizerPath);
1193 return NameStrings.insert(key: DataDIGlobal.Name).first->getKey();
1194}
1195
1196void ProfiledBinary::computeInlinedContextSizeForRange(uint64_t RangeBegin,
1197 uint64_t RangeEnd) {
1198 InstructionPointer IP(this, RangeBegin, true);
1199
1200 if (IP.Address != RangeBegin)
1201 WithColor::warning() << "Invalid start instruction at "
1202 << format(Fmt: "%8" PRIx64, Vals: RangeBegin) << "\n";
1203
1204 if (IP.Address >= RangeEnd)
1205 return;
1206
1207 do {
1208 const SampleContextFrameVector SymbolizedCallStack =
1209 getFrameLocationStack(Address: IP.Address, UseProbeDiscriminator: usePseudoProbes());
1210 uint64_t Size = AddressToInstSizeMap[IP.Address];
1211 // Record instruction size for the corresponding context
1212 FuncSizeTracker.addInstructionForContext(Context: SymbolizedCallStack, InstrSize: Size);
1213
1214 } while (IP.advance() && IP.Address < RangeEnd);
1215}
1216
1217void ProfiledBinary::computeInlinedContextSizeForFunc(
1218 const BinaryFunction *Func) {
1219 // Note that a function can be spilt into multiple ranges, so compute for all
1220 // ranges of the function.
1221 for (const auto &Range : Func->Ranges)
1222 computeInlinedContextSizeForRange(RangeBegin: Range.first, RangeEnd: Range.second);
1223
1224 // Track optimized-away inlinee for probed binary. A function inlined and then
1225 // optimized away should still have their probes left over in places.
1226 if (usePseudoProbes()) {
1227 auto I = TopLevelProbeFrameMap.find(Key: Func->FuncName);
1228 if (I != TopLevelProbeFrameMap.end()) {
1229 BinarySizeContextTracker::ProbeFrameStack ProbeContext;
1230 FuncSizeTracker.trackInlineesOptimizedAway(ProbeDecoder, ProbeNode: *I->second,
1231 ProbeContext);
1232 }
1233 }
1234}
1235
1236void ProfiledBinary::loadSymbolsFromPseudoProbe() {
1237 if (!usePseudoProbes())
1238 return;
1239
1240 const AddressProbesMap &Address2ProbesMap = getAddress2ProbesMap();
1241 for (auto *Func : ProfiledFunctions) {
1242 if (Func->NameStatus != DwarfNameStatus::Mismatch)
1243 continue;
1244 for (auto &[StartAddr, EndAddr] : Func->Ranges) {
1245 auto Range = findFuncRangeForStartAddr(Address: StartAddr);
1246 if (!Range->IsFuncEntry)
1247 continue;
1248 const auto &Probe = Address2ProbesMap.find(From: StartAddr, To: EndAddr);
1249 if (Probe.begin() != Probe.end()) {
1250 const MCDecodedPseudoProbeInlineTree *InlineTreeNode =
1251 Probe.begin()->get().getInlineTreeNode();
1252 while (!InlineTreeNode->isTopLevelFunc())
1253 InlineTreeNode = static_cast<MCDecodedPseudoProbeInlineTree *>(
1254 InlineTreeNode->Parent);
1255
1256 assert(llvm::any_of(InlineTreeNode->getProbes(),
1257 [Start = StartAddr, End = EndAddr](const auto &P) {
1258 return P.getAddress() >= Start &&
1259 P.getAddress() < End;
1260 }) &&
1261 "Top level pseudo probe does not match function range");
1262
1263 const auto *ProbeDesc = getFuncDescForGUID(GUID: InlineTreeNode->Guid);
1264 auto Ret = PseudoProbeNames.try_emplace(Key: Func, Args: ProbeDesc->FuncName);
1265 if (!Ret.second && Ret.first->second != ProbeDesc->FuncName &&
1266 ShowDetailedWarning)
1267 WithColor::warning()
1268 << "Mismatched pseudo probe names in function " << Func->FuncName
1269 << " at range: (" << format(Fmt: "%8" PRIx64, Vals: StartAddr) << ", "
1270 << format(Fmt: "%8" PRIx64, Vals: EndAddr) << "). "
1271 << "The previously found pseudo probe name is "
1272 << Ret.first->second << " but it conflicts with name "
1273 << ProbeDesc->FuncName
1274 << " This likely indicates a DWARF error that produces "
1275 "conflicting symbols at the same starting address.\n";
1276 }
1277 }
1278 }
1279}
1280
1281StringRef ProfiledBinary::findPseudoProbeName(const BinaryFunction *Func) {
1282 auto ProbeName = PseudoProbeNames.find(Val: Func);
1283 if (ProbeName == PseudoProbeNames.end())
1284 return StringRef();
1285 return ProbeName->second;
1286}
1287
1288void ProfiledBinary::inferMissingFrames(
1289 const SmallVectorImpl<uint64_t> &Context,
1290 SmallVectorImpl<uint64_t> &NewContext) {
1291 MissingContextInferrer->inferMissingFrames(Context, NewContext);
1292}
1293
1294InstructionPointer::InstructionPointer(const ProfiledBinary *Binary,
1295 uint64_t Address, bool RoundToNext)
1296 : Binary(Binary), Address(Address) {
1297 Index = Binary->getIndexForAddr(Address);
1298 if (RoundToNext) {
1299 // we might get address which is not the code
1300 // it should round to the next valid address
1301 if (Index >= Binary->getCodeAddrVecSize())
1302 this->Address = UINT64_MAX;
1303 else
1304 this->Address = Binary->getAddressforIndex(Index);
1305 }
1306}
1307
1308bool InstructionPointer::advance() {
1309 Index++;
1310 if (Index >= Binary->getCodeAddrVecSize()) {
1311 Address = UINT64_MAX;
1312 return false;
1313 }
1314 Address = Binary->getAddressforIndex(Index);
1315 return true;
1316}
1317
1318bool InstructionPointer::backward() {
1319 if (Index == 0) {
1320 Address = 0;
1321 return false;
1322 }
1323 Index--;
1324 Address = Binary->getAddressforIndex(Index);
1325 return true;
1326}
1327
1328void InstructionPointer::update(uint64_t Addr) {
1329 Address = Addr;
1330 Index = Binary->getIndexForAddr(Address);
1331}
1332
1333} // end namespace sampleprof
1334} // end namespace llvm
1335