1//===-- ProfiledBinary.cpp - Binary decoder ---------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "ProfiledBinary.h"
10#include "ErrorHandling.h"
11#include "MissingFrameInferrer.h"
12#include "Options.h"
13#include "ProfileGenerator.h"
14#include "llvm/ADT/StringExtras.h"
15#include "llvm/BinaryFormat/Magic.h"
16#include "llvm/DebugInfo/PDB/IPDBSession.h"
17#include "llvm/DebugInfo/PDB/PDB.h"
18#include "llvm/DebugInfo/PDB/PDBSymbolFunc.h"
19#include "llvm/DebugInfo/Symbolize/SymbolizableModule.h"
20#include "llvm/Demangle/Demangle.h"
21#include "llvm/IR/DebugInfoMetadata.h"
22#include "llvm/MC/TargetRegistry.h"
23#include "llvm/Object/COFF.h"
24#include "llvm/Support/CommandLine.h"
25#include "llvm/Support/Debug.h"
26#include "llvm/Support/Format.h"
27#include "llvm/Support/TargetSelect.h"
28#include "llvm/TargetParser/Triple.h"
29#include <optional>
30
31#define DEBUG_TYPE "load-binary"
32
33namespace llvm {
34
35using namespace object;
36
37cl::opt<bool> ShowDisassemblyOnly("show-disassembly-only",
38 cl::desc("Print disassembled code."),
39 cl::cat(ProfGenCategory));
40
41cl::opt<bool> ShowSourceLocations("show-source-locations",
42 cl::desc("Print source locations."),
43 cl::cat(ProfGenCategory));
44
45cl::opt<bool> LoadFunctionFromSymbol(
46 "load-function-from-symbol", cl::init(Val: true),
47 cl::desc("Gather additional binary function info from symbols (e.g. "
48 "symtab) in case dwarf info is incomplete."),
49 cl::cat(ProfGenCategory));
50
51static cl::opt<bool>
52 ShowCanonicalFnName("show-canonical-fname",
53 cl::desc("Print canonical function name."),
54 cl::cat(ProfGenCategory));
55
56static cl::opt<bool> ShowPseudoProbe(
57 "show-pseudo-probe",
58 cl::desc("Print pseudo probe section and disassembled info."),
59 cl::cat(ProfGenCategory));
60
61static cl::opt<bool> UseDwarfCorrelation(
62 "use-dwarf-correlation",
63 cl::desc("Use dwarf for profile correlation even when binary contains "
64 "pseudo probe."),
65 cl::cat(ProfGenCategory));
66
67static cl::opt<std::string>
68 DWPPath("dwp", cl::init(Val: ""),
69 cl::desc("Path of .dwp file. When not specified, it will be "
70 "<binary>.dwp in the same directory as the main binary."),
71 cl::cat(ProfGenCategory));
72
73static cl::list<std::string> DisassembleFunctions(
74 "disassemble-functions", cl::CommaSeparated,
75 cl::desc("List of functions to print disassembly for. Accept demangled "
76 "names only. Only work with show-disassembly-only"),
77 cl::cat(ProfGenCategory));
78
79static cl::opt<bool>
80 KernelBinary("kernel",
81 cl::desc("Generate the profile for Linux kernel binary."),
82 cl::cat(ProfGenCategory));
83
84static cl::opt<bool>
85 WarnNotSymbolized("warn-not-symbolized", cl::init(Val: false),
86 cl::desc("Generate warnings for unsymbolized addresses"),
87 cl::cat(ProfGenCategory));
88
89namespace sampleprof {
90
91// Internal suffixes which are not reflected in the source code.
92static constexpr StringRef CanonicalSuffixes[] = {
93 // Internal suffixes from CoroSplit pass
94 ".cleanup", ".destroy", ".resume",
95 // Internal suffixes from Bolt
96 ".cold", ".warm",
97 // Compiler/LTO internal
98 ".llvm.", ".part.", ".isra.", ".constprop.", ".lto_priv."};
99static const StringRef CoroSuffixes[] = {".cleanup", ".destroy", ".resume"};
100
101static const Target *getTarget(const ObjectFile *Obj) {
102 Triple TheTriple = Obj->makeTriple();
103 std::string Error;
104 std::string ArchName;
105 const Target *TheTarget =
106 TargetRegistry::lookupTarget(ArchName, TheTriple, Error);
107 if (!TheTarget)
108 exitWithError(Message: Error, Whence: Obj->getFileName());
109 return TheTarget;
110}
111
112void BinarySizeContextTracker::addInstructionForContext(
113 const SampleContextFrameVector &Context, uint32_t InstrSize) {
114 ContextTrieNode *CurNode = &RootContext;
115 bool IsLeaf = true;
116 for (const auto &Callsite : reverse(C: Context)) {
117 FunctionId CallerName = Callsite.Func;
118 LineLocation CallsiteLoc = IsLeaf ? LineLocation(0, 0) : Callsite.Location;
119 CurNode = CurNode->getOrCreateChildContext(CallSite: CallsiteLoc, ChildName: CallerName);
120 IsLeaf = false;
121 }
122
123 CurNode->addFunctionSize(FSize: InstrSize);
124}
125
126uint32_t
127BinarySizeContextTracker::getFuncSizeForContext(const ContextTrieNode *Node) {
128 ContextTrieNode *CurrNode = &RootContext;
129 ContextTrieNode *PrevNode = nullptr;
130
131 std::optional<uint32_t> Size;
132
133 // Start from top-level context-less function, traverse down the reverse
134 // context trie to find the best/longest match for given context, then
135 // retrieve the size.
136 LineLocation CallSiteLoc(0, 0);
137 while (CurrNode && Node->getParentContext() != nullptr) {
138 PrevNode = CurrNode;
139 CurrNode = CurrNode->getChildContext(CallSite: CallSiteLoc, ChildName: Node->getFuncName());
140 if (CurrNode && CurrNode->getFunctionSize())
141 Size = *CurrNode->getFunctionSize();
142 CallSiteLoc = Node->getCallSiteLoc();
143 Node = Node->getParentContext();
144 }
145
146 // If we traversed all nodes along the path of the context and haven't
147 // found a size yet, pivot to look for size from sibling nodes, i.e size
148 // of inlinee under different context.
149 if (!Size) {
150 if (!CurrNode)
151 CurrNode = PrevNode;
152 while (!Size && CurrNode && !CurrNode->getAllChildContext().empty()) {
153 CurrNode = &CurrNode->getAllChildContext().begin()->second;
154 if (CurrNode->getFunctionSize())
155 Size = *CurrNode->getFunctionSize();
156 }
157 }
158
159 assert(Size && "We should at least find one context size.");
160 return *Size;
161}
162
163void BinarySizeContextTracker::trackInlineesOptimizedAway(
164 MCPseudoProbeDecoder &ProbeDecoder) {
165 ProbeFrameStack ProbeContext;
166 for (const auto &Child : ProbeDecoder.getDummyInlineRoot().getChildren())
167 trackInlineesOptimizedAway(ProbeDecoder, ProbeNode: Child, Context&: ProbeContext);
168}
169
170void BinarySizeContextTracker::trackInlineesOptimizedAway(
171 MCPseudoProbeDecoder &ProbeDecoder,
172 const MCDecodedPseudoProbeInlineTree &ProbeNode,
173 ProbeFrameStack &ProbeContext) {
174 StringRef FuncName =
175 ProbeDecoder.getFuncDescForGUID(GUID: ProbeNode.Guid)->FuncName;
176 ProbeContext.emplace_back(Args&: FuncName, Args: 0);
177
178 // This ProbeContext has a probe, so it has code before inlining and
179 // optimization. Make sure we mark its size as known.
180 if (!ProbeNode.getProbes().empty()) {
181 ContextTrieNode *SizeContext = &RootContext;
182 for (auto &ProbeFrame : reverse(C&: ProbeContext)) {
183 StringRef CallerName = ProbeFrame.first;
184 LineLocation CallsiteLoc(ProbeFrame.second, 0);
185 SizeContext =
186 SizeContext->getOrCreateChildContext(CallSite: CallsiteLoc,
187 ChildName: FunctionId(CallerName));
188 }
189 // Add 0 size to make known.
190 SizeContext->addFunctionSize(FSize: 0);
191 }
192
193 // DFS down the probe inline tree
194 for (const auto &ChildNode : ProbeNode.getChildren()) {
195 InlineSite Location = ChildNode.getInlineSite();
196 ProbeContext.back().second = std::get<1>(t&: Location);
197 trackInlineesOptimizedAway(ProbeDecoder, ProbeNode: ChildNode, ProbeContext);
198 }
199
200 ProbeContext.pop_back();
201}
202
203ProfiledBinary::ProfiledBinary(const cl::LibraryOptionsContext &LibraryOptions,
204 const StringRef ExeBinPath,
205 const StringRef DebugBinPath)
206 : LibraryOptions(LibraryOptions), Path(ExeBinPath),
207 DebugBinaryPath(DebugBinPath), SymbolizerOpts(getSymbolizerOpts()),
208 ProEpilogTracker(this),
209 Symbolizer(std::make_unique<symbolize::LLVMSymbolizer>(args&: SymbolizerOpts)),
210 TrackFuncContextSize(EnableCSPreInliner && UseContextCostForPreInliner) {
211 // Point to executable binary if debug info binary is not specified.
212 SymbolizerPath = DebugBinPath.empty() ? ExeBinPath : DebugBinPath;
213 if (InferMissingFrames)
214 MissingContextInferrer = std::make_unique<MissingFrameInferrer>(args: this);
215}
216
217ProfiledBinary::~ProfiledBinary() = default;
218
219void ProfiledBinary::warnNoFuncEntry() {
220 uint64_t NoFuncEntryNum = 0;
221 for (auto &F : BinaryFunctions) {
222 if (F.second.Ranges.empty())
223 continue;
224 bool hasFuncEntry = false;
225 for (auto &R : F.second.Ranges) {
226 if (FuncRange *FR = findFuncRangeForStartAddr(Address: R.first)) {
227 if (FR->IsFuncEntry) {
228 hasFuncEntry = true;
229 break;
230 }
231 }
232 }
233
234 if (!hasFuncEntry) {
235 NoFuncEntryNum++;
236 if (ShowDetailedWarning)
237 WithColor::warning()
238 << "Failed to determine function entry for " << F.first()
239 << " due to inconsistent name from symbol table and dwarf info.\n";
240 }
241 }
242 emitWarningSummary(Num: NoFuncEntryNum, Total: BinaryFunctions.size(),
243 Msg: "of functions failed to determine function entry due to "
244 "inconsistent name from symbol table and dwarf info.");
245}
246
247void ProfiledBinary::load(StringRef TripleStr) {
248 // Attempt to open the binary.
249 OBinary = unwrapOrError(EO: createBinary(Path), Args&: Path);
250 Binary &ExeBinary = *OBinary.getBinary();
251
252 IsCOFF = isa<COFFObjectFile>(Val: &ExeBinary);
253 if (!isa<ELFObjectFileBase>(Val: &ExeBinary) && !IsCOFF)
254 exitWithError(Message: "not a valid ELF/COFF image", Whence: Path);
255
256 auto *Obj = cast<ObjectFile>(Val: &ExeBinary);
257 if (!TripleStr.empty())
258 TheTriple = Triple(TripleStr);
259 else
260 TheTriple = Obj->makeTriple();
261
262 LLVM_DEBUG(dbgs() << "Loading " << Path << "\n");
263
264 // Mark the binary as a kernel image;
265 IsKernel = KernelBinary;
266
267 // Find the preferred load address for text sections.
268 setPreferredTextSegmentAddresses(Obj);
269
270 // For shared libraries, read build ID to filter perfscript addresses
271 // in [buildid:]addr format. Main executables (including PIE) use empty
272 // FilterBuildID since their addresses have no buildid prefix.
273 // Both PIE executables and shared libraries are ET_DYN, but only PIE
274 // executables have a PT_INTERP program header.
275 file_magic Magic;
276 if (auto EC = identify_magic(path: Path, result&: Magic);
277 !EC && Magic == file_magic::elf_shared_object && !HasInterp) {
278 auto BID = object::getBuildID(Obj);
279 if (!BID.empty())
280 FilterBuildID = llvm::toHex(Input: BID, /*LowerCase=*/true);
281 }
282
283 // Load debug info of subprograms from DWARF section.
284 // If path of debug info binary is specified, use the debug info from it,
285 // otherwise use the debug info from the executable binary.
286 OwningBinary<Binary> DebugBinary;
287 ObjectFile *PseudoProbeObj = nullptr;
288 if (!DebugBinaryPath.empty()) {
289 DebugBinary = unwrapOrError(EO: createBinary(Path: DebugBinaryPath), Args&: DebugBinaryPath);
290 ObjectFile *DebugObj = cast<ObjectFile>(Val: DebugBinary.getBinary());
291 loadSymbolsFromDWARF(Obj&: *DebugObj);
292 if (checkPseudoProbe(Obj: DebugObj, ObjPath: DebugBinaryPath))
293 PseudoProbeObj = DebugObj;
294 } else {
295 loadSymbolsFromDWARF(Obj&: *Obj);
296 }
297
298 // Prefer loading pseudo probe from binary.
299 if (checkPseudoProbe(Obj, ObjPath: Path))
300 PseudoProbeObj = Obj;
301
302 DisassembleFunctionSet.insert_range(R&: DisassembleFunctions);
303
304 if (usePseudoProbes())
305 populateSymbolAddressList(O: Obj);
306
307 if (ShowDisassemblyOnly && PseudoProbeObj)
308 decodePseudoProbe(Obj: PseudoProbeObj);
309
310 if (LoadFunctionFromSymbol && usePseudoProbes())
311 loadSymbolsFromSymtab(O: Obj);
312
313 // Disassemble the text sections.
314 disassemble(O: Obj);
315
316 // Use function start and return address to infer prolog and epilog
317 ProEpilogTracker.inferPrologAddresses(FuncStartAddressMap&: StartAddrToFuncRangeMap);
318 ProEpilogTracker.inferEpilogAddresses(RetAddrs&: RetAddressSet);
319
320 warnNoFuncEntry();
321
322 // TODO: decode other sections.
323}
324
325bool ProfiledBinary::inlineContextEqual(uint64_t Address1, uint64_t Address2) {
326 const SampleContextFrameVector &Context1 =
327 getCachedFrameLocationStack(Address: Address1);
328 const SampleContextFrameVector &Context2 =
329 getCachedFrameLocationStack(Address: Address2);
330 if (Context1.size() != Context2.size())
331 return false;
332 if (Context1.empty())
333 return false;
334 // The leaf frame contains location within the leaf, and it
335 // needs to be remove that as it's not part of the calling context
336 return std::equal(first1: Context1.begin(), last1: Context1.begin() + Context1.size() - 1,
337 first2: Context2.begin(), last2: Context2.begin() + Context2.size() - 1);
338}
339
340SampleContextFrameVector
341ProfiledBinary::getExpandedContext(const SmallVectorImpl<uint64_t> &Stack,
342 bool &WasLeafInlined) {
343 SampleContextFrameVector ContextVec;
344 if (Stack.empty())
345 return ContextVec;
346 // Process from frame root to leaf
347 for (auto Address : Stack) {
348 const SampleContextFrameVector &ExpandedContext =
349 getCachedFrameLocationStack(Address);
350 // An instruction without a valid debug line will be ignored by sample
351 // processing
352 if (ExpandedContext.empty())
353 return SampleContextFrameVector();
354 // Set WasLeafInlined to the size of inlined frame count for the last
355 // address which is leaf
356 WasLeafInlined = (ExpandedContext.size() > 1);
357 ContextVec.append(RHS: ExpandedContext);
358 }
359
360 // Replace with decoded base discriminator
361 for (auto &Frame : ContextVec) {
362 Frame.Location.Discriminator = ProfileGeneratorBase::getBaseDiscriminator(
363 Discriminator: Frame.Location.Discriminator, UseFSD: UseFSDiscriminator);
364 }
365
366 assert(ContextVec.size() && "Context length should be at least 1");
367
368 // Compress the context string except for the leaf frame
369 auto LeafFrame = ContextVec.back();
370 LeafFrame.Location = LineLocation(0, 0);
371 ContextVec.pop_back();
372 CSProfileGenerator::compressRecursionContext(Context&: ContextVec);
373 CSProfileGenerator::trimContext(S&: ContextVec);
374 ContextVec.push_back(Elt: LeafFrame);
375 return ContextVec;
376}
377
378template <class ELFT>
379void ProfiledBinary::setPreferredTextSegmentAddresses(const ELFFile<ELFT> &Obj,
380 StringRef FileName) {
381 const auto &PhdrRange = unwrapOrError(Obj.program_headers(), FileName);
382 bool SeenFirstLoadableSegment = false;
383 for (const typename ELFT::Phdr &Phdr : PhdrRange) {
384 if (Phdr.p_type == ELF::PT_INTERP)
385 HasInterp = true;
386 if (Phdr.p_type == ELF::PT_LOAD) {
387 if (!SeenFirstLoadableSegment) {
388 // Derive the preferred address corresponding to file offset zero
389 // without assuming a page size.
390 FirstLoadableAddress = Phdr.p_vaddr - Phdr.p_offset;
391 SeenFirstLoadableSegment = true;
392 }
393 if (Phdr.p_flags & ELF::PF_X) {
394 PreferredTextSegmentAddresses.push_back(Phdr.p_vaddr);
395 TextSegmentOffsets.push_back(Phdr.p_offset);
396 } else {
397 PhdrInfo Info;
398 Info.FileOffset = Phdr.p_offset;
399 Info.FileSz = Phdr.p_filesz;
400 Info.VirtualAddr = Phdr.p_vaddr;
401 NonTextPhdrInfo.push_back(Elt: Info);
402 }
403 }
404 }
405
406 if (PreferredTextSegmentAddresses.empty())
407 exitWithError(Message: "no executable segment found", Whence: FileName);
408}
409
410uint64_t ProfiledBinary::CanonicalizeNonTextAddress(uint64_t Address) {
411 uint64_t FileOffset = 0;
412 auto MMapIter = NonTextMMapEvents.lower_bound(x: Address);
413 if (MMapIter == NonTextMMapEvents.end())
414 return Address; // No non-text mmap event found, return the address as is.
415
416 const auto &MMapEvent = MMapIter->second;
417
418 // If the address is within the non-text mmap event, calculate its file
419 // offset in the binary.
420 if (MMapEvent.Address <= Address &&
421 Address < MMapEvent.Address + MMapEvent.Size)
422 FileOffset = Address - MMapEvent.Address + MMapEvent.Offset;
423
424 // If the address is not within the non-text mmap event, return the address
425 // as is.
426 if (FileOffset == 0)
427 return Address;
428
429 for (const auto &PhdrInfo : NonTextPhdrInfo) {
430 // Find the program section that contains the file offset and map the
431 // file offset to the virtual address.
432 if (PhdrInfo.FileOffset <= FileOffset &&
433 FileOffset < PhdrInfo.FileOffset + PhdrInfo.FileSz)
434 return PhdrInfo.VirtualAddr + (FileOffset - PhdrInfo.FileOffset);
435 }
436
437 return Address;
438}
439
440void ProfiledBinary::setPreferredTextSegmentAddresses(const COFFObjectFile *Obj,
441 StringRef FileName) {
442 uint64_t ImageBase = Obj->getImageBase();
443 if (!ImageBase)
444 exitWithError(Message: "Not a COFF image", Whence: FileName);
445
446 PreferredTextSegmentAddresses.push_back(x: ImageBase);
447 FirstLoadableAddress = ImageBase;
448
449 for (SectionRef Section : Obj->sections()) {
450 const coff_section *Sec = Obj->getCOFFSection(Section);
451 if (Sec->Characteristics & COFF::IMAGE_SCN_CNT_CODE)
452 TextSegmentOffsets.push_back(x: Sec->VirtualAddress);
453 }
454}
455
456void ProfiledBinary::setPreferredTextSegmentAddresses(const ObjectFile *Obj) {
457 if (const auto *ELFObj = dyn_cast<ELF32LEObjectFile>(Val: Obj))
458 setPreferredTextSegmentAddresses(Obj: ELFObj->getELFFile(), FileName: Obj->getFileName());
459 else if (const auto *ELFObj = dyn_cast<ELF32BEObjectFile>(Val: Obj))
460 setPreferredTextSegmentAddresses(Obj: ELFObj->getELFFile(), FileName: Obj->getFileName());
461 else if (const auto *ELFObj = dyn_cast<ELF64LEObjectFile>(Val: Obj))
462 setPreferredTextSegmentAddresses(Obj: ELFObj->getELFFile(), FileName: Obj->getFileName());
463 else if (const auto *ELFObj = dyn_cast<ELF64BEObjectFile>(Val: Obj))
464 setPreferredTextSegmentAddresses(Obj: ELFObj->getELFFile(), FileName: Obj->getFileName());
465 else if (const auto *COFFObj = dyn_cast<COFFObjectFile>(Val: Obj))
466 setPreferredTextSegmentAddresses(Obj: COFFObj, FileName: Obj->getFileName());
467 else
468 llvm_unreachable("invalid object format");
469}
470
471bool ProfiledBinary::checkPseudoProbe(const ObjectFile *Obj,
472 StringRef ObjPath) {
473 if (UseDwarfCorrelation)
474 return false;
475
476 bool HasProbeDescSection = false;
477 bool HasPseudoProbeSection = false;
478
479 StringRef FileName = Obj->getFileName();
480 for (section_iterator SI = Obj->section_begin(), SE = Obj->section_end();
481 SI != SE; ++SI) {
482 const SectionRef &Section = *SI;
483 StringRef SectionName = unwrapOrError(EO: Section.getName(), Args&: FileName);
484 if (SectionName == ".pseudo_probe_desc") {
485 HasProbeDescSection = true;
486 } else if (SectionName == ".pseudo_probe") {
487 HasPseudoProbeSection = true;
488 }
489 }
490
491 if (HasProbeDescSection && HasPseudoProbeSection) {
492 PseudoProbeBinPath = ObjPath;
493 return true;
494 }
495
496 return false;
497}
498
499void ProfiledBinary::decodePseudoProbe(const ObjectFile *Obj) {
500 if (!usePseudoProbes())
501 return;
502
503 LLVM_DEBUG(dbgs() << "Decoding pseudo probe in " << Obj->getFileName()
504 << "\n");
505
506 MCPseudoProbeDecoder::Uint64Set GuidFilter;
507 MCPseudoProbeDecoder::Uint64Map FuncStartAddresses;
508 if (ShowDisassemblyOnly) {
509 if (DisassembleFunctionSet.empty()) {
510 FuncStartAddresses = SymbolStartAddrs;
511 } else {
512 for (auto &F : DisassembleFunctionSet) {
513 auto GUID = Function::getGUIDAssumingExternalLinkage(GlobalName: F.first());
514 if (auto StartAddr = SymbolStartAddrs.lookup(Val: GUID)) {
515 FuncStartAddresses[GUID] = StartAddr;
516 FuncRange &Range = StartAddrToFuncRangeMap[StartAddr];
517 GuidFilter.insert(
518 V: Function::getGUIDAssumingExternalLinkage(GlobalName: Range.getFuncName()));
519 }
520 }
521 }
522 } else {
523 for (auto *F : ProfiledFunctions) {
524 GuidFilter.insert(V: Function::getGUIDAssumingExternalLinkage(GlobalName: F->FuncName));
525 // DWARF name might be broken when a DWARF32 .debug_str.dwo section
526 // execeeds 4GB. We expect symbol table to contain the correct function
527 // names which matches the pseudo probe. Adding back all the GUIDs if
528 // possible.
529 auto AltGUIDs = AlternativeFunctionGUIDs.equal_range(x: F);
530 for (const auto &[_, Func] : make_range(p: AltGUIDs))
531 GuidFilter.insert(V: Func);
532 for (auto &Range : F->Ranges) {
533 auto GUIDs = StartAddrToSymMap.equal_range(x: Range.first);
534 for (const auto &[StartAddr, Func] : make_range(p: GUIDs))
535 FuncStartAddresses[Func] = StartAddr;
536 }
537 }
538 }
539
540 StringRef FileName = Obj->getFileName();
541 for (section_iterator SI = Obj->section_begin(), SE = Obj->section_end();
542 SI != SE; ++SI) {
543 const SectionRef &Section = *SI;
544 StringRef SectionName = unwrapOrError(EO: Section.getName(), Args&: FileName);
545
546 if (SectionName == ".pseudo_probe_desc") {
547 StringRef Contents = unwrapOrError(EO: Section.getContents(), Args&: FileName);
548 if (!ProbeDecoder.buildGUID2FuncDescMap(
549 Start: reinterpret_cast<const uint8_t *>(Contents.data()),
550 Size: Contents.size(), /*IsMMapped=*/false, VerboseWarnings: ShowDetailedWarning))
551 exitWithError(
552 Message: "Pseudo Probe decoder fail in .pseudo_probe_desc section");
553 } else if (SectionName == ".pseudo_probe") {
554 StringRef Contents = unwrapOrError(EO: Section.getContents(), Args&: FileName);
555 if (!ProbeDecoder.buildAddress2ProbeMap(
556 Start: reinterpret_cast<const uint8_t *>(Contents.data()),
557 Size: Contents.size(), GuildFilter: GuidFilter, FuncStartAddrs: FuncStartAddresses))
558 exitWithError(Message: "Pseudo Probe decoder fail in .pseudo_probe section");
559 }
560 }
561
562 // Build TopLevelProbeFrameMap to track size for optimized inlinees when probe
563 // is available
564 if (TrackFuncContextSize) {
565 for (auto &Child : ProbeDecoder.getDummyInlineRoot().getChildren()) {
566 auto *Frame = &Child;
567 StringRef FuncName =
568 ProbeDecoder.getFuncDescForGUID(GUID: Frame->Guid)->FuncName;
569 TopLevelProbeFrameMap[FuncName] = Frame;
570 }
571 }
572
573 if (ShowPseudoProbe)
574 ProbeDecoder.printGUID2FuncDescMap(OS&: outs());
575}
576
577void ProfiledBinary::decodePseudoProbe() {
578 OwningBinary<Binary> OBinary =
579 unwrapOrError(EO: createBinary(Path: PseudoProbeBinPath), Args&: PseudoProbeBinPath);
580 auto *Obj = cast<ObjectFile>(Val: OBinary.getBinary());
581 decodePseudoProbe(Obj);
582}
583
584void ProfiledBinary::setIsFuncEntry(FuncRange *FuncRange,
585 StringRef RangeSymName) {
586 // Skip external function symbol.
587 if (!FuncRange)
588 return;
589
590 // Set IsFuncEntry to ture if there is only one range in the function or the
591 // RangeSymName from ELF is equal to its DWARF-based function name.
592 if (FuncRange->Func->Ranges.size() == 1 ||
593 (!FuncRange->IsFuncEntry &&
594 (FuncRange->getFuncName() == RangeSymName ||
595 FuncRange->Func->NameStatus != DwarfNameStatus::Matched)))
596 FuncRange->IsFuncEntry = true;
597}
598
599bool ProfiledBinary::dissassembleSymbol(std::size_t SI, ArrayRef<uint8_t> Bytes,
600 SectionSymbolsTy &Symbols,
601 const SectionRef &Section) {
602 std::size_t SE = Symbols.size();
603 uint64_t SectionAddress = Section.getAddress();
604 uint64_t SectSize = Section.getSize();
605 uint64_t StartAddress = Symbols[SI].Addr;
606 uint64_t NextStartAddress =
607 (SI + 1 < SE) ? Symbols[SI + 1].Addr : SectionAddress + SectSize;
608 FuncRange *FRange = findFuncRange(Address: StartAddress);
609 setIsFuncEntry(FuncRange: FRange, RangeSymName: FunctionSamples::getCanonicalFnName(FnName: Symbols[SI].Name));
610 StringRef SymbolName =
611 ShowCanonicalFnName
612 ? FunctionSamples::getCanonicalFnName(FnName: Symbols[SI].Name)
613 : Symbols[SI].Name;
614 bool ShowDisassembly =
615 ShowDisassemblyOnly && (DisassembleFunctionSet.empty() ||
616 DisassembleFunctionSet.count(Key: SymbolName));
617 if (ShowDisassembly)
618 outs() << '<' << SymbolName << ">:\n";
619
620 uint64_t Address = StartAddress;
621 // Size of a consecutive invalid instruction range starting from Address -1
622 // backwards.
623 uint64_t InvalidInstLength = 0;
624 while (Address < NextStartAddress) {
625 MCInst Inst;
626 uint64_t Size;
627 // Disassemble an instruction.
628 bool Disassembled = DisAsm->getInstruction(
629 Instr&: Inst, Size, Bytes: Bytes.slice(N: Address - SectionAddress), Address, CStream&: nulls());
630 if (Size == 0)
631 Size = 1;
632
633 if (ShowDisassembly) {
634 if (ShowPseudoProbe) {
635 ProbeDecoder.printProbeForAddress(OS&: outs(), Address);
636 }
637 outs() << format(Fmt: "%8" PRIx64 ":", Vals: Address);
638 size_t Start = outs().tell();
639 if (Disassembled)
640 IPrinter->printInst(MI: &Inst, Address: Address + Size, Annot: "", STI: *STI, OS&: outs());
641 else
642 outs() << "\t<unknown>";
643 if (ShowSourceLocations) {
644 unsigned Cur = outs().tell() - Start;
645 if (Cur < 40)
646 outs().indent(NumSpaces: 40 - Cur);
647 InstructionPointer IP(this, Address);
648 outs() << getReversedLocWithContext(
649 Context: symbolize(IP, UseCanonicalFnName: ShowCanonicalFnName, UseProbeDiscriminator: ShowPseudoProbe));
650 }
651 outs() << "\n";
652 }
653
654 if (Disassembled) {
655 const MCInstrDesc &MCDesc = MII->get(Opcode: Inst.getOpcode());
656
657 // Record instruction size.
658 AddressToInstSizeMap[Address] = Size;
659
660 // Populate address maps.
661 CodeAddressVec.push_back(x: Address);
662 if (MCDesc.isCall()) {
663 CallAddressSet.insert(V: Address);
664 UncondBranchAddrSet.insert(x: Address);
665 // Record the instruction after call as the branch target of a ret
666 BranchTargetAddressSet.insert(V: Address + Size);
667 } else if (MCDesc.isReturn()) {
668 RetAddressSet.insert(V: Address);
669 UncondBranchAddrSet.insert(x: Address);
670 } else if (MCDesc.isBranch()) {
671 if (MCDesc.isUnconditionalBranch())
672 UncondBranchAddrSet.insert(x: Address);
673 BranchAddressSet.insert(V: Address);
674 }
675
676 if (MCDesc.isIndirectBranch()) {
677 IndirectBranchAddressSet.insert(V: Address);
678 }
679
680 // Record branch target addresses for branches and calls.
681 if (MCDesc.isCall() || MCDesc.isBranch()) {
682 uint64_t Target = 0;
683 if (MIA->evaluateBranch(Inst, Addr: Address, Size, Target))
684 BranchTargetAddressSet.insert(V: Target);
685 }
686
687 // Record potential call targets for tail frame inference later-on.
688 if (InferMissingFrames && FRange) {
689 uint64_t Target = 0;
690 [[maybe_unused]] bool Err =
691 MIA->evaluateBranch(Inst, Addr: Address, Size, Target);
692 if (MCDesc.isCall()) {
693 // Indirect call targets are unknown at this point. Recording the
694 // unknown target (zero) for further LBR-based refinement.
695 MissingContextInferrer->CallEdges[Address].insert(V: Target);
696 } else if (MCDesc.isUnconditionalBranch()) {
697 assert(Err &&
698 "target should be known for unconditional direct branch");
699 // Any inter-function unconditional jump is considered tail call at
700 // this point. This is not 100% accurate and could further be
701 // optimized based on some source annotation.
702 FuncRange *ToFRange = findFuncRange(Address: Target);
703 if (ToFRange && ToFRange->Func != FRange->Func)
704 MissingContextInferrer->TailCallEdges[Address].insert(V: Target);
705 LLVM_DEBUG({
706 dbgs() << "Direct Tail call: " << format("%8" PRIx64 ":", Address);
707 IPrinter->printInst(&Inst, Address + Size, "", *STI.get(), dbgs());
708 dbgs() << "\n";
709 });
710 } else if (MCDesc.isIndirectBranch() && MCDesc.isBarrier()) {
711 // This is an indirect branch but not necessarily an indirect tail
712 // call. The isBarrier check is to filter out conditional branch.
713 // Similar with indirect call targets, recording the unknown target
714 // (zero) for further LBR-based refinement.
715 MissingContextInferrer->TailCallEdges[Address].insert(V: Target);
716 LLVM_DEBUG({
717 dbgs() << "Indirect Tail call: "
718 << format("%8" PRIx64 ":", Address);
719 IPrinter->printInst(&Inst, Address + Size, "", *STI.get(), dbgs());
720 dbgs() << "\n";
721 });
722 }
723 }
724
725 if (InvalidInstLength) {
726 AddrsWithInvalidInstruction.insert(
727 V: {Address - InvalidInstLength, Address - 1});
728 InvalidInstLength = 0;
729 }
730 } else {
731 InvalidInstLength += Size;
732 }
733
734 Address += Size;
735 }
736
737 if (InvalidInstLength)
738 AddrsWithInvalidInstruction.insert(
739 V: {Address - InvalidInstLength, Address - 1});
740
741 if (ShowDisassembly)
742 outs() << "\n";
743
744 return true;
745}
746
747void ProfiledBinary::setUpDisassembler(const ObjectFile *Obj) {
748 const Target *TheTarget = getTarget(Obj);
749 StringRef FileName = Obj->getFileName();
750
751 MRI.reset(p: TheTarget->createMCRegInfo(TT: TheTriple));
752 if (!MRI)
753 exitWithError(Message: "no register info for target " + TheTriple.str(), Whence: FileName);
754
755 MCTargetOptions MCOptions;
756 MCOptions.LibraryOptions = &LibraryOptions;
757 AsmInfo.reset(p: TheTarget->createMCAsmInfo(MRI: *MRI, TheTriple, Options: MCOptions));
758 if (!AsmInfo)
759 exitWithError(Message: "no assembly info for target " + TheTriple.str(), Whence: FileName);
760
761 Expected<SubtargetFeatures> Features = Obj->getFeatures();
762 if (!Features)
763 exitWithError(E: Features.takeError(), Whence: FileName);
764 // AArch64 object files do not generally carry complete ISA feature metadata,
765 // so the subtarget would default to the baseline (Armv8.0-A) feature set.
766 // That disassembler cannot decode feature-gated instructions (LSE atomics,
767 // RCPC loads, SVE, ...) that are pervasive in modern AArch64 binaries; they
768 // would be miscounted as "invalid instructions" and, worse, their addresses
769 // would be absent from the code/branch maps used for sample attribution.
770 // Enable all instructions so the disassembler recognizes whatever the
771 // compiler emitted, matching llvm-objdump's default for AArch64.
772 if (TheTriple.isAArch64())
773 Features->AddFeature(String: "+all");
774 STI.reset(
775 p: TheTarget->createMCSubtargetInfo(TheTriple, CPU: "", Features: Features->getString()));
776 if (!STI)
777 exitWithError(Message: "no subtarget info for target " + TheTriple.str(), Whence: FileName);
778
779 MII.reset(p: TheTarget->createMCInstrInfo());
780 if (!MII)
781 exitWithError(Message: "no instruction info for target " + TheTriple.str(),
782 Whence: FileName);
783
784 MCContext Ctx(TheTriple, *AsmInfo, *MRI, *STI);
785 std::unique_ptr<MCObjectFileInfo> MOFI(
786 TheTarget->createMCObjectFileInfo(Ctx, /*PIC=*/false));
787 Ctx.setObjectFileInfo(MOFI.get());
788 DisAsm.reset(p: TheTarget->createMCDisassembler(STI: *STI, Ctx));
789 if (!DisAsm)
790 exitWithError(Message: "no disassembler for target " + TheTriple.str(), Whence: FileName);
791
792 MIA.reset(p: TheTarget->createMCInstrAnalysis(Info: MII.get()));
793
794 int AsmPrinterVariant = AsmInfo->getAssemblerDialect();
795 IPrinter.reset(p: TheTarget->createMCInstPrinter(T: TheTriple, SyntaxVariant: AsmPrinterVariant,
796 MAI: *AsmInfo, MII: *MII, MRI: *MRI));
797 IPrinter->setPrintBranchImmAsAddress(true);
798}
799
800void ProfiledBinary::disassemble(const ObjectFile *Obj) {
801 // Set up disassembler and related components.
802 setUpDisassembler(Obj);
803
804 // Create a mapping from virtual address to symbol name. The symbols in text
805 // sections are the candidates to dissassemble.
806 std::map<SectionRef, SectionSymbolsTy> AllSymbols;
807 StringRef FileName = Obj->getFileName();
808 for (const SymbolRef &Symbol : Obj->symbols()) {
809 const uint64_t Addr = unwrapOrError(EO: Symbol.getAddress(), Args&: FileName);
810 const StringRef Name = unwrapOrError(EO: Symbol.getName(), Args&: FileName);
811 section_iterator SecI = unwrapOrError(EO: Symbol.getSection(), Args&: FileName);
812 if (SecI != Obj->section_end())
813 AllSymbols[*SecI].push_back(x: SymbolInfoTy(Addr, Name, ELF::STT_NOTYPE));
814 }
815
816 // Sort all the symbols. Use a stable sort to stabilize the output.
817 for (std::pair<const SectionRef, SectionSymbolsTy> &SecSyms : AllSymbols)
818 stable_sort(Range&: SecSyms.second);
819
820 assert((DisassembleFunctionSet.empty() || ShowDisassemblyOnly) &&
821 "Functions to disassemble should be only specified together with "
822 "--show-disassembly-only");
823
824 if (ShowDisassemblyOnly)
825 outs() << "\nDisassembly of " << FileName << ":\n";
826
827 // Dissassemble a text section.
828 for (section_iterator SI = Obj->section_begin(), SE = Obj->section_end();
829 SI != SE; ++SI) {
830 const SectionRef &Section = *SI;
831 if (!Section.isText())
832 continue;
833
834 uint64_t ImageLoadAddr = getPreferredBaseAddress();
835 uint64_t SectionAddress = Section.getAddress() - ImageLoadAddr;
836 uint64_t SectSize = Section.getSize();
837 if (!SectSize)
838 continue;
839
840 // Register the text section.
841 TextSections.insert(x: {SectionAddress, SectSize});
842
843 StringRef SectionName = unwrapOrError(EO: Section.getName(), Args&: FileName);
844
845 if (ShowDisassemblyOnly) {
846 outs() << "\nDisassembly of section " << SectionName;
847 outs() << " [" << format(Fmt: "0x%" PRIx64, Vals: Section.getAddress()) << ", "
848 << format(Fmt: "0x%" PRIx64, Vals: Section.getAddress() + SectSize)
849 << "]:\n\n";
850 }
851
852 if (isa<ELFObjectFileBase>(Val: Obj) && SectionName == ".plt")
853 continue;
854
855 // Get the section data.
856 ArrayRef<uint8_t> Bytes =
857 arrayRefFromStringRef(Input: unwrapOrError(EO: Section.getContents(), Args&: FileName));
858
859 // Get the list of all the symbols in this section.
860 SectionSymbolsTy &Symbols = AllSymbols[Section];
861
862 // Disassemble symbol by symbol.
863 for (std::size_t SI = 0, SE = Symbols.size(); SI != SE; ++SI) {
864 if (!dissassembleSymbol(SI, Bytes, Symbols, Section))
865 exitWithError(Message: "disassembling error", Whence: FileName);
866 }
867 }
868
869 if (!AddrsWithInvalidInstruction.empty()) {
870 if (ShowDetailedWarning) {
871 for (auto &Addr : AddrsWithInvalidInstruction) {
872 WithColor::warning()
873 << "Invalid instructions at " << format(Fmt: "%8" PRIx64, Vals: Addr.first)
874 << " - " << format(Fmt: "%8" PRIx64, Vals: Addr.second) << "\n";
875 }
876 }
877 WithColor::warning() << "Found " << AddrsWithInvalidInstruction.size()
878 << " invalid instructions\n";
879 AddrsWithInvalidInstruction.clear();
880 }
881
882 // Dissassemble rodata section to check if FS discriminator symbol exists.
883 checkUseFSDiscriminator(Obj, AllSymbols);
884}
885
886void ProfiledBinary::checkUseFSDiscriminator(
887 const ObjectFile *Obj, std::map<SectionRef, SectionSymbolsTy> &AllSymbols) {
888 const char *FSDiscriminatorVar = "__llvm_fs_discriminator__";
889 for (section_iterator SI = Obj->section_begin(), SE = Obj->section_end();
890 SI != SE; ++SI) {
891 const SectionRef &Section = *SI;
892 if (!Section.isData() || Section.getSize() == 0)
893 continue;
894 SectionSymbolsTy &Symbols = AllSymbols[Section];
895
896 for (std::size_t SI = 0, SE = Symbols.size(); SI != SE; ++SI) {
897 if (Symbols[SI].Name == FSDiscriminatorVar) {
898 UseFSDiscriminator = true;
899 return;
900 }
901 }
902 }
903}
904
905void ProfiledBinary::populateSymbolAddressList(const ObjectFile *Obj) {
906 // Create a mapping from virtual address to symbol GUID and the other way
907 // around.
908 StringRef FileName = Obj->getFileName();
909 for (const SymbolRef &Symbol : Obj->symbols()) {
910 const uint64_t Addr = unwrapOrError(EO: Symbol.getAddress(), Args&: FileName);
911 const StringRef Name = unwrapOrError(EO: Symbol.getName(), Args&: FileName);
912 uint64_t GUID = Function::getGUIDAssumingExternalLinkage(GlobalName: Name);
913 SymbolStartAddrs[GUID] = Addr;
914 StartAddrToSymMap.emplace(args: Addr, args&: GUID);
915 }
916}
917
918void ProfiledBinary::loadSymbolsFromSymtab(const ObjectFile *Obj) {
919 // Load binary functions from symbol table when Debug info is incomplete.
920 StringRef FileName = Obj->getFileName();
921
922 // COFF symtab does not have size field. Try to load size from PDB instead.
923 std::unique_ptr<pdb::IPDBSession> PDBSession;
924 if (auto *COFFObj = dyn_cast<COFFObjectFile>(Val: Obj)) {
925 if (auto E = pdb::loadDataForEXE(Type: pdb::PDB_ReaderType::Native, Path: FileName,
926 Session&: PDBSession)) {
927 StringRef PdbPath;
928 const codeview::DebugInfo *PdbInfo;
929 if (auto Err = COFFObj->getDebugPDBInfo(Info&: PdbInfo, PDBFileName&: PdbPath))
930 consumeError(Err: std::move(Err));
931
932 auto Style = PdbPath.starts_with(Prefix: "/") ? sys::path::Style::posix
933 : sys::path::Style::windows;
934 WithColor::warning() << "Cannot load PDB file "
935 << sys::path::filename(path: PdbPath, style: Style) << " for "
936 << FileName << ": " << E << "\n";
937 consumeError(Err: std::move(E));
938 } else {
939 PDBSession->setLoadAddress(FirstLoadableAddress);
940 }
941 }
942
943 for (const SymbolRef &Symbol : Obj->symbols()) {
944 const SymbolRef::Type Type = unwrapOrError(EO: Symbol.getType(), Args&: FileName);
945 const uint64_t StartAddr = unwrapOrError(EO: Symbol.getAddress(), Args&: FileName);
946 const StringRef Name = unwrapOrError(EO: Symbol.getName(), Args&: FileName);
947 uint64_t Size = 0;
948 if (isa<ELFObjectFileBase>(Val: Obj)) {
949 ELFSymbolRef ElfSymbol(Symbol);
950 Size = ElfSymbol.getSize();
951 } else if (PDBSession) {
952 if (std::unique_ptr<pdb::PDBSymbol> Sym = PDBSession->findSymbolByAddress(
953 Address: StartAddr, Type: pdb::PDB_SymType::Function)) {
954 auto FuncSym = cast<pdb::PDBSymbolFunc>(Val: std::move(Sym));
955 if (StartAddr == FuncSym->getVirtualAddress())
956 Size = FuncSym->getLength();
957 }
958 }
959
960 if (Size == 0 || Type != SymbolRef::ST_Function)
961 continue;
962
963 const uint64_t EndAddr = StartAddr + Size;
964 const StringRef SymName =
965 FunctionSamples::getCanonicalFnName(FnName: Name, Suffixes: CanonicalSuffixes);
966 assert(StartAddr < EndAddr && StartAddr >= getPreferredBaseAddress() &&
967 "Function range is invalid.");
968
969 auto Range = findFuncRange(Address: StartAddr);
970 if (!Range) {
971 assert(findFuncRange(EndAddr - 1) == nullptr &&
972 "Function range overlaps with existing functions.");
973 // Function from symbol table not found previously in DWARF, store ranges.
974 auto Ret = BinaryFunctions.try_emplace(Key: SymName);
975 auto &Func = Ret.first->second;
976 if (Ret.second) {
977 Func.FuncName = Ret.first->first();
978 HashBinaryFunctions[Function::getGUIDAssumingExternalLinkage(GlobalName: SymName)] =
979 &Func;
980 }
981
982 Func.NameStatus = DwarfNameStatus::Missing;
983 Func.Ranges.emplace_back(args: StartAddr, args: EndAddr);
984
985 auto R = StartAddrToFuncRangeMap.emplace(args: StartAddr, args: FuncRange());
986 FuncRange &FRange = R.first->second;
987
988 FRange.Func = &Func;
989 FRange.StartAddress = StartAddr;
990 FRange.EndAddress = EndAddr;
991
992 } else if (SymName != Range->getFuncName()) {
993 // Function range already found from DWARF or symtab, but the symbol name
994 // from symbol table is inconsistent with the existing name associated
995 // with the range. Log this discrepancy and the alternative function GUID.
996 if (ShowDetailedWarning)
997 WithColor::warning()
998 << "Conflicting name for symbol " << Name << " with range ("
999 << format(Fmt: "%8" PRIx64, Vals: StartAddr) << ", "
1000 << format(Fmt: "%8" PRIx64, Vals: EndAddr) << ")"
1001 << ", but the existing symbol " << Range->getFuncName()
1002 << " indicates an overlapping range ("
1003 << format(Fmt: "%8" PRIx64, Vals: Range->StartAddress) << ", "
1004 << format(Fmt: "%8" PRIx64, Vals: Range->EndAddress) << ")\n";
1005
1006 assert(StartAddr == Range->StartAddress && EndAddr == Range->EndAddress &&
1007 "Mismatched function range");
1008
1009 Range->Func->NameStatus = DwarfNameStatus::Mismatch;
1010 AlternativeFunctionGUIDs.emplace(
1011 args&: Range->Func, args: Function::getGUIDAssumingExternalLinkage(GlobalName: SymName));
1012
1013 } else if (StartAddr != Range->StartAddress &&
1014 EndAddr != Range->EndAddress) {
1015 // Function already found in DWARF or symtab, but the address range from
1016 // symbol table conflicts/overlaps with the existing one.
1017 WithColor::warning() << "Conflicting range for symbol " << Name
1018 << " with range (" << format(Fmt: "%8" PRIx64, Vals: StartAddr)
1019 << ", " << format(Fmt: "%8" PRIx64, Vals: EndAddr) << ")"
1020 << ", but the existing symbol "
1021 << Range->getFuncName()
1022 << " indicates another range ("
1023 << format(Fmt: "%8" PRIx64, Vals: Range->StartAddress) << ", "
1024 << format(Fmt: "%8" PRIx64, Vals: Range->EndAddress) << ")\n";
1025 }
1026 }
1027}
1028
1029void ProfiledBinary::loadSymbolsFromDWARFUnit(DWARFUnit &CompilationUnit) {
1030 for (const auto &DieInfo : CompilationUnit.dies()) {
1031 llvm::DWARFDie Die(&CompilationUnit, &DieInfo);
1032
1033 if (!Die.isSubprogramDIE())
1034 continue;
1035 auto Name = Die.getName(Kind: llvm::DINameKind::LinkageName);
1036 if (!Name)
1037 Name = Die.getName(Kind: llvm::DINameKind::ShortName);
1038 if (!Name)
1039 continue;
1040
1041 auto CanonName = FunctionSamples::getCanonicalCoroFnName(FnName: Name);
1042 auto RangesOrError = Die.getAddressRanges();
1043 if (!RangesOrError)
1044 continue;
1045 const DWARFAddressRangesVector &Ranges = RangesOrError.get();
1046
1047 if (Ranges.empty())
1048 continue;
1049
1050 // Different DWARF symbols can have same function name, search or create
1051 // BinaryFunction indexed by the name.
1052 auto Ret = BinaryFunctions.try_emplace(Key: CanonName);
1053 auto &Func = Ret.first->second;
1054 if (Ret.second)
1055 Func.FuncName = Ret.first->first();
1056
1057 for (const auto &Range : Ranges) {
1058 uint64_t StartAddress = Range.LowPC;
1059 uint64_t EndAddress = Range.HighPC;
1060
1061 if (EndAddress <= StartAddress ||
1062 StartAddress < getPreferredBaseAddress())
1063 continue;
1064
1065 // We may want to know all ranges for one function. Here group the
1066 // ranges and store them into BinaryFunction.
1067 Func.Ranges.emplace_back(args&: StartAddress, args&: EndAddress);
1068
1069 auto R = StartAddrToFuncRangeMap.emplace(args&: StartAddress, args: FuncRange());
1070 if (R.second) {
1071 FuncRange &FRange = R.first->second;
1072 FRange.Func = &Func;
1073 FRange.StartAddress = StartAddress;
1074 FRange.EndAddress = EndAddress;
1075 } else {
1076 AddrsWithMultipleSymbols.insert(V: StartAddress);
1077 if (ShowDetailedWarning)
1078 WithColor::warning()
1079 << "Duplicated symbol start address at "
1080 << format(Fmt: "%8" PRIx64, Vals: StartAddress) << " "
1081 << R.first->second.getFuncName() << " and " << Name << "\n";
1082 }
1083 }
1084 }
1085}
1086
1087void ProfiledBinary::loadSymbolsFromDWARF(ObjectFile &Obj) {
1088 auto DebugContext = llvm::DWARFContext::create(
1089 Obj, RelocAction: DWARFContext::ProcessDebugRelocations::Process, L: nullptr, DWPName: DWPPath);
1090 if (!DebugContext)
1091 exitWithError(Message: "Error creating the debug info context", Whence: Path);
1092
1093 for (const auto &CompilationUnit : DebugContext->compile_units())
1094 loadSymbolsFromDWARFUnit(CompilationUnit&: *CompilationUnit);
1095
1096 // Handles DWO sections that can either be in .o, .dwo or .dwp files.
1097 uint32_t NumOfDWOMissing = 0;
1098 for (const auto &CompilationUnit : DebugContext->compile_units()) {
1099 DWARFUnit *const DwarfUnit = CompilationUnit.get();
1100 if (DwarfUnit->getDWOId()) {
1101 DWARFUnit *DWOCU = DwarfUnit->getNonSkeletonUnitDIE(ExtractUnitDIEOnly: false).getDwarfUnit();
1102 if (!DWOCU->isDWOUnit()) {
1103 NumOfDWOMissing++;
1104 if (ShowDetailedWarning) {
1105 std::string DWOName = dwarf::toString(
1106 V: DwarfUnit->getUnitDIE().find(
1107 Attrs: {dwarf::DW_AT_dwo_name, dwarf::DW_AT_GNU_dwo_name}),
1108 Default: "");
1109 WithColor::warning() << "DWO debug information for " << DWOName
1110 << " was not loaded.\n";
1111 }
1112 continue;
1113 }
1114 loadSymbolsFromDWARFUnit(CompilationUnit&: *DWOCU);
1115 }
1116 }
1117
1118 if (NumOfDWOMissing)
1119 WithColor::warning()
1120 << " DWO debug information was not loaded for " << NumOfDWOMissing
1121 << " modules. Please check the .o, .dwo or .dwp path.\n";
1122 if (BinaryFunctions.empty())
1123 WithColor::warning() << "Loading of DWARF info completed, but no binary "
1124 "functions have been retrieved.\n";
1125 // Populate the hash binary function map for MD5 function name lookup. This
1126 // is done after BinaryFunctions are finalized.
1127 for (auto &BinaryFunction : BinaryFunctions) {
1128 HashBinaryFunctions[MD5Hash(Str: BinaryFunction.first())] =
1129 &BinaryFunction.second;
1130 }
1131
1132 if (!AddrsWithMultipleSymbols.empty()) {
1133 WithColor::warning() << "Found " << AddrsWithMultipleSymbols.size()
1134 << " start addresses with multiple symbols\n";
1135 AddrsWithMultipleSymbols.clear();
1136 }
1137}
1138
1139void ProfiledBinary::populateSymbolListFromDWARF(
1140 ProfileSymbolList &SymbolList) {
1141 for (auto &I : StartAddrToFuncRangeMap)
1142 SymbolList.add(Name: I.second.getFuncName());
1143}
1144
1145symbolize::LLVMSymbolizer::Options ProfiledBinary::getSymbolizerOpts() const {
1146 symbolize::LLVMSymbolizer::Options SymbolizerOpts;
1147 SymbolizerOpts.PrintFunctions =
1148 DILineInfoSpecifier::FunctionNameKind::LinkageName;
1149 SymbolizerOpts.Demangle = false;
1150 SymbolizerOpts.DefaultArch = TheTriple.getArchName().str();
1151 SymbolizerOpts.UseSymbolTable = false;
1152 SymbolizerOpts.RelativeAddresses = false;
1153 SymbolizerOpts.DWPName = DWPPath;
1154 return SymbolizerOpts;
1155}
1156
1157SampleContextFrameVector ProfiledBinary::symbolize(const InstructionPointer &IP,
1158 bool UseCanonicalFnName,
1159 bool UseProbeDiscriminator) {
1160 assert(this == IP.Binary &&
1161 "Binary should only symbolize its own instruction");
1162 DIInliningInfo InlineStack =
1163 unwrapOrError(EO: Symbolizer->symbolizeInlinedCode(
1164 ModuleName: SymbolizerPath.str(), ModuleOffset: getSectionedAddress(Address: IP.Address)),
1165 Args&: SymbolizerPath);
1166
1167 SampleContextFrameVector CallStack;
1168 for (int32_t I = InlineStack.getNumberOfFrames() - 1; I >= 0; I--) {
1169 const auto &CallerFrame = InlineStack.getFrame(Index: I);
1170 if (CallerFrame.FunctionName.empty() ||
1171 (CallerFrame.FunctionName == "<invalid>"))
1172 break;
1173
1174 StringRef FunctionName(CallerFrame.FunctionName);
1175 if (UseCanonicalFnName)
1176 FunctionName =
1177 FunctionSamples::getCanonicalFnName(FnName: FunctionName, Suffixes: CanonicalSuffixes);
1178 else
1179 FunctionName =
1180 FunctionSamples::getCanonicalFnName(FnName: FunctionName, Suffixes: CoroSuffixes);
1181
1182 uint32_t Discriminator = CallerFrame.Discriminator;
1183 uint32_t LineOffset = (CallerFrame.Line - CallerFrame.StartLine) & 0xffff;
1184 if (UseProbeDiscriminator) {
1185 LineOffset =
1186 PseudoProbeDwarfDiscriminator::extractProbeIndex(Value: Discriminator);
1187 Discriminator = 0;
1188 }
1189
1190 LineLocation Line(LineOffset, Discriminator);
1191 auto It = NameStrings.insert(key: FunctionName);
1192 CallStack.emplace_back(Args: FunctionId(It.first->getKey()), Args&: Line);
1193 }
1194
1195 if (WarnNotSymbolized && CallStack.empty()) {
1196 uint64_t VAddr = IP.Address + BaseAddress - getPreferredBaseAddress();
1197 if (isVaddrMMapped(VAddr))
1198 WithColor::warning() << "Failed to symbolize address "
1199 << format(Fmt: "%8" PRIx64, Vals: IP.Address)
1200 << " (vaddr=" << format(Fmt: "%8" PRIx64, Vals: VAddr) << ")\n";
1201 }
1202
1203 return CallStack;
1204}
1205
1206StringRef ProfiledBinary::symbolizeDataAddress(uint64_t Address) {
1207 DIGlobal DataDIGlobal =
1208 unwrapOrError(EO: Symbolizer->symbolizeData(ModuleName: SymbolizerPath.str(),
1209 ModuleOffset: getSectionedAddress(Address)),
1210 Args&: SymbolizerPath);
1211 return NameStrings.insert(key: DataDIGlobal.Name).first->getKey();
1212}
1213
1214void ProfiledBinary::computeInlinedContextSizeForRange(uint64_t RangeBegin,
1215 uint64_t RangeEnd) {
1216 InstructionPointer IP(this, RangeBegin, true);
1217
1218 if (IP.Address != RangeBegin)
1219 WithColor::warning() << "Invalid start instruction at "
1220 << format(Fmt: "%8" PRIx64, Vals: RangeBegin) << "\n";
1221
1222 if (IP.Address >= RangeEnd)
1223 return;
1224
1225 do {
1226 const SampleContextFrameVector SymbolizedCallStack =
1227 getFrameLocationStack(Address: IP.Address, UseProbeDiscriminator: usePseudoProbes());
1228 uint64_t Size = AddressToInstSizeMap[IP.Address];
1229 // Record instruction size for the corresponding context
1230 FuncSizeTracker.addInstructionForContext(Context: SymbolizedCallStack, InstrSize: Size);
1231
1232 } while (IP.advance() && IP.Address < RangeEnd);
1233}
1234
1235void ProfiledBinary::computeInlinedContextSizeForFunc(
1236 const BinaryFunction *Func) {
1237 // Note that a function can be spilt into multiple ranges, so compute for all
1238 // ranges of the function.
1239 for (const auto &Range : Func->Ranges)
1240 computeInlinedContextSizeForRange(RangeBegin: Range.first, RangeEnd: Range.second);
1241
1242 // Track optimized-away inlinee for probed binary. A function inlined and then
1243 // optimized away should still have their probes left over in places.
1244 if (usePseudoProbes()) {
1245 auto I = TopLevelProbeFrameMap.find(Key: Func->FuncName);
1246 if (I != TopLevelProbeFrameMap.end()) {
1247 BinarySizeContextTracker::ProbeFrameStack ProbeContext;
1248 FuncSizeTracker.trackInlineesOptimizedAway(ProbeDecoder, ProbeNode: *I->second,
1249 ProbeContext);
1250 }
1251 }
1252}
1253
1254void ProfiledBinary::loadSymbolsFromPseudoProbe() {
1255 if (!usePseudoProbes())
1256 return;
1257
1258 const AddressProbesMap &Address2ProbesMap = getAddress2ProbesMap();
1259 for (auto *Func : ProfiledFunctions) {
1260 if (Func->NameStatus != DwarfNameStatus::Mismatch)
1261 continue;
1262 for (auto &[StartAddr, EndAddr] : Func->Ranges) {
1263 auto Range = findFuncRangeForStartAddr(Address: StartAddr);
1264 if (!Range->IsFuncEntry)
1265 continue;
1266 const auto &Probe = Address2ProbesMap.find(From: StartAddr, To: EndAddr);
1267 if (Probe.begin() != Probe.end()) {
1268 const MCDecodedPseudoProbeInlineTree *InlineTreeNode =
1269 Probe.begin()->get().getInlineTreeNode();
1270 while (!InlineTreeNode->isTopLevelFunc())
1271 InlineTreeNode = static_cast<MCDecodedPseudoProbeInlineTree *>(
1272 InlineTreeNode->Parent);
1273
1274 assert(llvm::any_of(InlineTreeNode->getProbes(),
1275 [Start = StartAddr, End = EndAddr](const auto &P) {
1276 return P.getAddress() >= Start &&
1277 P.getAddress() < End;
1278 }) &&
1279 "Top level pseudo probe does not match function range");
1280
1281 const auto *ProbeDesc = getFuncDescForGUID(GUID: InlineTreeNode->Guid);
1282 auto Ret = PseudoProbeNames.try_emplace(Key: Func, Args: ProbeDesc->FuncName);
1283 if (!Ret.second && Ret.first->second != ProbeDesc->FuncName &&
1284 ShowDetailedWarning)
1285 WithColor::warning()
1286 << "Mismatched pseudo probe names in function " << Func->FuncName
1287 << " at range: (" << format(Fmt: "%8" PRIx64, Vals: StartAddr) << ", "
1288 << format(Fmt: "%8" PRIx64, Vals: EndAddr) << "). "
1289 << "The previously found pseudo probe name is "
1290 << Ret.first->second << " but it conflicts with name "
1291 << ProbeDesc->FuncName
1292 << " This likely indicates a DWARF error that produces "
1293 "conflicting symbols at the same starting address.\n";
1294 }
1295 }
1296 }
1297}
1298
1299StringRef ProfiledBinary::findPseudoProbeName(const BinaryFunction *Func) {
1300 auto ProbeName = PseudoProbeNames.find(Val: Func);
1301 if (ProbeName == PseudoProbeNames.end())
1302 return StringRef();
1303 return ProbeName->second;
1304}
1305
1306void ProfiledBinary::inferMissingFrames(
1307 const SmallVectorImpl<uint64_t> &Context,
1308 SmallVectorImpl<uint64_t> &NewContext) {
1309 MissingContextInferrer->inferMissingFrames(Context, NewContext);
1310}
1311
1312InstructionPointer::InstructionPointer(const ProfiledBinary *Binary,
1313 uint64_t Address, bool RoundToNext)
1314 : Binary(Binary), Address(Address) {
1315 Index = Binary->getIndexForAddr(Address);
1316 if (RoundToNext) {
1317 // we might get address which is not the code
1318 // it should round to the next valid address
1319 if (Index >= Binary->getCodeAddrVecSize())
1320 this->Address = UINT64_MAX;
1321 else
1322 this->Address = Binary->getAddressforIndex(Index);
1323 }
1324}
1325
1326bool InstructionPointer::advance() {
1327 Index++;
1328 if (Index >= Binary->getCodeAddrVecSize()) {
1329 Address = UINT64_MAX;
1330 return false;
1331 }
1332 Address = Binary->getAddressforIndex(Index);
1333 return true;
1334}
1335
1336bool InstructionPointer::backward() {
1337 if (Index == 0) {
1338 Address = 0;
1339 return false;
1340 }
1341 Index--;
1342 Address = Binary->getAddressforIndex(Index);
1343 return true;
1344}
1345
1346void InstructionPointer::update(uint64_t Addr) {
1347 Address = Addr;
1348 Index = Binary->getIndexForAddr(Address);
1349}
1350
1351} // end namespace sampleprof
1352} // end namespace llvm
1353