1//===-- ProfiledBinary.cpp - Binary decoder ---------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "ProfiledBinary.h"
10#include "ErrorHandling.h"
11#include "MissingFrameInferrer.h"
12#include "Options.h"
13#include "ProfileGenerator.h"
14#include "llvm/ADT/StringExtras.h"
15#include "llvm/BinaryFormat/Magic.h"
16#include "llvm/DebugInfo/PDB/IPDBSession.h"
17#include "llvm/DebugInfo/PDB/PDB.h"
18#include "llvm/DebugInfo/PDB/PDBSymbolFunc.h"
19#include "llvm/DebugInfo/Symbolize/SymbolizableModule.h"
20#include "llvm/Demangle/Demangle.h"
21#include "llvm/IR/DebugInfoMetadata.h"
22#include "llvm/MC/TargetRegistry.h"
23#include "llvm/Object/COFF.h"
24#include "llvm/Support/CommandLine.h"
25#include "llvm/Support/Debug.h"
26#include "llvm/Support/Format.h"
27#include "llvm/Support/TargetSelect.h"
28#include "llvm/TargetParser/Triple.h"
29#include <optional>
30
31#define DEBUG_TYPE "load-binary"
32
33namespace llvm {
34
35using namespace object;
36
37cl::opt<bool> ShowDisassemblyOnly("show-disassembly-only",
38 cl::desc("Print disassembled code."),
39 cl::cat(ProfGenCategory));
40
41cl::opt<bool> ShowSourceLocations("show-source-locations",
42 cl::desc("Print source locations."),
43 cl::cat(ProfGenCategory));
44
45cl::opt<bool> LoadFunctionFromSymbol(
46 "load-function-from-symbol", cl::init(Val: true),
47 cl::desc("Gather additional binary function info from symbols (e.g. "
48 "symtab) in case dwarf info is incomplete."),
49 cl::cat(ProfGenCategory));
50
51static cl::opt<bool>
52 ShowCanonicalFnName("show-canonical-fname",
53 cl::desc("Print canonical function name."),
54 cl::cat(ProfGenCategory));
55
56static cl::opt<bool> ShowPseudoProbe(
57 "show-pseudo-probe",
58 cl::desc("Print pseudo probe section and disassembled info."),
59 cl::cat(ProfGenCategory));
60
61static cl::opt<bool> UseDwarfCorrelation(
62 "use-dwarf-correlation",
63 cl::desc("Use dwarf for profile correlation even when binary contains "
64 "pseudo probe."),
65 cl::cat(ProfGenCategory));
66
67static cl::opt<std::string>
68 DWPPath("dwp", cl::init(Val: ""),
69 cl::desc("Path of .dwp file. When not specified, it will be "
70 "<binary>.dwp in the same directory as the main binary."),
71 cl::cat(ProfGenCategory));
72
73static cl::list<std::string> DisassembleFunctions(
74 "disassemble-functions", cl::CommaSeparated,
75 cl::desc("List of functions to print disassembly for. Accept demangled "
76 "names only. Only work with show-disassembly-only"),
77 cl::cat(ProfGenCategory));
78
79static cl::opt<bool>
80 KernelBinary("kernel",
81 cl::desc("Generate the profile for Linux kernel binary."),
82 cl::cat(ProfGenCategory));
83
84static cl::opt<bool>
85 WarnNotSymbolized("warn-not-symbolized", cl::init(Val: false),
86 cl::desc("Generate warnings for unsymbolized addresses"),
87 cl::cat(ProfGenCategory));
88
89namespace sampleprof {
90
91// Internal suffixes which are not reflected in the source code.
92static constexpr StringRef CanonicalSuffixes[] = {
93 // Internal suffixes from CoroSplit pass
94 ".cleanup", ".destroy", ".resume",
95 // Internal suffixes from Bolt
96 ".cold", ".warm",
97 // Compiler/LTO internal
98 ".llvm.", ".part.", ".isra.", ".constprop.", ".lto_priv."};
99static const StringRef CoroSuffixes[] = {".cleanup", ".destroy", ".resume"};
100
101static const Target *getTarget(const ObjectFile *Obj) {
102 Triple TheTriple = Obj->makeTriple();
103 std::string Error;
104 std::string ArchName;
105 const Target *TheTarget =
106 TargetRegistry::lookupTarget(ArchName, TheTriple, Error);
107 if (!TheTarget)
108 exitWithError(Message: Error, Whence: Obj->getFileName());
109 return TheTarget;
110}
111
112void BinarySizeContextTracker::addInstructionForContext(
113 const SampleContextFrameVector &Context, uint32_t InstrSize) {
114 ContextTrieNode *CurNode = &RootContext;
115 bool IsLeaf = true;
116 for (const auto &Callsite : reverse(C: Context)) {
117 FunctionId CallerName = Callsite.Func;
118 LineLocation CallsiteLoc = IsLeaf ? LineLocation(0, 0) : Callsite.Location;
119 CurNode = CurNode->getOrCreateChildContext(CallSite: CallsiteLoc, ChildName: CallerName);
120 IsLeaf = false;
121 }
122
123 CurNode->addFunctionSize(FSize: InstrSize);
124}
125
126uint32_t
127BinarySizeContextTracker::getFuncSizeForContext(const ContextTrieNode *Node) {
128 ContextTrieNode *CurrNode = &RootContext;
129 ContextTrieNode *PrevNode = nullptr;
130
131 std::optional<uint32_t> Size;
132
133 // Start from top-level context-less function, traverse down the reverse
134 // context trie to find the best/longest match for given context, then
135 // retrieve the size.
136 LineLocation CallSiteLoc(0, 0);
137 while (CurrNode && Node->getParentContext() != nullptr) {
138 PrevNode = CurrNode;
139 CurrNode = CurrNode->getChildContext(CallSite: CallSiteLoc, ChildName: Node->getFuncName());
140 if (CurrNode && CurrNode->getFunctionSize())
141 Size = *CurrNode->getFunctionSize();
142 CallSiteLoc = Node->getCallSiteLoc();
143 Node = Node->getParentContext();
144 }
145
146 // If we traversed all nodes along the path of the context and haven't
147 // found a size yet, pivot to look for size from sibling nodes, i.e size
148 // of inlinee under different context.
149 if (!Size) {
150 if (!CurrNode)
151 CurrNode = PrevNode;
152 while (!Size && CurrNode && !CurrNode->getAllChildContext().empty()) {
153 CurrNode = &CurrNode->getAllChildContext().begin()->second;
154 if (CurrNode->getFunctionSize())
155 Size = *CurrNode->getFunctionSize();
156 }
157 }
158
159 assert(Size && "We should at least find one context size.");
160 return *Size;
161}
162
163void BinarySizeContextTracker::trackInlineesOptimizedAway(
164 MCPseudoProbeDecoder &ProbeDecoder) {
165 ProbeFrameStack ProbeContext;
166 for (const auto &Child : ProbeDecoder.getDummyInlineRoot().getChildren())
167 trackInlineesOptimizedAway(ProbeDecoder, ProbeNode: Child, Context&: ProbeContext);
168}
169
170void BinarySizeContextTracker::trackInlineesOptimizedAway(
171 MCPseudoProbeDecoder &ProbeDecoder,
172 const MCDecodedPseudoProbeInlineTree &ProbeNode,
173 ProbeFrameStack &ProbeContext) {
174 StringRef FuncName =
175 ProbeDecoder.getFuncDescForGUID(GUID: ProbeNode.Guid)->FuncName;
176 ProbeContext.emplace_back(Args&: FuncName, Args: 0);
177
178 // This ProbeContext has a probe, so it has code before inlining and
179 // optimization. Make sure we mark its size as known.
180 if (!ProbeNode.getProbes().empty()) {
181 ContextTrieNode *SizeContext = &RootContext;
182 for (auto &ProbeFrame : reverse(C&: ProbeContext)) {
183 StringRef CallerName = ProbeFrame.first;
184 LineLocation CallsiteLoc(ProbeFrame.second, 0);
185 SizeContext =
186 SizeContext->getOrCreateChildContext(CallSite: CallsiteLoc,
187 ChildName: FunctionId(CallerName));
188 }
189 // Add 0 size to make known.
190 SizeContext->addFunctionSize(FSize: 0);
191 }
192
193 // DFS down the probe inline tree
194 for (const auto &ChildNode : ProbeNode.getChildren()) {
195 InlineSite Location = ChildNode.getInlineSite();
196 ProbeContext.back().second = std::get<1>(t&: Location);
197 trackInlineesOptimizedAway(ProbeDecoder, ProbeNode: ChildNode, ProbeContext);
198 }
199
200 ProbeContext.pop_back();
201}
202
203ProfiledBinary::ProfiledBinary(const StringRef ExeBinPath,
204 const StringRef DebugBinPath)
205 : Path(ExeBinPath), DebugBinaryPath(DebugBinPath),
206 SymbolizerOpts(getSymbolizerOpts()), ProEpilogTracker(this),
207 Symbolizer(std::make_unique<symbolize::LLVMSymbolizer>(args&: SymbolizerOpts)),
208 TrackFuncContextSize(EnableCSPreInliner && UseContextCostForPreInliner) {
209 // Point to executable binary if debug info binary is not specified.
210 SymbolizerPath = DebugBinPath.empty() ? ExeBinPath : DebugBinPath;
211 if (InferMissingFrames)
212 MissingContextInferrer = std::make_unique<MissingFrameInferrer>(args: this);
213}
214
215ProfiledBinary::~ProfiledBinary() = default;
216
217void ProfiledBinary::warnNoFuncEntry() {
218 uint64_t NoFuncEntryNum = 0;
219 for (auto &F : BinaryFunctions) {
220 if (F.second.Ranges.empty())
221 continue;
222 bool hasFuncEntry = false;
223 for (auto &R : F.second.Ranges) {
224 if (FuncRange *FR = findFuncRangeForStartAddr(Address: R.first)) {
225 if (FR->IsFuncEntry) {
226 hasFuncEntry = true;
227 break;
228 }
229 }
230 }
231
232 if (!hasFuncEntry) {
233 NoFuncEntryNum++;
234 if (ShowDetailedWarning)
235 WithColor::warning()
236 << "Failed to determine function entry for " << F.first()
237 << " due to inconsistent name from symbol table and dwarf info.\n";
238 }
239 }
240 emitWarningSummary(Num: NoFuncEntryNum, Total: BinaryFunctions.size(),
241 Msg: "of functions failed to determine function entry due to "
242 "inconsistent name from symbol table and dwarf info.");
243}
244
245void ProfiledBinary::load(StringRef TripleStr) {
246 // Attempt to open the binary.
247 OBinary = unwrapOrError(EO: createBinary(Path), Args&: Path);
248 Binary &ExeBinary = *OBinary.getBinary();
249
250 IsCOFF = isa<COFFObjectFile>(Val: &ExeBinary);
251 if (!isa<ELFObjectFileBase>(Val: &ExeBinary) && !IsCOFF)
252 exitWithError(Message: "not a valid ELF/COFF image", Whence: Path);
253
254 auto *Obj = cast<ObjectFile>(Val: &ExeBinary);
255 if (!TripleStr.empty())
256 TheTriple = Triple(TripleStr);
257 else
258 TheTriple = Obj->makeTriple();
259
260 LLVM_DEBUG(dbgs() << "Loading " << Path << "\n");
261
262 // Mark the binary as a kernel image;
263 IsKernel = KernelBinary;
264
265 // Find the preferred load address for text sections.
266 setPreferredTextSegmentAddresses(Obj);
267
268 // For shared libraries, read build ID to filter perfscript addresses
269 // in [buildid:]addr format. Main executables (including PIE) use empty
270 // FilterBuildID since their addresses have no buildid prefix.
271 // Both PIE executables and shared libraries are ET_DYN, but only PIE
272 // executables have a PT_INTERP program header.
273 file_magic Magic;
274 if (auto EC = identify_magic(path: Path, result&: Magic);
275 !EC && Magic == file_magic::elf_shared_object && !HasInterp) {
276 auto BID = object::getBuildID(Obj);
277 if (!BID.empty())
278 FilterBuildID = llvm::toHex(Input: BID, /*LowerCase=*/true);
279 }
280
281 // Load debug info of subprograms from DWARF section.
282 // If path of debug info binary is specified, use the debug info from it,
283 // otherwise use the debug info from the executable binary.
284 OwningBinary<Binary> DebugBinary;
285 ObjectFile *PseudoProbeObj = nullptr;
286 if (!DebugBinaryPath.empty()) {
287 DebugBinary = unwrapOrError(EO: createBinary(Path: DebugBinaryPath), Args&: DebugBinaryPath);
288 ObjectFile *DebugObj = cast<ObjectFile>(Val: DebugBinary.getBinary());
289 loadSymbolsFromDWARF(Obj&: *DebugObj);
290 if (checkPseudoProbe(Obj: DebugObj, ObjPath: DebugBinaryPath))
291 PseudoProbeObj = DebugObj;
292 } else {
293 loadSymbolsFromDWARF(Obj&: *Obj);
294 }
295
296 // Prefer loading pseudo probe from binary.
297 if (checkPseudoProbe(Obj, ObjPath: Path))
298 PseudoProbeObj = Obj;
299
300 DisassembleFunctionSet.insert_range(R&: DisassembleFunctions);
301
302 if (usePseudoProbes())
303 populateSymbolAddressList(O: Obj);
304
305 if (ShowDisassemblyOnly && PseudoProbeObj)
306 decodePseudoProbe(Obj: PseudoProbeObj);
307
308 if (LoadFunctionFromSymbol && usePseudoProbes())
309 loadSymbolsFromSymtab(O: Obj);
310
311 // Disassemble the text sections.
312 disassemble(O: Obj);
313
314 // Use function start and return address to infer prolog and epilog
315 ProEpilogTracker.inferPrologAddresses(FuncStartAddressMap&: StartAddrToFuncRangeMap);
316 ProEpilogTracker.inferEpilogAddresses(RetAddrs&: RetAddressSet);
317
318 warnNoFuncEntry();
319
320 // TODO: decode other sections.
321}
322
323bool ProfiledBinary::inlineContextEqual(uint64_t Address1, uint64_t Address2) {
324 const SampleContextFrameVector &Context1 =
325 getCachedFrameLocationStack(Address: Address1);
326 const SampleContextFrameVector &Context2 =
327 getCachedFrameLocationStack(Address: Address2);
328 if (Context1.size() != Context2.size())
329 return false;
330 if (Context1.empty())
331 return false;
332 // The leaf frame contains location within the leaf, and it
333 // needs to be remove that as it's not part of the calling context
334 return std::equal(first1: Context1.begin(), last1: Context1.begin() + Context1.size() - 1,
335 first2: Context2.begin(), last2: Context2.begin() + Context2.size() - 1);
336}
337
338SampleContextFrameVector
339ProfiledBinary::getExpandedContext(const SmallVectorImpl<uint64_t> &Stack,
340 bool &WasLeafInlined) {
341 SampleContextFrameVector ContextVec;
342 if (Stack.empty())
343 return ContextVec;
344 // Process from frame root to leaf
345 for (auto Address : Stack) {
346 const SampleContextFrameVector &ExpandedContext =
347 getCachedFrameLocationStack(Address);
348 // An instruction without a valid debug line will be ignored by sample
349 // processing
350 if (ExpandedContext.empty())
351 return SampleContextFrameVector();
352 // Set WasLeafInlined to the size of inlined frame count for the last
353 // address which is leaf
354 WasLeafInlined = (ExpandedContext.size() > 1);
355 ContextVec.append(RHS: ExpandedContext);
356 }
357
358 // Replace with decoded base discriminator
359 for (auto &Frame : ContextVec) {
360 Frame.Location.Discriminator = ProfileGeneratorBase::getBaseDiscriminator(
361 Discriminator: Frame.Location.Discriminator, UseFSD: UseFSDiscriminator);
362 }
363
364 assert(ContextVec.size() && "Context length should be at least 1");
365
366 // Compress the context string except for the leaf frame
367 auto LeafFrame = ContextVec.back();
368 LeafFrame.Location = LineLocation(0, 0);
369 ContextVec.pop_back();
370 CSProfileGenerator::compressRecursionContext(Context&: ContextVec);
371 CSProfileGenerator::trimContext(S&: ContextVec);
372 ContextVec.push_back(Elt: LeafFrame);
373 return ContextVec;
374}
375
376template <class ELFT>
377void ProfiledBinary::setPreferredTextSegmentAddresses(const ELFFile<ELFT> &Obj,
378 StringRef FileName) {
379 const auto &PhdrRange = unwrapOrError(Obj.program_headers(), FileName);
380 bool SeenFirstLoadableSegment = false;
381 for (const typename ELFT::Phdr &Phdr : PhdrRange) {
382 if (Phdr.p_type == ELF::PT_INTERP)
383 HasInterp = true;
384 if (Phdr.p_type == ELF::PT_LOAD) {
385 if (!SeenFirstLoadableSegment) {
386 // Derive the preferred address corresponding to file offset zero
387 // without assuming a page size.
388 FirstLoadableAddress = Phdr.p_vaddr - Phdr.p_offset;
389 SeenFirstLoadableSegment = true;
390 }
391 if (Phdr.p_flags & ELF::PF_X) {
392 PreferredTextSegmentAddresses.push_back(Phdr.p_vaddr);
393 TextSegmentOffsets.push_back(Phdr.p_offset);
394 } else {
395 PhdrInfo Info;
396 Info.FileOffset = Phdr.p_offset;
397 Info.FileSz = Phdr.p_filesz;
398 Info.VirtualAddr = Phdr.p_vaddr;
399 NonTextPhdrInfo.push_back(Elt: Info);
400 }
401 }
402 }
403
404 if (PreferredTextSegmentAddresses.empty())
405 exitWithError(Message: "no executable segment found", Whence: FileName);
406}
407
408uint64_t ProfiledBinary::CanonicalizeNonTextAddress(uint64_t Address) {
409 uint64_t FileOffset = 0;
410 auto MMapIter = NonTextMMapEvents.lower_bound(x: Address);
411 if (MMapIter == NonTextMMapEvents.end())
412 return Address; // No non-text mmap event found, return the address as is.
413
414 const auto &MMapEvent = MMapIter->second;
415
416 // If the address is within the non-text mmap event, calculate its file
417 // offset in the binary.
418 if (MMapEvent.Address <= Address &&
419 Address < MMapEvent.Address + MMapEvent.Size)
420 FileOffset = Address - MMapEvent.Address + MMapEvent.Offset;
421
422 // If the address is not within the non-text mmap event, return the address
423 // as is.
424 if (FileOffset == 0)
425 return Address;
426
427 for (const auto &PhdrInfo : NonTextPhdrInfo) {
428 // Find the program section that contains the file offset and map the
429 // file offset to the virtual address.
430 if (PhdrInfo.FileOffset <= FileOffset &&
431 FileOffset < PhdrInfo.FileOffset + PhdrInfo.FileSz)
432 return PhdrInfo.VirtualAddr + (FileOffset - PhdrInfo.FileOffset);
433 }
434
435 return Address;
436}
437
438void ProfiledBinary::setPreferredTextSegmentAddresses(const COFFObjectFile *Obj,
439 StringRef FileName) {
440 uint64_t ImageBase = Obj->getImageBase();
441 if (!ImageBase)
442 exitWithError(Message: "Not a COFF image", Whence: FileName);
443
444 PreferredTextSegmentAddresses.push_back(x: ImageBase);
445 FirstLoadableAddress = ImageBase;
446
447 for (SectionRef Section : Obj->sections()) {
448 const coff_section *Sec = Obj->getCOFFSection(Section);
449 if (Sec->Characteristics & COFF::IMAGE_SCN_CNT_CODE)
450 TextSegmentOffsets.push_back(x: Sec->VirtualAddress);
451 }
452}
453
454void ProfiledBinary::setPreferredTextSegmentAddresses(const ObjectFile *Obj) {
455 if (const auto *ELFObj = dyn_cast<ELF32LEObjectFile>(Val: Obj))
456 setPreferredTextSegmentAddresses(Obj: ELFObj->getELFFile(), FileName: Obj->getFileName());
457 else if (const auto *ELFObj = dyn_cast<ELF32BEObjectFile>(Val: Obj))
458 setPreferredTextSegmentAddresses(Obj: ELFObj->getELFFile(), FileName: Obj->getFileName());
459 else if (const auto *ELFObj = dyn_cast<ELF64LEObjectFile>(Val: Obj))
460 setPreferredTextSegmentAddresses(Obj: ELFObj->getELFFile(), FileName: Obj->getFileName());
461 else if (const auto *ELFObj = dyn_cast<ELF64BEObjectFile>(Val: Obj))
462 setPreferredTextSegmentAddresses(Obj: ELFObj->getELFFile(), FileName: Obj->getFileName());
463 else if (const auto *COFFObj = dyn_cast<COFFObjectFile>(Val: Obj))
464 setPreferredTextSegmentAddresses(Obj: COFFObj, FileName: Obj->getFileName());
465 else
466 llvm_unreachable("invalid object format");
467}
468
469bool ProfiledBinary::checkPseudoProbe(const ObjectFile *Obj,
470 StringRef ObjPath) {
471 if (UseDwarfCorrelation)
472 return false;
473
474 bool HasProbeDescSection = false;
475 bool HasPseudoProbeSection = false;
476
477 StringRef FileName = Obj->getFileName();
478 for (section_iterator SI = Obj->section_begin(), SE = Obj->section_end();
479 SI != SE; ++SI) {
480 const SectionRef &Section = *SI;
481 StringRef SectionName = unwrapOrError(EO: Section.getName(), Args&: FileName);
482 if (SectionName == ".pseudo_probe_desc") {
483 HasProbeDescSection = true;
484 } else if (SectionName == ".pseudo_probe") {
485 HasPseudoProbeSection = true;
486 }
487 }
488
489 if (HasProbeDescSection && HasPseudoProbeSection) {
490 PseudoProbeBinPath = ObjPath;
491 return true;
492 }
493
494 return false;
495}
496
497void ProfiledBinary::decodePseudoProbe(const ObjectFile *Obj) {
498 if (!usePseudoProbes())
499 return;
500
501 LLVM_DEBUG(dbgs() << "Decoding pseudo probe in " << Obj->getFileName()
502 << "\n");
503
504 MCPseudoProbeDecoder::Uint64Set GuidFilter;
505 MCPseudoProbeDecoder::Uint64Map FuncStartAddresses;
506 if (ShowDisassemblyOnly) {
507 if (DisassembleFunctionSet.empty()) {
508 FuncStartAddresses = SymbolStartAddrs;
509 } else {
510 for (auto &F : DisassembleFunctionSet) {
511 auto GUID = Function::getGUIDAssumingExternalLinkage(GlobalName: F.first());
512 if (auto StartAddr = SymbolStartAddrs.lookup(Val: GUID)) {
513 FuncStartAddresses[GUID] = StartAddr;
514 FuncRange &Range = StartAddrToFuncRangeMap[StartAddr];
515 GuidFilter.insert(
516 V: Function::getGUIDAssumingExternalLinkage(GlobalName: Range.getFuncName()));
517 }
518 }
519 }
520 } else {
521 for (auto *F : ProfiledFunctions) {
522 GuidFilter.insert(V: Function::getGUIDAssumingExternalLinkage(GlobalName: F->FuncName));
523 // DWARF name might be broken when a DWARF32 .debug_str.dwo section
524 // execeeds 4GB. We expect symbol table to contain the correct function
525 // names which matches the pseudo probe. Adding back all the GUIDs if
526 // possible.
527 auto AltGUIDs = AlternativeFunctionGUIDs.equal_range(x: F);
528 for (const auto &[_, Func] : make_range(p: AltGUIDs))
529 GuidFilter.insert(V: Func);
530 for (auto &Range : F->Ranges) {
531 auto GUIDs = StartAddrToSymMap.equal_range(x: Range.first);
532 for (const auto &[StartAddr, Func] : make_range(p: GUIDs))
533 FuncStartAddresses[Func] = StartAddr;
534 }
535 }
536 }
537
538 StringRef FileName = Obj->getFileName();
539 for (section_iterator SI = Obj->section_begin(), SE = Obj->section_end();
540 SI != SE; ++SI) {
541 const SectionRef &Section = *SI;
542 StringRef SectionName = unwrapOrError(EO: Section.getName(), Args&: FileName);
543
544 if (SectionName == ".pseudo_probe_desc") {
545 StringRef Contents = unwrapOrError(EO: Section.getContents(), Args&: FileName);
546 if (!ProbeDecoder.buildGUID2FuncDescMap(
547 Start: reinterpret_cast<const uint8_t *>(Contents.data()),
548 Size: Contents.size(), /*IsMMapped=*/false, VerboseWarnings: ShowDetailedWarning))
549 exitWithError(
550 Message: "Pseudo Probe decoder fail in .pseudo_probe_desc section");
551 } else if (SectionName == ".pseudo_probe") {
552 StringRef Contents = unwrapOrError(EO: Section.getContents(), Args&: FileName);
553 if (!ProbeDecoder.buildAddress2ProbeMap(
554 Start: reinterpret_cast<const uint8_t *>(Contents.data()),
555 Size: Contents.size(), GuildFilter: GuidFilter, FuncStartAddrs: FuncStartAddresses))
556 exitWithError(Message: "Pseudo Probe decoder fail in .pseudo_probe section");
557 }
558 }
559
560 // Build TopLevelProbeFrameMap to track size for optimized inlinees when probe
561 // is available
562 if (TrackFuncContextSize) {
563 for (auto &Child : ProbeDecoder.getDummyInlineRoot().getChildren()) {
564 auto *Frame = &Child;
565 StringRef FuncName =
566 ProbeDecoder.getFuncDescForGUID(GUID: Frame->Guid)->FuncName;
567 TopLevelProbeFrameMap[FuncName] = Frame;
568 }
569 }
570
571 if (ShowPseudoProbe)
572 ProbeDecoder.printGUID2FuncDescMap(OS&: outs());
573}
574
575void ProfiledBinary::decodePseudoProbe() {
576 OwningBinary<Binary> OBinary =
577 unwrapOrError(EO: createBinary(Path: PseudoProbeBinPath), Args&: PseudoProbeBinPath);
578 auto *Obj = cast<ObjectFile>(Val: OBinary.getBinary());
579 decodePseudoProbe(Obj);
580}
581
582void ProfiledBinary::setIsFuncEntry(FuncRange *FuncRange,
583 StringRef RangeSymName) {
584 // Skip external function symbol.
585 if (!FuncRange)
586 return;
587
588 // Set IsFuncEntry to ture if there is only one range in the function or the
589 // RangeSymName from ELF is equal to its DWARF-based function name.
590 if (FuncRange->Func->Ranges.size() == 1 ||
591 (!FuncRange->IsFuncEntry &&
592 (FuncRange->getFuncName() == RangeSymName ||
593 FuncRange->Func->NameStatus != DwarfNameStatus::Matched)))
594 FuncRange->IsFuncEntry = true;
595}
596
597bool ProfiledBinary::dissassembleSymbol(std::size_t SI, ArrayRef<uint8_t> Bytes,
598 SectionSymbolsTy &Symbols,
599 const SectionRef &Section) {
600 std::size_t SE = Symbols.size();
601 uint64_t SectionAddress = Section.getAddress();
602 uint64_t SectSize = Section.getSize();
603 uint64_t StartAddress = Symbols[SI].Addr;
604 uint64_t NextStartAddress =
605 (SI + 1 < SE) ? Symbols[SI + 1].Addr : SectionAddress + SectSize;
606 FuncRange *FRange = findFuncRange(Address: StartAddress);
607 setIsFuncEntry(FuncRange: FRange, RangeSymName: FunctionSamples::getCanonicalFnName(FnName: Symbols[SI].Name));
608 StringRef SymbolName =
609 ShowCanonicalFnName
610 ? FunctionSamples::getCanonicalFnName(FnName: Symbols[SI].Name)
611 : Symbols[SI].Name;
612 bool ShowDisassembly =
613 ShowDisassemblyOnly && (DisassembleFunctionSet.empty() ||
614 DisassembleFunctionSet.count(Key: SymbolName));
615 if (ShowDisassembly)
616 outs() << '<' << SymbolName << ">:\n";
617
618 uint64_t Address = StartAddress;
619 // Size of a consecutive invalid instruction range starting from Address -1
620 // backwards.
621 uint64_t InvalidInstLength = 0;
622 while (Address < NextStartAddress) {
623 MCInst Inst;
624 uint64_t Size;
625 // Disassemble an instruction.
626 bool Disassembled = DisAsm->getInstruction(
627 Instr&: Inst, Size, Bytes: Bytes.slice(N: Address - SectionAddress), Address, CStream&: nulls());
628 if (Size == 0)
629 Size = 1;
630
631 if (ShowDisassembly) {
632 if (ShowPseudoProbe) {
633 ProbeDecoder.printProbeForAddress(OS&: outs(), Address);
634 }
635 outs() << format(Fmt: "%8" PRIx64 ":", Vals: Address);
636 size_t Start = outs().tell();
637 if (Disassembled)
638 IPrinter->printInst(MI: &Inst, Address: Address + Size, Annot: "", STI: *STI, OS&: outs());
639 else
640 outs() << "\t<unknown>";
641 if (ShowSourceLocations) {
642 unsigned Cur = outs().tell() - Start;
643 if (Cur < 40)
644 outs().indent(NumSpaces: 40 - Cur);
645 InstructionPointer IP(this, Address);
646 outs() << getReversedLocWithContext(
647 Context: symbolize(IP, UseCanonicalFnName: ShowCanonicalFnName, UseProbeDiscriminator: ShowPseudoProbe));
648 }
649 outs() << "\n";
650 }
651
652 if (Disassembled) {
653 const MCInstrDesc &MCDesc = MII->get(Opcode: Inst.getOpcode());
654
655 // Record instruction size.
656 AddressToInstSizeMap[Address] = Size;
657
658 // Populate address maps.
659 CodeAddressVec.push_back(x: Address);
660 if (MCDesc.isCall()) {
661 CallAddressSet.insert(V: Address);
662 UncondBranchAddrSet.insert(x: Address);
663 // Record the instruction after call as the branch target of a ret
664 BranchTargetAddressSet.insert(V: Address + Size);
665 } else if (MCDesc.isReturn()) {
666 RetAddressSet.insert(V: Address);
667 UncondBranchAddrSet.insert(x: Address);
668 } else if (MCDesc.isBranch()) {
669 if (MCDesc.isUnconditionalBranch())
670 UncondBranchAddrSet.insert(x: Address);
671 BranchAddressSet.insert(V: Address);
672 }
673
674 if (MCDesc.isIndirectBranch()) {
675 IndirectBranchAddressSet.insert(V: Address);
676 }
677
678 // Record branch target addresses for branches and calls.
679 if (MCDesc.isCall() || MCDesc.isBranch()) {
680 uint64_t Target = 0;
681 if (MIA->evaluateBranch(Inst, Addr: Address, Size, Target))
682 BranchTargetAddressSet.insert(V: Target);
683 }
684
685 // Record potential call targets for tail frame inference later-on.
686 if (InferMissingFrames && FRange) {
687 uint64_t Target = 0;
688 [[maybe_unused]] bool Err =
689 MIA->evaluateBranch(Inst, Addr: Address, Size, Target);
690 if (MCDesc.isCall()) {
691 // Indirect call targets are unknown at this point. Recording the
692 // unknown target (zero) for further LBR-based refinement.
693 MissingContextInferrer->CallEdges[Address].insert(V: Target);
694 } else if (MCDesc.isUnconditionalBranch()) {
695 assert(Err &&
696 "target should be known for unconditional direct branch");
697 // Any inter-function unconditional jump is considered tail call at
698 // this point. This is not 100% accurate and could further be
699 // optimized based on some source annotation.
700 FuncRange *ToFRange = findFuncRange(Address: Target);
701 if (ToFRange && ToFRange->Func != FRange->Func)
702 MissingContextInferrer->TailCallEdges[Address].insert(V: Target);
703 LLVM_DEBUG({
704 dbgs() << "Direct Tail call: " << format("%8" PRIx64 ":", Address);
705 IPrinter->printInst(&Inst, Address + Size, "", *STI.get(), dbgs());
706 dbgs() << "\n";
707 });
708 } else if (MCDesc.isIndirectBranch() && MCDesc.isBarrier()) {
709 // This is an indirect branch but not necessarily an indirect tail
710 // call. The isBarrier check is to filter out conditional branch.
711 // Similar with indirect call targets, recording the unknown target
712 // (zero) for further LBR-based refinement.
713 MissingContextInferrer->TailCallEdges[Address].insert(V: Target);
714 LLVM_DEBUG({
715 dbgs() << "Indirect Tail call: "
716 << format("%8" PRIx64 ":", Address);
717 IPrinter->printInst(&Inst, Address + Size, "", *STI.get(), dbgs());
718 dbgs() << "\n";
719 });
720 }
721 }
722
723 if (InvalidInstLength) {
724 AddrsWithInvalidInstruction.insert(
725 V: {Address - InvalidInstLength, Address - 1});
726 InvalidInstLength = 0;
727 }
728 } else {
729 InvalidInstLength += Size;
730 }
731
732 Address += Size;
733 }
734
735 if (InvalidInstLength)
736 AddrsWithInvalidInstruction.insert(
737 V: {Address - InvalidInstLength, Address - 1});
738
739 if (ShowDisassembly)
740 outs() << "\n";
741
742 return true;
743}
744
745void ProfiledBinary::setUpDisassembler(const ObjectFile *Obj) {
746 const Target *TheTarget = getTarget(Obj);
747 StringRef FileName = Obj->getFileName();
748
749 MRI.reset(p: TheTarget->createMCRegInfo(TT: TheTriple));
750 if (!MRI)
751 exitWithError(Message: "no register info for target " + TheTriple.str(), Whence: FileName);
752
753 MCTargetOptions MCOptions;
754 AsmInfo.reset(p: TheTarget->createMCAsmInfo(MRI: *MRI, TheTriple, Options: MCOptions));
755 if (!AsmInfo)
756 exitWithError(Message: "no assembly info for target " + TheTriple.str(), Whence: FileName);
757
758 Expected<SubtargetFeatures> Features = Obj->getFeatures();
759 if (!Features)
760 exitWithError(E: Features.takeError(), Whence: FileName);
761 // AArch64 object files do not generally carry complete ISA feature metadata,
762 // so the subtarget would default to the baseline (Armv8.0-A) feature set.
763 // That disassembler cannot decode feature-gated instructions (LSE atomics,
764 // RCPC loads, SVE, ...) that are pervasive in modern AArch64 binaries; they
765 // would be miscounted as "invalid instructions" and, worse, their addresses
766 // would be absent from the code/branch maps used for sample attribution.
767 // Enable all instructions so the disassembler recognizes whatever the
768 // compiler emitted, matching llvm-objdump's default for AArch64.
769 if (TheTriple.isAArch64())
770 Features->AddFeature(String: "+all");
771 STI.reset(
772 p: TheTarget->createMCSubtargetInfo(TheTriple, CPU: "", Features: Features->getString()));
773 if (!STI)
774 exitWithError(Message: "no subtarget info for target " + TheTriple.str(), Whence: FileName);
775
776 MII.reset(p: TheTarget->createMCInstrInfo());
777 if (!MII)
778 exitWithError(Message: "no instruction info for target " + TheTriple.str(),
779 Whence: FileName);
780
781 MCContext Ctx(TheTriple, *AsmInfo, *MRI, *STI);
782 std::unique_ptr<MCObjectFileInfo> MOFI(
783 TheTarget->createMCObjectFileInfo(Ctx, /*PIC=*/false));
784 Ctx.setObjectFileInfo(MOFI.get());
785 DisAsm.reset(p: TheTarget->createMCDisassembler(STI: *STI, Ctx));
786 if (!DisAsm)
787 exitWithError(Message: "no disassembler for target " + TheTriple.str(), Whence: FileName);
788
789 MIA.reset(p: TheTarget->createMCInstrAnalysis(Info: MII.get()));
790
791 int AsmPrinterVariant = AsmInfo->getAssemblerDialect();
792 IPrinter.reset(p: TheTarget->createMCInstPrinter(T: TheTriple, SyntaxVariant: AsmPrinterVariant,
793 MAI: *AsmInfo, MII: *MII, MRI: *MRI));
794 IPrinter->setPrintBranchImmAsAddress(true);
795}
796
797void ProfiledBinary::disassemble(const ObjectFile *Obj) {
798 // Set up disassembler and related components.
799 setUpDisassembler(Obj);
800
801 // Create a mapping from virtual address to symbol name. The symbols in text
802 // sections are the candidates to dissassemble.
803 std::map<SectionRef, SectionSymbolsTy> AllSymbols;
804 StringRef FileName = Obj->getFileName();
805 for (const SymbolRef &Symbol : Obj->symbols()) {
806 const uint64_t Addr = unwrapOrError(EO: Symbol.getAddress(), Args&: FileName);
807 const StringRef Name = unwrapOrError(EO: Symbol.getName(), Args&: FileName);
808 section_iterator SecI = unwrapOrError(EO: Symbol.getSection(), Args&: FileName);
809 if (SecI != Obj->section_end())
810 AllSymbols[*SecI].push_back(x: SymbolInfoTy(Addr, Name, ELF::STT_NOTYPE));
811 }
812
813 // Sort all the symbols. Use a stable sort to stabilize the output.
814 for (std::pair<const SectionRef, SectionSymbolsTy> &SecSyms : AllSymbols)
815 stable_sort(Range&: SecSyms.second);
816
817 assert((DisassembleFunctionSet.empty() || ShowDisassemblyOnly) &&
818 "Functions to disassemble should be only specified together with "
819 "--show-disassembly-only");
820
821 if (ShowDisassemblyOnly)
822 outs() << "\nDisassembly of " << FileName << ":\n";
823
824 // Dissassemble a text section.
825 for (section_iterator SI = Obj->section_begin(), SE = Obj->section_end();
826 SI != SE; ++SI) {
827 const SectionRef &Section = *SI;
828 if (!Section.isText())
829 continue;
830
831 uint64_t ImageLoadAddr = getPreferredBaseAddress();
832 uint64_t SectionAddress = Section.getAddress() - ImageLoadAddr;
833 uint64_t SectSize = Section.getSize();
834 if (!SectSize)
835 continue;
836
837 // Register the text section.
838 TextSections.insert(x: {SectionAddress, SectSize});
839
840 StringRef SectionName = unwrapOrError(EO: Section.getName(), Args&: FileName);
841
842 if (ShowDisassemblyOnly) {
843 outs() << "\nDisassembly of section " << SectionName;
844 outs() << " [" << format(Fmt: "0x%" PRIx64, Vals: Section.getAddress()) << ", "
845 << format(Fmt: "0x%" PRIx64, Vals: Section.getAddress() + SectSize)
846 << "]:\n\n";
847 }
848
849 if (isa<ELFObjectFileBase>(Val: Obj) && SectionName == ".plt")
850 continue;
851
852 // Get the section data.
853 ArrayRef<uint8_t> Bytes =
854 arrayRefFromStringRef(Input: unwrapOrError(EO: Section.getContents(), Args&: FileName));
855
856 // Get the list of all the symbols in this section.
857 SectionSymbolsTy &Symbols = AllSymbols[Section];
858
859 // Disassemble symbol by symbol.
860 for (std::size_t SI = 0, SE = Symbols.size(); SI != SE; ++SI) {
861 if (!dissassembleSymbol(SI, Bytes, Symbols, Section))
862 exitWithError(Message: "disassembling error", Whence: FileName);
863 }
864 }
865
866 if (!AddrsWithInvalidInstruction.empty()) {
867 if (ShowDetailedWarning) {
868 for (auto &Addr : AddrsWithInvalidInstruction) {
869 WithColor::warning()
870 << "Invalid instructions at " << format(Fmt: "%8" PRIx64, Vals: Addr.first)
871 << " - " << format(Fmt: "%8" PRIx64, Vals: Addr.second) << "\n";
872 }
873 }
874 WithColor::warning() << "Found " << AddrsWithInvalidInstruction.size()
875 << " invalid instructions\n";
876 AddrsWithInvalidInstruction.clear();
877 }
878
879 // Dissassemble rodata section to check if FS discriminator symbol exists.
880 checkUseFSDiscriminator(Obj, AllSymbols);
881}
882
883void ProfiledBinary::checkUseFSDiscriminator(
884 const ObjectFile *Obj, std::map<SectionRef, SectionSymbolsTy> &AllSymbols) {
885 const char *FSDiscriminatorVar = "__llvm_fs_discriminator__";
886 for (section_iterator SI = Obj->section_begin(), SE = Obj->section_end();
887 SI != SE; ++SI) {
888 const SectionRef &Section = *SI;
889 if (!Section.isData() || Section.getSize() == 0)
890 continue;
891 SectionSymbolsTy &Symbols = AllSymbols[Section];
892
893 for (std::size_t SI = 0, SE = Symbols.size(); SI != SE; ++SI) {
894 if (Symbols[SI].Name == FSDiscriminatorVar) {
895 UseFSDiscriminator = true;
896 return;
897 }
898 }
899 }
900}
901
902void ProfiledBinary::populateSymbolAddressList(const ObjectFile *Obj) {
903 // Create a mapping from virtual address to symbol GUID and the other way
904 // around.
905 StringRef FileName = Obj->getFileName();
906 for (const SymbolRef &Symbol : Obj->symbols()) {
907 const uint64_t Addr = unwrapOrError(EO: Symbol.getAddress(), Args&: FileName);
908 const StringRef Name = unwrapOrError(EO: Symbol.getName(), Args&: FileName);
909 uint64_t GUID = Function::getGUIDAssumingExternalLinkage(GlobalName: Name);
910 SymbolStartAddrs[GUID] = Addr;
911 StartAddrToSymMap.emplace(args: Addr, args&: GUID);
912 }
913}
914
915void ProfiledBinary::loadSymbolsFromSymtab(const ObjectFile *Obj) {
916 // Load binary functions from symbol table when Debug info is incomplete.
917 StringRef FileName = Obj->getFileName();
918
919 // COFF symtab does not have size field. Try to load size from PDB instead.
920 std::unique_ptr<pdb::IPDBSession> PDBSession;
921 if (auto *COFFObj = dyn_cast<COFFObjectFile>(Val: Obj)) {
922 if (auto E = pdb::loadDataForEXE(Type: pdb::PDB_ReaderType::Native, Path: FileName,
923 Session&: PDBSession)) {
924 StringRef PdbPath;
925 const codeview::DebugInfo *PdbInfo;
926 if (auto Err = COFFObj->getDebugPDBInfo(Info&: PdbInfo, PDBFileName&: PdbPath))
927 consumeError(Err: std::move(Err));
928
929 auto Style = PdbPath.starts_with(Prefix: "/") ? sys::path::Style::posix
930 : sys::path::Style::windows;
931 WithColor::warning() << "Cannot load PDB file "
932 << sys::path::filename(path: PdbPath, style: Style) << " for "
933 << FileName << ": " << E << "\n";
934 consumeError(Err: std::move(E));
935 } else {
936 PDBSession->setLoadAddress(FirstLoadableAddress);
937 }
938 }
939
940 for (const SymbolRef &Symbol : Obj->symbols()) {
941 const SymbolRef::Type Type = unwrapOrError(EO: Symbol.getType(), Args&: FileName);
942 const uint64_t StartAddr = unwrapOrError(EO: Symbol.getAddress(), Args&: FileName);
943 const StringRef Name = unwrapOrError(EO: Symbol.getName(), Args&: FileName);
944 uint64_t Size = 0;
945 if (isa<ELFObjectFileBase>(Val: Obj)) {
946 ELFSymbolRef ElfSymbol(Symbol);
947 Size = ElfSymbol.getSize();
948 } else if (PDBSession) {
949 if (std::unique_ptr<pdb::PDBSymbol> Sym = PDBSession->findSymbolByAddress(
950 Address: StartAddr, Type: pdb::PDB_SymType::Function)) {
951 auto FuncSym = cast<pdb::PDBSymbolFunc>(Val: std::move(Sym));
952 if (StartAddr == FuncSym->getVirtualAddress())
953 Size = FuncSym->getLength();
954 }
955 }
956
957 if (Size == 0 || Type != SymbolRef::ST_Function)
958 continue;
959
960 const uint64_t EndAddr = StartAddr + Size;
961 const StringRef SymName =
962 FunctionSamples::getCanonicalFnName(FnName: Name, Suffixes: CanonicalSuffixes);
963 assert(StartAddr < EndAddr && StartAddr >= getPreferredBaseAddress() &&
964 "Function range is invalid.");
965
966 auto Range = findFuncRange(Address: StartAddr);
967 if (!Range) {
968 assert(findFuncRange(EndAddr - 1) == nullptr &&
969 "Function range overlaps with existing functions.");
970 // Function from symbol table not found previously in DWARF, store ranges.
971 auto Ret = BinaryFunctions.try_emplace(Key: SymName);
972 auto &Func = Ret.first->second;
973 if (Ret.second) {
974 Func.FuncName = Ret.first->first();
975 HashBinaryFunctions[Function::getGUIDAssumingExternalLinkage(GlobalName: SymName)] =
976 &Func;
977 }
978
979 Func.NameStatus = DwarfNameStatus::Missing;
980 Func.Ranges.emplace_back(args: StartAddr, args: EndAddr);
981
982 auto R = StartAddrToFuncRangeMap.emplace(args: StartAddr, args: FuncRange());
983 FuncRange &FRange = R.first->second;
984
985 FRange.Func = &Func;
986 FRange.StartAddress = StartAddr;
987 FRange.EndAddress = EndAddr;
988
989 } else if (SymName != Range->getFuncName()) {
990 // Function range already found from DWARF or symtab, but the symbol name
991 // from symbol table is inconsistent with the existing name associated
992 // with the range. Log this discrepancy and the alternative function GUID.
993 if (ShowDetailedWarning)
994 WithColor::warning()
995 << "Conflicting name for symbol " << Name << " with range ("
996 << format(Fmt: "%8" PRIx64, Vals: StartAddr) << ", "
997 << format(Fmt: "%8" PRIx64, Vals: EndAddr) << ")"
998 << ", but the existing symbol " << Range->getFuncName()
999 << " indicates an overlapping range ("
1000 << format(Fmt: "%8" PRIx64, Vals: Range->StartAddress) << ", "
1001 << format(Fmt: "%8" PRIx64, Vals: Range->EndAddress) << ")\n";
1002
1003 assert(StartAddr == Range->StartAddress && EndAddr == Range->EndAddress &&
1004 "Mismatched function range");
1005
1006 Range->Func->NameStatus = DwarfNameStatus::Mismatch;
1007 AlternativeFunctionGUIDs.emplace(
1008 args&: Range->Func, args: Function::getGUIDAssumingExternalLinkage(GlobalName: SymName));
1009
1010 } else if (StartAddr != Range->StartAddress &&
1011 EndAddr != Range->EndAddress) {
1012 // Function already found in DWARF or symtab, but the address range from
1013 // symbol table conflicts/overlaps with the existing one.
1014 WithColor::warning() << "Conflicting range for symbol " << Name
1015 << " with range (" << format(Fmt: "%8" PRIx64, Vals: StartAddr)
1016 << ", " << format(Fmt: "%8" PRIx64, Vals: EndAddr) << ")"
1017 << ", but the existing symbol "
1018 << Range->getFuncName()
1019 << " indicates another range ("
1020 << format(Fmt: "%8" PRIx64, Vals: Range->StartAddress) << ", "
1021 << format(Fmt: "%8" PRIx64, Vals: Range->EndAddress) << ")\n";
1022 }
1023 }
1024}
1025
1026void ProfiledBinary::loadSymbolsFromDWARFUnit(DWARFUnit &CompilationUnit) {
1027 for (const auto &DieInfo : CompilationUnit.dies()) {
1028 llvm::DWARFDie Die(&CompilationUnit, &DieInfo);
1029
1030 if (!Die.isSubprogramDIE())
1031 continue;
1032 auto Name = Die.getName(Kind: llvm::DINameKind::LinkageName);
1033 if (!Name)
1034 Name = Die.getName(Kind: llvm::DINameKind::ShortName);
1035 if (!Name)
1036 continue;
1037
1038 auto CanonName = FunctionSamples::getCanonicalCoroFnName(FnName: Name);
1039 auto RangesOrError = Die.getAddressRanges();
1040 if (!RangesOrError)
1041 continue;
1042 const DWARFAddressRangesVector &Ranges = RangesOrError.get();
1043
1044 if (Ranges.empty())
1045 continue;
1046
1047 // Different DWARF symbols can have same function name, search or create
1048 // BinaryFunction indexed by the name.
1049 auto Ret = BinaryFunctions.try_emplace(Key: CanonName);
1050 auto &Func = Ret.first->second;
1051 if (Ret.second)
1052 Func.FuncName = Ret.first->first();
1053
1054 for (const auto &Range : Ranges) {
1055 uint64_t StartAddress = Range.LowPC;
1056 uint64_t EndAddress = Range.HighPC;
1057
1058 if (EndAddress <= StartAddress ||
1059 StartAddress < getPreferredBaseAddress())
1060 continue;
1061
1062 // We may want to know all ranges for one function. Here group the
1063 // ranges and store them into BinaryFunction.
1064 Func.Ranges.emplace_back(args&: StartAddress, args&: EndAddress);
1065
1066 auto R = StartAddrToFuncRangeMap.emplace(args&: StartAddress, args: FuncRange());
1067 if (R.second) {
1068 FuncRange &FRange = R.first->second;
1069 FRange.Func = &Func;
1070 FRange.StartAddress = StartAddress;
1071 FRange.EndAddress = EndAddress;
1072 } else {
1073 AddrsWithMultipleSymbols.insert(V: StartAddress);
1074 if (ShowDetailedWarning)
1075 WithColor::warning()
1076 << "Duplicated symbol start address at "
1077 << format(Fmt: "%8" PRIx64, Vals: StartAddress) << " "
1078 << R.first->second.getFuncName() << " and " << Name << "\n";
1079 }
1080 }
1081 }
1082}
1083
1084void ProfiledBinary::loadSymbolsFromDWARF(ObjectFile &Obj) {
1085 auto DebugContext = llvm::DWARFContext::create(
1086 Obj, RelocAction: DWARFContext::ProcessDebugRelocations::Process, L: nullptr, DWPName: DWPPath);
1087 if (!DebugContext)
1088 exitWithError(Message: "Error creating the debug info context", Whence: Path);
1089
1090 for (const auto &CompilationUnit : DebugContext->compile_units())
1091 loadSymbolsFromDWARFUnit(CompilationUnit&: *CompilationUnit);
1092
1093 // Handles DWO sections that can either be in .o, .dwo or .dwp files.
1094 uint32_t NumOfDWOMissing = 0;
1095 for (const auto &CompilationUnit : DebugContext->compile_units()) {
1096 DWARFUnit *const DwarfUnit = CompilationUnit.get();
1097 if (DwarfUnit->getDWOId()) {
1098 DWARFUnit *DWOCU = DwarfUnit->getNonSkeletonUnitDIE(ExtractUnitDIEOnly: false).getDwarfUnit();
1099 if (!DWOCU->isDWOUnit()) {
1100 NumOfDWOMissing++;
1101 if (ShowDetailedWarning) {
1102 std::string DWOName = dwarf::toString(
1103 V: DwarfUnit->getUnitDIE().find(
1104 Attrs: {dwarf::DW_AT_dwo_name, dwarf::DW_AT_GNU_dwo_name}),
1105 Default: "");
1106 WithColor::warning() << "DWO debug information for " << DWOName
1107 << " was not loaded.\n";
1108 }
1109 continue;
1110 }
1111 loadSymbolsFromDWARFUnit(CompilationUnit&: *DWOCU);
1112 }
1113 }
1114
1115 if (NumOfDWOMissing)
1116 WithColor::warning()
1117 << " DWO debug information was not loaded for " << NumOfDWOMissing
1118 << " modules. Please check the .o, .dwo or .dwp path.\n";
1119 if (BinaryFunctions.empty())
1120 WithColor::warning() << "Loading of DWARF info completed, but no binary "
1121 "functions have been retrieved.\n";
1122 // Populate the hash binary function map for MD5 function name lookup. This
1123 // is done after BinaryFunctions are finalized.
1124 for (auto &BinaryFunction : BinaryFunctions) {
1125 HashBinaryFunctions[MD5Hash(Str: BinaryFunction.first())] =
1126 &BinaryFunction.second;
1127 }
1128
1129 if (!AddrsWithMultipleSymbols.empty()) {
1130 WithColor::warning() << "Found " << AddrsWithMultipleSymbols.size()
1131 << " start addresses with multiple symbols\n";
1132 AddrsWithMultipleSymbols.clear();
1133 }
1134}
1135
1136void ProfiledBinary::populateSymbolListFromDWARF(
1137 ProfileSymbolList &SymbolList) {
1138 for (auto &I : StartAddrToFuncRangeMap)
1139 SymbolList.add(Name: I.second.getFuncName());
1140}
1141
1142symbolize::LLVMSymbolizer::Options ProfiledBinary::getSymbolizerOpts() const {
1143 symbolize::LLVMSymbolizer::Options SymbolizerOpts;
1144 SymbolizerOpts.PrintFunctions =
1145 DILineInfoSpecifier::FunctionNameKind::LinkageName;
1146 SymbolizerOpts.Demangle = false;
1147 SymbolizerOpts.DefaultArch = TheTriple.getArchName().str();
1148 SymbolizerOpts.UseSymbolTable = false;
1149 SymbolizerOpts.RelativeAddresses = false;
1150 SymbolizerOpts.DWPName = DWPPath;
1151 return SymbolizerOpts;
1152}
1153
1154SampleContextFrameVector ProfiledBinary::symbolize(const InstructionPointer &IP,
1155 bool UseCanonicalFnName,
1156 bool UseProbeDiscriminator) {
1157 assert(this == IP.Binary &&
1158 "Binary should only symbolize its own instruction");
1159 DIInliningInfo InlineStack =
1160 unwrapOrError(EO: Symbolizer->symbolizeInlinedCode(
1161 ModuleName: SymbolizerPath.str(), ModuleOffset: getSectionedAddress(Address: IP.Address)),
1162 Args&: SymbolizerPath);
1163
1164 SampleContextFrameVector CallStack;
1165 for (int32_t I = InlineStack.getNumberOfFrames() - 1; I >= 0; I--) {
1166 const auto &CallerFrame = InlineStack.getFrame(Index: I);
1167 if (CallerFrame.FunctionName.empty() ||
1168 (CallerFrame.FunctionName == "<invalid>"))
1169 break;
1170
1171 StringRef FunctionName(CallerFrame.FunctionName);
1172 if (UseCanonicalFnName)
1173 FunctionName =
1174 FunctionSamples::getCanonicalFnName(FnName: FunctionName, Suffixes: CanonicalSuffixes);
1175 else
1176 FunctionName =
1177 FunctionSamples::getCanonicalFnName(FnName: FunctionName, Suffixes: CoroSuffixes);
1178
1179 uint32_t Discriminator = CallerFrame.Discriminator;
1180 uint32_t LineOffset = (CallerFrame.Line - CallerFrame.StartLine) & 0xffff;
1181 if (UseProbeDiscriminator) {
1182 LineOffset =
1183 PseudoProbeDwarfDiscriminator::extractProbeIndex(Value: Discriminator);
1184 Discriminator = 0;
1185 }
1186
1187 LineLocation Line(LineOffset, Discriminator);
1188 auto It = NameStrings.insert(key: FunctionName);
1189 CallStack.emplace_back(Args: FunctionId(It.first->getKey()), Args&: Line);
1190 }
1191
1192 if (WarnNotSymbolized && CallStack.empty()) {
1193 uint64_t VAddr = IP.Address + BaseAddress - getPreferredBaseAddress();
1194 if (isVaddrMMapped(VAddr))
1195 WithColor::warning() << "Failed to symbolize address "
1196 << format(Fmt: "%8" PRIx64, Vals: IP.Address)
1197 << " (vaddr=" << format(Fmt: "%8" PRIx64, Vals: VAddr) << ")\n";
1198 }
1199
1200 return CallStack;
1201}
1202
1203StringRef ProfiledBinary::symbolizeDataAddress(uint64_t Address) {
1204 DIGlobal DataDIGlobal =
1205 unwrapOrError(EO: Symbolizer->symbolizeData(ModuleName: SymbolizerPath.str(),
1206 ModuleOffset: getSectionedAddress(Address)),
1207 Args&: SymbolizerPath);
1208 return NameStrings.insert(key: DataDIGlobal.Name).first->getKey();
1209}
1210
1211void ProfiledBinary::computeInlinedContextSizeForRange(uint64_t RangeBegin,
1212 uint64_t RangeEnd) {
1213 InstructionPointer IP(this, RangeBegin, true);
1214
1215 if (IP.Address != RangeBegin)
1216 WithColor::warning() << "Invalid start instruction at "
1217 << format(Fmt: "%8" PRIx64, Vals: RangeBegin) << "\n";
1218
1219 if (IP.Address >= RangeEnd)
1220 return;
1221
1222 do {
1223 const SampleContextFrameVector SymbolizedCallStack =
1224 getFrameLocationStack(Address: IP.Address, UseProbeDiscriminator: usePseudoProbes());
1225 uint64_t Size = AddressToInstSizeMap[IP.Address];
1226 // Record instruction size for the corresponding context
1227 FuncSizeTracker.addInstructionForContext(Context: SymbolizedCallStack, InstrSize: Size);
1228
1229 } while (IP.advance() && IP.Address < RangeEnd);
1230}
1231
1232void ProfiledBinary::computeInlinedContextSizeForFunc(
1233 const BinaryFunction *Func) {
1234 // Note that a function can be spilt into multiple ranges, so compute for all
1235 // ranges of the function.
1236 for (const auto &Range : Func->Ranges)
1237 computeInlinedContextSizeForRange(RangeBegin: Range.first, RangeEnd: Range.second);
1238
1239 // Track optimized-away inlinee for probed binary. A function inlined and then
1240 // optimized away should still have their probes left over in places.
1241 if (usePseudoProbes()) {
1242 auto I = TopLevelProbeFrameMap.find(Key: Func->FuncName);
1243 if (I != TopLevelProbeFrameMap.end()) {
1244 BinarySizeContextTracker::ProbeFrameStack ProbeContext;
1245 FuncSizeTracker.trackInlineesOptimizedAway(ProbeDecoder, ProbeNode: *I->second,
1246 ProbeContext);
1247 }
1248 }
1249}
1250
1251void ProfiledBinary::loadSymbolsFromPseudoProbe() {
1252 if (!usePseudoProbes())
1253 return;
1254
1255 const AddressProbesMap &Address2ProbesMap = getAddress2ProbesMap();
1256 for (auto *Func : ProfiledFunctions) {
1257 if (Func->NameStatus != DwarfNameStatus::Mismatch)
1258 continue;
1259 for (auto &[StartAddr, EndAddr] : Func->Ranges) {
1260 auto Range = findFuncRangeForStartAddr(Address: StartAddr);
1261 if (!Range->IsFuncEntry)
1262 continue;
1263 const auto &Probe = Address2ProbesMap.find(From: StartAddr, To: EndAddr);
1264 if (Probe.begin() != Probe.end()) {
1265 const MCDecodedPseudoProbeInlineTree *InlineTreeNode =
1266 Probe.begin()->get().getInlineTreeNode();
1267 while (!InlineTreeNode->isTopLevelFunc())
1268 InlineTreeNode = static_cast<MCDecodedPseudoProbeInlineTree *>(
1269 InlineTreeNode->Parent);
1270
1271 assert(llvm::any_of(InlineTreeNode->getProbes(),
1272 [Start = StartAddr, End = EndAddr](const auto &P) {
1273 return P.getAddress() >= Start &&
1274 P.getAddress() < End;
1275 }) &&
1276 "Top level pseudo probe does not match function range");
1277
1278 const auto *ProbeDesc = getFuncDescForGUID(GUID: InlineTreeNode->Guid);
1279 auto Ret = PseudoProbeNames.try_emplace(Key: Func, Args: ProbeDesc->FuncName);
1280 if (!Ret.second && Ret.first->second != ProbeDesc->FuncName &&
1281 ShowDetailedWarning)
1282 WithColor::warning()
1283 << "Mismatched pseudo probe names in function " << Func->FuncName
1284 << " at range: (" << format(Fmt: "%8" PRIx64, Vals: StartAddr) << ", "
1285 << format(Fmt: "%8" PRIx64, Vals: EndAddr) << "). "
1286 << "The previously found pseudo probe name is "
1287 << Ret.first->second << " but it conflicts with name "
1288 << ProbeDesc->FuncName
1289 << " This likely indicates a DWARF error that produces "
1290 "conflicting symbols at the same starting address.\n";
1291 }
1292 }
1293 }
1294}
1295
1296StringRef ProfiledBinary::findPseudoProbeName(const BinaryFunction *Func) {
1297 auto ProbeName = PseudoProbeNames.find(Val: Func);
1298 if (ProbeName == PseudoProbeNames.end())
1299 return StringRef();
1300 return ProbeName->second;
1301}
1302
1303void ProfiledBinary::inferMissingFrames(
1304 const SmallVectorImpl<uint64_t> &Context,
1305 SmallVectorImpl<uint64_t> &NewContext) {
1306 MissingContextInferrer->inferMissingFrames(Context, NewContext);
1307}
1308
1309InstructionPointer::InstructionPointer(const ProfiledBinary *Binary,
1310 uint64_t Address, bool RoundToNext)
1311 : Binary(Binary), Address(Address) {
1312 Index = Binary->getIndexForAddr(Address);
1313 if (RoundToNext) {
1314 // we might get address which is not the code
1315 // it should round to the next valid address
1316 if (Index >= Binary->getCodeAddrVecSize())
1317 this->Address = UINT64_MAX;
1318 else
1319 this->Address = Binary->getAddressforIndex(Index);
1320 }
1321}
1322
1323bool InstructionPointer::advance() {
1324 Index++;
1325 if (Index >= Binary->getCodeAddrVecSize()) {
1326 Address = UINT64_MAX;
1327 return false;
1328 }
1329 Address = Binary->getAddressforIndex(Index);
1330 return true;
1331}
1332
1333bool InstructionPointer::backward() {
1334 if (Index == 0) {
1335 Address = 0;
1336 return false;
1337 }
1338 Index--;
1339 Address = Binary->getAddressforIndex(Index);
1340 return true;
1341}
1342
1343void InstructionPointer::update(uint64_t Addr) {
1344 Address = Addr;
1345 Index = Binary->getIndexForAddr(Address);
1346}
1347
1348} // end namespace sampleprof
1349} // end namespace llvm
1350