1//===-- ProfiledBinary.cpp - Binary decoder ---------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "ProfiledBinary.h"
10#include "ErrorHandling.h"
11#include "MissingFrameInferrer.h"
12#include "ProfileGenerator.h"
13#include "llvm/DebugInfo/Symbolize/SymbolizableModule.h"
14#include "llvm/Demangle/Demangle.h"
15#include "llvm/IR/DebugInfoMetadata.h"
16#include "llvm/MC/TargetRegistry.h"
17#include "llvm/Object/COFF.h"
18#include "llvm/Support/CommandLine.h"
19#include "llvm/Support/Debug.h"
20#include "llvm/Support/Format.h"
21#include "llvm/Support/TargetSelect.h"
22#include "llvm/TargetParser/Triple.h"
23#include <optional>
24
25#define DEBUG_TYPE "load-binary"
26
27using namespace llvm;
28using namespace sampleprof;
29
30cl::opt<bool> ShowDisassemblyOnly("show-disassembly-only",
31 cl::desc("Print disassembled code."));
32
33cl::opt<bool> ShowSourceLocations("show-source-locations",
34 cl::desc("Print source locations."));
35
36static cl::opt<bool>
37 ShowCanonicalFnName("show-canonical-fname",
38 cl::desc("Print canonical function name."));
39
40static cl::opt<bool> ShowPseudoProbe(
41 "show-pseudo-probe",
42 cl::desc("Print pseudo probe section and disassembled info."));
43
44static cl::opt<bool> UseDwarfCorrelation(
45 "use-dwarf-correlation",
46 cl::desc("Use dwarf for profile correlation even when binary contains "
47 "pseudo probe."));
48
49static cl::opt<std::string>
50 DWPPath("dwp", cl::init(Val: ""),
51 cl::desc("Path of .dwp file. When not specified, it will be "
52 "<binary>.dwp in the same directory as the main binary."));
53
54static cl::list<std::string> DisassembleFunctions(
55 "disassemble-functions", cl::CommaSeparated,
56 cl::desc("List of functions to print disassembly for. Accept demangled "
57 "names only. Only work with show-disassembly-only"));
58
59static cl::opt<bool>
60 KernelBinary("kernel",
61 cl::desc("Generate the profile for Linux kernel binary."));
62
63extern cl::opt<bool> ShowDetailedWarning;
64extern cl::opt<bool> InferMissingFrames;
65
66namespace llvm {
67namespace sampleprof {
68
69static const Target *getTarget(const ObjectFile *Obj) {
70 Triple TheTriple = Obj->makeTriple();
71 std::string Error;
72 std::string ArchName;
73 const Target *TheTarget =
74 TargetRegistry::lookupTarget(ArchName, TheTriple, Error);
75 if (!TheTarget)
76 exitWithError(Message: Error, Whence: Obj->getFileName());
77 return TheTarget;
78}
79
80void BinarySizeContextTracker::addInstructionForContext(
81 const SampleContextFrameVector &Context, uint32_t InstrSize) {
82 ContextTrieNode *CurNode = &RootContext;
83 bool IsLeaf = true;
84 for (const auto &Callsite : reverse(C: Context)) {
85 FunctionId CallerName = Callsite.Func;
86 LineLocation CallsiteLoc = IsLeaf ? LineLocation(0, 0) : Callsite.Location;
87 CurNode = CurNode->getOrCreateChildContext(CallSite: CallsiteLoc, ChildName: CallerName);
88 IsLeaf = false;
89 }
90
91 CurNode->addFunctionSize(FSize: InstrSize);
92}
93
94uint32_t
95BinarySizeContextTracker::getFuncSizeForContext(const ContextTrieNode *Node) {
96 ContextTrieNode *CurrNode = &RootContext;
97 ContextTrieNode *PrevNode = nullptr;
98
99 std::optional<uint32_t> Size;
100
101 // Start from top-level context-less function, traverse down the reverse
102 // context trie to find the best/longest match for given context, then
103 // retrieve the size.
104 LineLocation CallSiteLoc(0, 0);
105 while (CurrNode && Node->getParentContext() != nullptr) {
106 PrevNode = CurrNode;
107 CurrNode = CurrNode->getChildContext(CallSite: CallSiteLoc, ChildName: Node->getFuncName());
108 if (CurrNode && CurrNode->getFunctionSize())
109 Size = *CurrNode->getFunctionSize();
110 CallSiteLoc = Node->getCallSiteLoc();
111 Node = Node->getParentContext();
112 }
113
114 // If we traversed all nodes along the path of the context and haven't
115 // found a size yet, pivot to look for size from sibling nodes, i.e size
116 // of inlinee under different context.
117 if (!Size) {
118 if (!CurrNode)
119 CurrNode = PrevNode;
120 while (!Size && CurrNode && !CurrNode->getAllChildContext().empty()) {
121 CurrNode = &CurrNode->getAllChildContext().begin()->second;
122 if (CurrNode->getFunctionSize())
123 Size = *CurrNode->getFunctionSize();
124 }
125 }
126
127 assert(Size && "We should at least find one context size.");
128 return *Size;
129}
130
131void BinarySizeContextTracker::trackInlineesOptimizedAway(
132 MCPseudoProbeDecoder &ProbeDecoder) {
133 ProbeFrameStack ProbeContext;
134 for (const auto &Child : ProbeDecoder.getDummyInlineRoot().getChildren())
135 trackInlineesOptimizedAway(ProbeDecoder, ProbeNode&: *Child.second, Context&: ProbeContext);
136}
137
138void BinarySizeContextTracker::trackInlineesOptimizedAway(
139 MCPseudoProbeDecoder &ProbeDecoder,
140 MCDecodedPseudoProbeInlineTree &ProbeNode, ProbeFrameStack &ProbeContext) {
141 StringRef FuncName =
142 ProbeDecoder.getFuncDescForGUID(GUID: ProbeNode.Guid)->FuncName;
143 ProbeContext.emplace_back(Args&: FuncName, Args: 0);
144
145 // This ProbeContext has a probe, so it has code before inlining and
146 // optimization. Make sure we mark its size as known.
147 if (!ProbeNode.getProbes().empty()) {
148 ContextTrieNode *SizeContext = &RootContext;
149 for (auto &ProbeFrame : reverse(C&: ProbeContext)) {
150 StringRef CallerName = ProbeFrame.first;
151 LineLocation CallsiteLoc(ProbeFrame.second, 0);
152 SizeContext =
153 SizeContext->getOrCreateChildContext(CallSite: CallsiteLoc,
154 ChildName: FunctionId(CallerName));
155 }
156 // Add 0 size to make known.
157 SizeContext->addFunctionSize(FSize: 0);
158 }
159
160 // DFS down the probe inline tree
161 for (const auto &ChildNode : ProbeNode.getChildren()) {
162 InlineSite Location = ChildNode.first;
163 ProbeContext.back().second = std::get<1>(t&: Location);
164 trackInlineesOptimizedAway(ProbeDecoder, ProbeNode&: *ChildNode.second, ProbeContext);
165 }
166
167 ProbeContext.pop_back();
168}
169
170ProfiledBinary::ProfiledBinary(const StringRef ExeBinPath,
171 const StringRef DebugBinPath)
172 : Path(ExeBinPath), DebugBinaryPath(DebugBinPath),
173 SymbolizerOpts(getSymbolizerOpts()), ProEpilogTracker(this),
174 Symbolizer(std::make_unique<symbolize::LLVMSymbolizer>(args&: SymbolizerOpts)),
175 TrackFuncContextSize(EnableCSPreInliner && UseContextCostForPreInliner) {
176 // Point to executable binary if debug info binary is not specified.
177 SymbolizerPath = DebugBinPath.empty() ? ExeBinPath : DebugBinPath;
178 if (InferMissingFrames)
179 MissingContextInferrer = std::make_unique<MissingFrameInferrer>(args: this);
180 load();
181}
182
183ProfiledBinary::~ProfiledBinary() {}
184
185void ProfiledBinary::warnNoFuncEntry() {
186 uint64_t NoFuncEntryNum = 0;
187 for (auto &F : BinaryFunctions) {
188 if (F.second.Ranges.empty())
189 continue;
190 bool hasFuncEntry = false;
191 for (auto &R : F.second.Ranges) {
192 if (FuncRange *FR = findFuncRangeForStartAddr(Address: R.first)) {
193 if (FR->IsFuncEntry) {
194 hasFuncEntry = true;
195 break;
196 }
197 }
198 }
199
200 if (!hasFuncEntry) {
201 NoFuncEntryNum++;
202 if (ShowDetailedWarning)
203 WithColor::warning()
204 << "Failed to determine function entry for " << F.first
205 << " due to inconsistent name from symbol table and dwarf info.\n";
206 }
207 }
208 emitWarningSummary(Num: NoFuncEntryNum, Total: BinaryFunctions.size(),
209 Msg: "of functions failed to determine function entry due to "
210 "inconsistent name from symbol table and dwarf info.");
211}
212
213void ProfiledBinary::load() {
214 // Attempt to open the binary.
215 OwningBinary<Binary> OBinary = unwrapOrError(EO: createBinary(Path), Args&: Path);
216 Binary &ExeBinary = *OBinary.getBinary();
217
218 IsCOFF = isa<COFFObjectFile>(Val: &ExeBinary);
219 if (!isa<ELFObjectFileBase>(Val: &ExeBinary) && !IsCOFF)
220 exitWithError(Message: "not a valid ELF/COFF image", Whence: Path);
221
222 auto *Obj = cast<ObjectFile>(Val: &ExeBinary);
223 TheTriple = Obj->makeTriple();
224
225 LLVM_DEBUG(dbgs() << "Loading " << Path << "\n");
226
227 // Mark the binary as a kernel image;
228 IsKernel = KernelBinary;
229
230 // Find the preferred load address for text sections.
231 setPreferredTextSegmentAddresses(Obj);
232
233 // Load debug info of subprograms from DWARF section.
234 // If path of debug info binary is specified, use the debug info from it,
235 // otherwise use the debug info from the executable binary.
236 if (!DebugBinaryPath.empty()) {
237 OwningBinary<Binary> DebugPath =
238 unwrapOrError(EO: createBinary(Path: DebugBinaryPath), Args&: DebugBinaryPath);
239 loadSymbolsFromDWARF(Obj&: *cast<ObjectFile>(Val: DebugPath.getBinary()));
240 } else {
241 loadSymbolsFromDWARF(Obj&: *cast<ObjectFile>(Val: &ExeBinary));
242 }
243
244 DisassembleFunctionSet.insert(begin: DisassembleFunctions.begin(),
245 end: DisassembleFunctions.end());
246
247 if (auto *ELFObj = dyn_cast<ELFObjectFileBase>(Val: Obj)) {
248 checkPseudoProbe(Obj: ELFObj);
249 if (UsePseudoProbes)
250 populateElfSymbolAddressList(O: ELFObj);
251
252 if (ShowDisassemblyOnly)
253 decodePseudoProbe(Obj: ELFObj);
254 }
255
256 // Disassemble the text sections.
257 disassemble(O: Obj);
258
259 // Use function start and return address to infer prolog and epilog
260 ProEpilogTracker.inferPrologAddresses(FuncStartAddressMap&: StartAddrToFuncRangeMap);
261 ProEpilogTracker.inferEpilogAddresses(RetAddrs&: RetAddressSet);
262
263 warnNoFuncEntry();
264
265 // TODO: decode other sections.
266}
267
268bool ProfiledBinary::inlineContextEqual(uint64_t Address1, uint64_t Address2) {
269 const SampleContextFrameVector &Context1 =
270 getCachedFrameLocationStack(Address: Address1);
271 const SampleContextFrameVector &Context2 =
272 getCachedFrameLocationStack(Address: Address2);
273 if (Context1.size() != Context2.size())
274 return false;
275 if (Context1.empty())
276 return false;
277 // The leaf frame contains location within the leaf, and it
278 // needs to be remove that as it's not part of the calling context
279 return std::equal(first1: Context1.begin(), last1: Context1.begin() + Context1.size() - 1,
280 first2: Context2.begin(), last2: Context2.begin() + Context2.size() - 1);
281}
282
283SampleContextFrameVector
284ProfiledBinary::getExpandedContext(const SmallVectorImpl<uint64_t> &Stack,
285 bool &WasLeafInlined) {
286 SampleContextFrameVector ContextVec;
287 if (Stack.empty())
288 return ContextVec;
289 // Process from frame root to leaf
290 for (auto Address : Stack) {
291 const SampleContextFrameVector &ExpandedContext =
292 getCachedFrameLocationStack(Address);
293 // An instruction without a valid debug line will be ignored by sample
294 // processing
295 if (ExpandedContext.empty())
296 return SampleContextFrameVector();
297 // Set WasLeafInlined to the size of inlined frame count for the last
298 // address which is leaf
299 WasLeafInlined = (ExpandedContext.size() > 1);
300 ContextVec.append(RHS: ExpandedContext);
301 }
302
303 // Replace with decoded base discriminator
304 for (auto &Frame : ContextVec) {
305 Frame.Location.Discriminator = ProfileGeneratorBase::getBaseDiscriminator(
306 Discriminator: Frame.Location.Discriminator, UseFSD: UseFSDiscriminator);
307 }
308
309 assert(ContextVec.size() && "Context length should be at least 1");
310
311 // Compress the context string except for the leaf frame
312 auto LeafFrame = ContextVec.back();
313 LeafFrame.Location = LineLocation(0, 0);
314 ContextVec.pop_back();
315 CSProfileGenerator::compressRecursionContext(Context&: ContextVec);
316 CSProfileGenerator::trimContext(S&: ContextVec);
317 ContextVec.push_back(Elt: LeafFrame);
318 return ContextVec;
319}
320
321template <class ELFT>
322void ProfiledBinary::setPreferredTextSegmentAddresses(const ELFFile<ELFT> &Obj,
323 StringRef FileName) {
324 const auto &PhdrRange = unwrapOrError(Obj.program_headers(), FileName);
325 // FIXME: This should be the page size of the system running profiling.
326 // However such info isn't available at post-processing time, assuming
327 // 4K page now. Note that we don't use EXEC_PAGESIZE from <linux/param.h>
328 // because we may build the tools on non-linux.
329 uint64_t PageSize = 0x1000;
330 for (const typename ELFT::Phdr &Phdr : PhdrRange) {
331 if (Phdr.p_type == ELF::PT_LOAD) {
332 if (!FirstLoadableAddress)
333 FirstLoadableAddress = Phdr.p_vaddr & ~(PageSize - 1U);
334 if (Phdr.p_flags & ELF::PF_X) {
335 // Segments will always be loaded at a page boundary.
336 PreferredTextSegmentAddresses.push_back(Phdr.p_vaddr &
337 ~(PageSize - 1U));
338 TextSegmentOffsets.push_back(Phdr.p_offset & ~(PageSize - 1U));
339 }
340 }
341 }
342
343 if (PreferredTextSegmentAddresses.empty())
344 exitWithError(Message: "no executable segment found", Whence: FileName);
345}
346
347void ProfiledBinary::setPreferredTextSegmentAddresses(const COFFObjectFile *Obj,
348 StringRef FileName) {
349 uint64_t ImageBase = Obj->getImageBase();
350 if (!ImageBase)
351 exitWithError(Message: "Not a COFF image", Whence: FileName);
352
353 PreferredTextSegmentAddresses.push_back(x: ImageBase);
354 FirstLoadableAddress = ImageBase;
355
356 for (SectionRef Section : Obj->sections()) {
357 const coff_section *Sec = Obj->getCOFFSection(Section);
358 if (Sec->Characteristics & COFF::IMAGE_SCN_CNT_CODE)
359 TextSegmentOffsets.push_back(x: Sec->VirtualAddress);
360 }
361}
362
363void ProfiledBinary::setPreferredTextSegmentAddresses(const ObjectFile *Obj) {
364 if (const auto *ELFObj = dyn_cast<ELF32LEObjectFile>(Val: Obj))
365 setPreferredTextSegmentAddresses(Obj: ELFObj->getELFFile(), FileName: Obj->getFileName());
366 else if (const auto *ELFObj = dyn_cast<ELF32BEObjectFile>(Val: Obj))
367 setPreferredTextSegmentAddresses(Obj: ELFObj->getELFFile(), FileName: Obj->getFileName());
368 else if (const auto *ELFObj = dyn_cast<ELF64LEObjectFile>(Val: Obj))
369 setPreferredTextSegmentAddresses(Obj: ELFObj->getELFFile(), FileName: Obj->getFileName());
370 else if (const auto *ELFObj = dyn_cast<ELF64BEObjectFile>(Val: Obj))
371 setPreferredTextSegmentAddresses(Obj: ELFObj->getELFFile(), FileName: Obj->getFileName());
372 else if (const auto *COFFObj = dyn_cast<COFFObjectFile>(Val: Obj))
373 setPreferredTextSegmentAddresses(Obj: COFFObj, FileName: Obj->getFileName());
374 else
375 llvm_unreachable("invalid object format");
376}
377
378void ProfiledBinary::checkPseudoProbe(const ELFObjectFileBase *Obj) {
379 if (UseDwarfCorrelation)
380 return;
381
382 bool HasProbeDescSection = false;
383 bool HasPseudoProbeSection = false;
384
385 StringRef FileName = Obj->getFileName();
386 for (section_iterator SI = Obj->section_begin(), SE = Obj->section_end();
387 SI != SE; ++SI) {
388 const SectionRef &Section = *SI;
389 StringRef SectionName = unwrapOrError(EO: Section.getName(), Args&: FileName);
390 if (SectionName == ".pseudo_probe_desc") {
391 HasProbeDescSection = true;
392 } else if (SectionName == ".pseudo_probe") {
393 HasPseudoProbeSection = true;
394 }
395 }
396
397 // set UsePseudoProbes flag, used for PerfReader
398 UsePseudoProbes = HasProbeDescSection && HasPseudoProbeSection;
399}
400
401void ProfiledBinary::decodePseudoProbe(const ELFObjectFileBase *Obj) {
402 if (!UsePseudoProbes)
403 return;
404
405 MCPseudoProbeDecoder::Uint64Set GuidFilter;
406 MCPseudoProbeDecoder::Uint64Map FuncStartAddresses;
407 if (ShowDisassemblyOnly) {
408 if (DisassembleFunctionSet.empty()) {
409 FuncStartAddresses = SymbolStartAddrs;
410 } else {
411 for (auto &F : DisassembleFunctionSet) {
412 auto GUID = Function::getGUID(GlobalName: F.first());
413 if (auto StartAddr = SymbolStartAddrs.lookup(Val: GUID)) {
414 FuncStartAddresses[GUID] = StartAddr;
415 FuncRange &Range = StartAddrToFuncRangeMap[StartAddr];
416 GuidFilter.insert(V: Function::getGUID(GlobalName: Range.getFuncName()));
417 }
418 }
419 }
420 } else {
421 for (auto *F : ProfiledFunctions) {
422 GuidFilter.insert(V: Function::getGUID(GlobalName: F->FuncName));
423 for (auto &Range : F->Ranges) {
424 auto GUIDs = StartAddrToSymMap.equal_range(x: Range.first);
425 for (auto I = GUIDs.first; I != GUIDs.second; ++I)
426 FuncStartAddresses[I->second] = I->first;
427 }
428 }
429 }
430
431 StringRef FileName = Obj->getFileName();
432 for (section_iterator SI = Obj->section_begin(), SE = Obj->section_end();
433 SI != SE; ++SI) {
434 const SectionRef &Section = *SI;
435 StringRef SectionName = unwrapOrError(EO: Section.getName(), Args&: FileName);
436
437 if (SectionName == ".pseudo_probe_desc") {
438 StringRef Contents = unwrapOrError(EO: Section.getContents(), Args&: FileName);
439 if (!ProbeDecoder.buildGUID2FuncDescMap(
440 Start: reinterpret_cast<const uint8_t *>(Contents.data()),
441 Size: Contents.size()))
442 exitWithError(
443 Message: "Pseudo Probe decoder fail in .pseudo_probe_desc section");
444 } else if (SectionName == ".pseudo_probe") {
445 StringRef Contents = unwrapOrError(EO: Section.getContents(), Args&: FileName);
446 if (!ProbeDecoder.buildAddress2ProbeMap(
447 Start: reinterpret_cast<const uint8_t *>(Contents.data()),
448 Size: Contents.size(), GuildFilter: GuidFilter, FuncStartAddrs: FuncStartAddresses))
449 exitWithError(Message: "Pseudo Probe decoder fail in .pseudo_probe section");
450 }
451 }
452
453 // Build TopLevelProbeFrameMap to track size for optimized inlinees when probe
454 // is available
455 if (TrackFuncContextSize) {
456 for (const auto &Child : ProbeDecoder.getDummyInlineRoot().getChildren()) {
457 auto *Frame = Child.second.get();
458 StringRef FuncName =
459 ProbeDecoder.getFuncDescForGUID(GUID: Frame->Guid)->FuncName;
460 TopLevelProbeFrameMap[FuncName] = Frame;
461 }
462 }
463
464 if (ShowPseudoProbe)
465 ProbeDecoder.printGUID2FuncDescMap(OS&: outs());
466}
467
468void ProfiledBinary::decodePseudoProbe() {
469 OwningBinary<Binary> OBinary = unwrapOrError(EO: createBinary(Path), Args&: Path);
470 Binary &ExeBinary = *OBinary.getBinary();
471 auto *Obj = cast<ELFObjectFileBase>(Val: &ExeBinary);
472 decodePseudoProbe(Obj);
473}
474
475void ProfiledBinary::setIsFuncEntry(FuncRange *FuncRange,
476 StringRef RangeSymName) {
477 // Skip external function symbol.
478 if (!FuncRange)
479 return;
480
481 // Set IsFuncEntry to ture if there is only one range in the function or the
482 // RangeSymName from ELF is equal to its DWARF-based function name.
483 if (FuncRange->Func->Ranges.size() == 1 ||
484 (!FuncRange->IsFuncEntry && FuncRange->getFuncName() == RangeSymName))
485 FuncRange->IsFuncEntry = true;
486}
487
488bool ProfiledBinary::dissassembleSymbol(std::size_t SI, ArrayRef<uint8_t> Bytes,
489 SectionSymbolsTy &Symbols,
490 const SectionRef &Section) {
491 std::size_t SE = Symbols.size();
492 uint64_t SectionAddress = Section.getAddress();
493 uint64_t SectSize = Section.getSize();
494 uint64_t StartAddress = Symbols[SI].Addr;
495 uint64_t NextStartAddress =
496 (SI + 1 < SE) ? Symbols[SI + 1].Addr : SectionAddress + SectSize;
497 FuncRange *FRange = findFuncRange(Address: StartAddress);
498 setIsFuncEntry(FuncRange: FRange, RangeSymName: FunctionSamples::getCanonicalFnName(FnName: Symbols[SI].Name));
499 StringRef SymbolName =
500 ShowCanonicalFnName
501 ? FunctionSamples::getCanonicalFnName(FnName: Symbols[SI].Name)
502 : Symbols[SI].Name;
503 bool ShowDisassembly =
504 ShowDisassemblyOnly && (DisassembleFunctionSet.empty() ||
505 DisassembleFunctionSet.count(Key: SymbolName));
506 if (ShowDisassembly)
507 outs() << '<' << SymbolName << ">:\n";
508
509 uint64_t Address = StartAddress;
510 // Size of a consecutive invalid instruction range starting from Address -1
511 // backwards.
512 uint64_t InvalidInstLength = 0;
513 while (Address < NextStartAddress) {
514 MCInst Inst;
515 uint64_t Size;
516 // Disassemble an instruction.
517 bool Disassembled = DisAsm->getInstruction(
518 Instr&: Inst, Size, Bytes: Bytes.slice(N: Address - SectionAddress), Address, CStream&: nulls());
519 if (Size == 0)
520 Size = 1;
521
522 if (ShowDisassembly) {
523 if (ShowPseudoProbe) {
524 ProbeDecoder.printProbeForAddress(OS&: outs(), Address);
525 }
526 outs() << format(Fmt: "%8" PRIx64 ":", Vals: Address);
527 size_t Start = outs().tell();
528 if (Disassembled)
529 IPrinter->printInst(MI: &Inst, Address: Address + Size, Annot: "", STI: *STI, OS&: outs());
530 else
531 outs() << "\t<unknown>";
532 if (ShowSourceLocations) {
533 unsigned Cur = outs().tell() - Start;
534 if (Cur < 40)
535 outs().indent(NumSpaces: 40 - Cur);
536 InstructionPointer IP(this, Address);
537 outs() << getReversedLocWithContext(
538 Context: symbolize(IP, UseCanonicalFnName: ShowCanonicalFnName, UseProbeDiscriminator: ShowPseudoProbe));
539 }
540 outs() << "\n";
541 }
542
543 if (Disassembled) {
544 const MCInstrDesc &MCDesc = MII->get(Opcode: Inst.getOpcode());
545
546 // Record instruction size.
547 AddressToInstSizeMap[Address] = Size;
548
549 // Populate address maps.
550 CodeAddressVec.push_back(x: Address);
551 if (MCDesc.isCall()) {
552 CallAddressSet.insert(x: Address);
553 UncondBranchAddrSet.insert(x: Address);
554 } else if (MCDesc.isReturn()) {
555 RetAddressSet.insert(x: Address);
556 UncondBranchAddrSet.insert(x: Address);
557 } else if (MCDesc.isBranch()) {
558 if (MCDesc.isUnconditionalBranch())
559 UncondBranchAddrSet.insert(x: Address);
560 BranchAddressSet.insert(x: Address);
561 }
562
563 // Record potential call targets for tail frame inference later-on.
564 if (InferMissingFrames && FRange) {
565 uint64_t Target = 0;
566 MIA->evaluateBranch(Inst, Addr: Address, Size, Target);
567 if (MCDesc.isCall()) {
568 // Indirect call targets are unknown at this point. Recording the
569 // unknown target (zero) for further LBR-based refinement.
570 MissingContextInferrer->CallEdges[Address].insert(x: Target);
571 } else if (MCDesc.isUnconditionalBranch()) {
572 assert(Target &&
573 "target should be known for unconditional direct branch");
574 // Any inter-function unconditional jump is considered tail call at
575 // this point. This is not 100% accurate and could further be
576 // optimized based on some source annotation.
577 FuncRange *ToFRange = findFuncRange(Address: Target);
578 if (ToFRange && ToFRange->Func != FRange->Func)
579 MissingContextInferrer->TailCallEdges[Address].insert(x: Target);
580 LLVM_DEBUG({
581 dbgs() << "Direct Tail call: " << format("%8" PRIx64 ":", Address);
582 IPrinter->printInst(&Inst, Address + Size, "", *STI.get(), dbgs());
583 dbgs() << "\n";
584 });
585 } else if (MCDesc.isIndirectBranch() && MCDesc.isBarrier()) {
586 // This is an indirect branch but not necessarily an indirect tail
587 // call. The isBarrier check is to filter out conditional branch.
588 // Similar with indirect call targets, recording the unknown target
589 // (zero) for further LBR-based refinement.
590 MissingContextInferrer->TailCallEdges[Address].insert(x: Target);
591 LLVM_DEBUG({
592 dbgs() << "Indirect Tail call: "
593 << format("%8" PRIx64 ":", Address);
594 IPrinter->printInst(&Inst, Address + Size, "", *STI.get(), dbgs());
595 dbgs() << "\n";
596 });
597 }
598 }
599
600 if (InvalidInstLength) {
601 AddrsWithInvalidInstruction.insert(
602 V: {Address - InvalidInstLength, Address - 1});
603 InvalidInstLength = 0;
604 }
605 } else {
606 InvalidInstLength += Size;
607 }
608
609 Address += Size;
610 }
611
612 if (InvalidInstLength)
613 AddrsWithInvalidInstruction.insert(
614 V: {Address - InvalidInstLength, Address - 1});
615
616 if (ShowDisassembly)
617 outs() << "\n";
618
619 return true;
620}
621
622void ProfiledBinary::setUpDisassembler(const ObjectFile *Obj) {
623 const Target *TheTarget = getTarget(Obj);
624 std::string TripleName = TheTriple.getTriple();
625 StringRef FileName = Obj->getFileName();
626
627 MRI.reset(p: TheTarget->createMCRegInfo(TT: TripleName));
628 if (!MRI)
629 exitWithError(Message: "no register info for target " + TripleName, Whence: FileName);
630
631 MCTargetOptions MCOptions;
632 AsmInfo.reset(p: TheTarget->createMCAsmInfo(MRI: *MRI, TheTriple: TripleName, Options: MCOptions));
633 if (!AsmInfo)
634 exitWithError(Message: "no assembly info for target " + TripleName, Whence: FileName);
635
636 Expected<SubtargetFeatures> Features = Obj->getFeatures();
637 if (!Features)
638 exitWithError(E: Features.takeError(), Whence: FileName);
639 STI.reset(
640 p: TheTarget->createMCSubtargetInfo(TheTriple: TripleName, CPU: "", Features: Features->getString()));
641 if (!STI)
642 exitWithError(Message: "no subtarget info for target " + TripleName, Whence: FileName);
643
644 MII.reset(p: TheTarget->createMCInstrInfo());
645 if (!MII)
646 exitWithError(Message: "no instruction info for target " + TripleName, Whence: FileName);
647
648 MCContext Ctx(Triple(TripleName), AsmInfo.get(), MRI.get(), STI.get());
649 std::unique_ptr<MCObjectFileInfo> MOFI(
650 TheTarget->createMCObjectFileInfo(Ctx, /*PIC=*/false));
651 Ctx.setObjectFileInfo(MOFI.get());
652 DisAsm.reset(p: TheTarget->createMCDisassembler(STI: *STI, Ctx));
653 if (!DisAsm)
654 exitWithError(Message: "no disassembler for target " + TripleName, Whence: FileName);
655
656 MIA.reset(p: TheTarget->createMCInstrAnalysis(Info: MII.get()));
657
658 int AsmPrinterVariant = AsmInfo->getAssemblerDialect();
659 IPrinter.reset(p: TheTarget->createMCInstPrinter(
660 T: Triple(TripleName), SyntaxVariant: AsmPrinterVariant, MAI: *AsmInfo, MII: *MII, MRI: *MRI));
661 IPrinter->setPrintBranchImmAsAddress(true);
662}
663
664void ProfiledBinary::disassemble(const ObjectFile *Obj) {
665 // Set up disassembler and related components.
666 setUpDisassembler(Obj);
667
668 // Create a mapping from virtual address to symbol name. The symbols in text
669 // sections are the candidates to dissassemble.
670 std::map<SectionRef, SectionSymbolsTy> AllSymbols;
671 StringRef FileName = Obj->getFileName();
672 for (const SymbolRef &Symbol : Obj->symbols()) {
673 const uint64_t Addr = unwrapOrError(EO: Symbol.getAddress(), Args&: FileName);
674 const StringRef Name = unwrapOrError(EO: Symbol.getName(), Args&: FileName);
675 section_iterator SecI = unwrapOrError(EO: Symbol.getSection(), Args&: FileName);
676 if (SecI != Obj->section_end())
677 AllSymbols[*SecI].push_back(x: SymbolInfoTy(Addr, Name, ELF::STT_NOTYPE));
678 }
679
680 // Sort all the symbols. Use a stable sort to stabilize the output.
681 for (std::pair<const SectionRef, SectionSymbolsTy> &SecSyms : AllSymbols)
682 stable_sort(Range&: SecSyms.second);
683
684 assert((DisassembleFunctionSet.empty() || ShowDisassemblyOnly) &&
685 "Functions to disassemble should be only specified together with "
686 "--show-disassembly-only");
687
688 if (ShowDisassemblyOnly)
689 outs() << "\nDisassembly of " << FileName << ":\n";
690
691 // Dissassemble a text section.
692 for (section_iterator SI = Obj->section_begin(), SE = Obj->section_end();
693 SI != SE; ++SI) {
694 const SectionRef &Section = *SI;
695 if (!Section.isText())
696 continue;
697
698 uint64_t ImageLoadAddr = getPreferredBaseAddress();
699 uint64_t SectionAddress = Section.getAddress() - ImageLoadAddr;
700 uint64_t SectSize = Section.getSize();
701 if (!SectSize)
702 continue;
703
704 // Register the text section.
705 TextSections.insert(x: {SectionAddress, SectSize});
706
707 StringRef SectionName = unwrapOrError(EO: Section.getName(), Args&: FileName);
708
709 if (ShowDisassemblyOnly) {
710 outs() << "\nDisassembly of section " << SectionName;
711 outs() << " [" << format(Fmt: "0x%" PRIx64, Vals: Section.getAddress()) << ", "
712 << format(Fmt: "0x%" PRIx64, Vals: Section.getAddress() + SectSize)
713 << "]:\n\n";
714 }
715
716 if (isa<ELFObjectFileBase>(Val: Obj) && SectionName == ".plt")
717 continue;
718
719 // Get the section data.
720 ArrayRef<uint8_t> Bytes =
721 arrayRefFromStringRef(Input: unwrapOrError(EO: Section.getContents(), Args&: FileName));
722
723 // Get the list of all the symbols in this section.
724 SectionSymbolsTy &Symbols = AllSymbols[Section];
725
726 // Disassemble symbol by symbol.
727 for (std::size_t SI = 0, SE = Symbols.size(); SI != SE; ++SI) {
728 if (!dissassembleSymbol(SI, Bytes, Symbols, Section))
729 exitWithError(Message: "disassembling error", Whence: FileName);
730 }
731 }
732
733 if (!AddrsWithInvalidInstruction.empty()) {
734 if (ShowDetailedWarning) {
735 for (auto &Addr : AddrsWithInvalidInstruction) {
736 WithColor::warning()
737 << "Invalid instructions at " << format(Fmt: "%8" PRIx64, Vals: Addr.first)
738 << " - " << format(Fmt: "%8" PRIx64, Vals: Addr.second) << "\n";
739 }
740 }
741 WithColor::warning() << "Found " << AddrsWithInvalidInstruction.size()
742 << " invalid instructions\n";
743 AddrsWithInvalidInstruction.clear();
744 }
745
746 // Dissassemble rodata section to check if FS discriminator symbol exists.
747 checkUseFSDiscriminator(Obj, AllSymbols);
748}
749
750void ProfiledBinary::checkUseFSDiscriminator(
751 const ObjectFile *Obj, std::map<SectionRef, SectionSymbolsTy> &AllSymbols) {
752 const char *FSDiscriminatorVar = "__llvm_fs_discriminator__";
753 for (section_iterator SI = Obj->section_begin(), SE = Obj->section_end();
754 SI != SE; ++SI) {
755 const SectionRef &Section = *SI;
756 if (!Section.isData() || Section.getSize() == 0)
757 continue;
758 SectionSymbolsTy &Symbols = AllSymbols[Section];
759
760 for (std::size_t SI = 0, SE = Symbols.size(); SI != SE; ++SI) {
761 if (Symbols[SI].Name == FSDiscriminatorVar) {
762 UseFSDiscriminator = true;
763 return;
764 }
765 }
766 }
767}
768
769void ProfiledBinary::populateElfSymbolAddressList(
770 const ELFObjectFileBase *Obj) {
771 // Create a mapping from virtual address to symbol GUID and the other way
772 // around.
773 StringRef FileName = Obj->getFileName();
774 for (const SymbolRef &Symbol : Obj->symbols()) {
775 const uint64_t Addr = unwrapOrError(EO: Symbol.getAddress(), Args&: FileName);
776 const StringRef Name = unwrapOrError(EO: Symbol.getName(), Args&: FileName);
777 uint64_t GUID = Function::getGUID(GlobalName: Name);
778 SymbolStartAddrs[GUID] = Addr;
779 StartAddrToSymMap.emplace(args: Addr, args&: GUID);
780 }
781}
782
783void ProfiledBinary::loadSymbolsFromDWARFUnit(DWARFUnit &CompilationUnit) {
784 for (const auto &DieInfo : CompilationUnit.dies()) {
785 llvm::DWARFDie Die(&CompilationUnit, &DieInfo);
786
787 if (!Die.isSubprogramDIE())
788 continue;
789 auto Name = Die.getName(Kind: llvm::DINameKind::LinkageName);
790 if (!Name)
791 Name = Die.getName(Kind: llvm::DINameKind::ShortName);
792 if (!Name)
793 continue;
794
795 auto RangesOrError = Die.getAddressRanges();
796 if (!RangesOrError)
797 continue;
798 const DWARFAddressRangesVector &Ranges = RangesOrError.get();
799
800 if (Ranges.empty())
801 continue;
802
803 // Different DWARF symbols can have same function name, search or create
804 // BinaryFunction indexed by the name.
805 auto Ret = BinaryFunctions.emplace(args&: Name, args: BinaryFunction());
806 auto &Func = Ret.first->second;
807 if (Ret.second)
808 Func.FuncName = Ret.first->first;
809
810 for (const auto &Range : Ranges) {
811 uint64_t StartAddress = Range.LowPC;
812 uint64_t EndAddress = Range.HighPC;
813
814 if (EndAddress <= StartAddress ||
815 StartAddress < getPreferredBaseAddress())
816 continue;
817
818 // We may want to know all ranges for one function. Here group the
819 // ranges and store them into BinaryFunction.
820 Func.Ranges.emplace_back(args&: StartAddress, args&: EndAddress);
821
822 auto R = StartAddrToFuncRangeMap.emplace(args&: StartAddress, args: FuncRange());
823 if (R.second) {
824 FuncRange &FRange = R.first->second;
825 FRange.Func = &Func;
826 FRange.StartAddress = StartAddress;
827 FRange.EndAddress = EndAddress;
828 } else {
829 AddrsWithMultipleSymbols.insert(V: StartAddress);
830 if (ShowDetailedWarning)
831 WithColor::warning()
832 << "Duplicated symbol start address at "
833 << format(Fmt: "%8" PRIx64, Vals: StartAddress) << " "
834 << R.first->second.getFuncName() << " and " << Name << "\n";
835 }
836 }
837 }
838}
839
840void ProfiledBinary::loadSymbolsFromDWARF(ObjectFile &Obj) {
841 auto DebugContext = llvm::DWARFContext::create(
842 Obj, RelocAction: DWARFContext::ProcessDebugRelocations::Process, L: nullptr, DWPName: DWPPath);
843 if (!DebugContext)
844 exitWithError(Message: "Error creating the debug info context", Whence: Path);
845
846 for (const auto &CompilationUnit : DebugContext->compile_units())
847 loadSymbolsFromDWARFUnit(CompilationUnit&: *CompilationUnit);
848
849 // Handles DWO sections that can either be in .o, .dwo or .dwp files.
850 uint32_t NumOfDWOMissing = 0;
851 for (const auto &CompilationUnit : DebugContext->compile_units()) {
852 DWARFUnit *const DwarfUnit = CompilationUnit.get();
853 if (DwarfUnit->getDWOId()) {
854 DWARFUnit *DWOCU = DwarfUnit->getNonSkeletonUnitDIE(ExtractUnitDIEOnly: false).getDwarfUnit();
855 if (!DWOCU->isDWOUnit()) {
856 NumOfDWOMissing++;
857 if (ShowDetailedWarning) {
858 std::string DWOName = dwarf::toString(
859 V: DwarfUnit->getUnitDIE().find(
860 Attrs: {dwarf::DW_AT_dwo_name, dwarf::DW_AT_GNU_dwo_name}),
861 Default: "");
862 WithColor::warning() << "DWO debug information for " << DWOName
863 << " was not loaded.\n";
864 }
865 continue;
866 }
867 loadSymbolsFromDWARFUnit(CompilationUnit&: *DWOCU);
868 }
869 }
870
871 if (NumOfDWOMissing)
872 WithColor::warning()
873 << " DWO debug information was not loaded for " << NumOfDWOMissing
874 << " modules. Please check the .o, .dwo or .dwp path.\n";
875 if (BinaryFunctions.empty())
876 WithColor::warning() << "Loading of DWARF info completed, but no binary "
877 "functions have been retrieved.\n";
878 // Populate the hash binary function map for MD5 function name lookup. This
879 // is done after BinaryFunctions are finalized.
880 for (auto &BinaryFunction : BinaryFunctions) {
881 HashBinaryFunctions[MD5Hash(Str: StringRef(BinaryFunction.first))] =
882 &BinaryFunction.second;
883 }
884
885 if (!AddrsWithMultipleSymbols.empty()) {
886 WithColor::warning() << "Found " << AddrsWithMultipleSymbols.size()
887 << " start addresses with multiple symbols\n";
888 AddrsWithMultipleSymbols.clear();
889 }
890}
891
892void ProfiledBinary::populateSymbolListFromDWARF(
893 ProfileSymbolList &SymbolList) {
894 for (auto &I : StartAddrToFuncRangeMap)
895 SymbolList.add(Name: I.second.getFuncName());
896}
897
898symbolize::LLVMSymbolizer::Options ProfiledBinary::getSymbolizerOpts() const {
899 symbolize::LLVMSymbolizer::Options SymbolizerOpts;
900 SymbolizerOpts.PrintFunctions =
901 DILineInfoSpecifier::FunctionNameKind::LinkageName;
902 SymbolizerOpts.Demangle = false;
903 SymbolizerOpts.DefaultArch = TheTriple.getArchName().str();
904 SymbolizerOpts.UseSymbolTable = false;
905 SymbolizerOpts.RelativeAddresses = false;
906 SymbolizerOpts.DWPName = DWPPath;
907 return SymbolizerOpts;
908}
909
910SampleContextFrameVector ProfiledBinary::symbolize(const InstructionPointer &IP,
911 bool UseCanonicalFnName,
912 bool UseProbeDiscriminator) {
913 assert(this == IP.Binary &&
914 "Binary should only symbolize its own instruction");
915 auto Addr = object::SectionedAddress{.Address: IP.Address,
916 .SectionIndex: object::SectionedAddress::UndefSection};
917 DIInliningInfo InlineStack = unwrapOrError(
918 EO: Symbolizer->symbolizeInlinedCode(ModuleName: SymbolizerPath.str(), ModuleOffset: Addr),
919 Args&: SymbolizerPath);
920
921 SampleContextFrameVector CallStack;
922 for (int32_t I = InlineStack.getNumberOfFrames() - 1; I >= 0; I--) {
923 const auto &CallerFrame = InlineStack.getFrame(Index: I);
924 if (CallerFrame.FunctionName.empty() ||
925 (CallerFrame.FunctionName == "<invalid>"))
926 break;
927
928 StringRef FunctionName(CallerFrame.FunctionName);
929 if (UseCanonicalFnName)
930 FunctionName = FunctionSamples::getCanonicalFnName(FnName: FunctionName);
931
932 uint32_t Discriminator = CallerFrame.Discriminator;
933 uint32_t LineOffset = (CallerFrame.Line - CallerFrame.StartLine) & 0xffff;
934 if (UseProbeDiscriminator) {
935 LineOffset =
936 PseudoProbeDwarfDiscriminator::extractProbeIndex(Value: Discriminator);
937 Discriminator = 0;
938 }
939
940 LineLocation Line(LineOffset, Discriminator);
941 auto It = NameStrings.insert(x: FunctionName.str());
942 CallStack.emplace_back(Args: FunctionId(StringRef(*It.first)), Args&: Line);
943 }
944
945 return CallStack;
946}
947
948void ProfiledBinary::computeInlinedContextSizeForRange(uint64_t RangeBegin,
949 uint64_t RangeEnd) {
950 InstructionPointer IP(this, RangeBegin, true);
951
952 if (IP.Address != RangeBegin)
953 WithColor::warning() << "Invalid start instruction at "
954 << format(Fmt: "%8" PRIx64, Vals: RangeBegin) << "\n";
955
956 if (IP.Address >= RangeEnd)
957 return;
958
959 do {
960 const SampleContextFrameVector SymbolizedCallStack =
961 getFrameLocationStack(Address: IP.Address, UseProbeDiscriminator: UsePseudoProbes);
962 uint64_t Size = AddressToInstSizeMap[IP.Address];
963 // Record instruction size for the corresponding context
964 FuncSizeTracker.addInstructionForContext(Context: SymbolizedCallStack, InstrSize: Size);
965
966 } while (IP.advance() && IP.Address < RangeEnd);
967}
968
969void ProfiledBinary::computeInlinedContextSizeForFunc(
970 const BinaryFunction *Func) {
971 // Note that a function can be spilt into multiple ranges, so compute for all
972 // ranges of the function.
973 for (const auto &Range : Func->Ranges)
974 computeInlinedContextSizeForRange(RangeBegin: Range.first, RangeEnd: Range.second);
975
976 // Track optimized-away inlinee for probed binary. A function inlined and then
977 // optimized away should still have their probes left over in places.
978 if (usePseudoProbes()) {
979 auto I = TopLevelProbeFrameMap.find(Key: Func->FuncName);
980 if (I != TopLevelProbeFrameMap.end()) {
981 BinarySizeContextTracker::ProbeFrameStack ProbeContext;
982 FuncSizeTracker.trackInlineesOptimizedAway(ProbeDecoder, ProbeNode&: *I->second,
983 ProbeContext);
984 }
985 }
986}
987
988void ProfiledBinary::inferMissingFrames(
989 const SmallVectorImpl<uint64_t> &Context,
990 SmallVectorImpl<uint64_t> &NewContext) {
991 MissingContextInferrer->inferMissingFrames(Context, NewContext);
992}
993
994InstructionPointer::InstructionPointer(const ProfiledBinary *Binary,
995 uint64_t Address, bool RoundToNext)
996 : Binary(Binary), Address(Address) {
997 Index = Binary->getIndexForAddr(Address);
998 if (RoundToNext) {
999 // we might get address which is not the code
1000 // it should round to the next valid address
1001 if (Index >= Binary->getCodeAddrVecSize())
1002 this->Address = UINT64_MAX;
1003 else
1004 this->Address = Binary->getAddressforIndex(Index);
1005 }
1006}
1007
1008bool InstructionPointer::advance() {
1009 Index++;
1010 if (Index >= Binary->getCodeAddrVecSize()) {
1011 Address = UINT64_MAX;
1012 return false;
1013 }
1014 Address = Binary->getAddressforIndex(Index);
1015 return true;
1016}
1017
1018bool InstructionPointer::backward() {
1019 if (Index == 0) {
1020 Address = 0;
1021 return false;
1022 }
1023 Index--;
1024 Address = Binary->getAddressforIndex(Index);
1025 return true;
1026}
1027
1028void InstructionPointer::update(uint64_t Addr) {
1029 Address = Addr;
1030 Index = Binary->getIndexForAddr(Address);
1031}
1032
1033} // end namespace sampleprof
1034} // end namespace llvm
1035