1//===- lib/MC/MCPseudoProbe.cpp - Pseudo probe encoding support ----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/MC/MCPseudoProbe.h"
10#include "llvm/ADT/STLExtras.h"
11#include "llvm/IR/PseudoProbe.h"
12#include "llvm/MC/MCAsmInfo.h"
13#include "llvm/MC/MCAssembler.h"
14#include "llvm/MC/MCContext.h"
15#include "llvm/MC/MCExpr.h"
16#include "llvm/MC/MCObjectFileInfo.h"
17#include "llvm/MC/MCObjectStreamer.h"
18#include "llvm/MC/MCSymbol.h"
19#include "llvm/Support/Endian.h"
20#include "llvm/Support/Error.h"
21#include "llvm/Support/LEB128.h"
22#include "llvm/Support/MD5.h"
23#include "llvm/Support/raw_ostream.h"
24#include <algorithm>
25#include <cassert>
26#include <limits>
27#include <memory>
28#include <sstream>
29#include <vector>
30
31#define DEBUG_TYPE "mcpseudoprobe"
32
33using namespace llvm;
34using namespace support;
35
36#ifndef NDEBUG
37int MCPseudoProbeTable::DdgPrintIndent = 0;
38#endif
39
40static const MCExpr *buildSymbolDiff(MCObjectStreamer *MCOS, const MCSymbol *A,
41 const MCSymbol *B) {
42 MCContext &Context = MCOS->getContext();
43 const MCExpr *ARef = MCSymbolRefExpr::create(Symbol: A, Ctx&: Context);
44 const MCExpr *BRef = MCSymbolRefExpr::create(Symbol: B, Ctx&: Context);
45 const MCExpr *AddrDelta =
46 MCBinaryExpr::create(Op: MCBinaryExpr::Sub, LHS: ARef, RHS: BRef, Ctx&: Context);
47 return AddrDelta;
48}
49
50uint64_t MCDecodedPseudoProbe::getGuid() const { return InlineTree->Guid; }
51
52void MCPseudoProbe::emit(MCObjectStreamer *MCOS,
53 const MCPseudoProbe *LastProbe) const {
54 bool IsSentinel = isSentinelProbe(Flags: getAttributes());
55 assert((LastProbe || IsSentinel) &&
56 "Last probe should not be null for non-sentinel probes");
57
58 // Emit Index
59 MCOS->emitULEB128IntValue(Value: Index);
60 // Emit Type and the flag:
61 // Type (bit 0 to 3), with bit 4 to 6 for attributes.
62 // Flag (bit 7, 0 - code address, 1 - address delta). This indicates whether
63 // the following field is a symbolic code address or an address delta.
64 // Emit FS discriminator
65 assert(Type <= 0xF && "Probe type too big to encode, exceeding 15");
66 auto NewAttributes = Attributes;
67 if (Discriminator)
68 NewAttributes |= (uint32_t)PseudoProbeAttributes::HasDiscriminator;
69 assert(NewAttributes <= 0x7 &&
70 "Probe attributes too big to encode, exceeding 7");
71 uint8_t PackedType = Type | (NewAttributes << 4);
72 uint8_t Flag =
73 !IsSentinel ? ((int8_t)MCPseudoProbeFlag::AddressDelta << 7) : 0;
74 MCOS->emitInt8(Value: Flag | PackedType);
75
76 if (!IsSentinel) {
77 // Emit the delta between the address label and LastProbe.
78 const MCExpr *AddrDelta =
79 buildSymbolDiff(MCOS, A: Label, B: LastProbe->getLabel());
80 int64_t Delta;
81 if (AddrDelta->evaluateAsAbsolute(Res&: Delta, Asm: MCOS->getAssemblerPtr())) {
82 MCOS->emitSLEB128IntValue(Value: Delta);
83 } else {
84 MCOS->insert(F: MCOS->getContext().allocFragment<MCPseudoProbeAddrFragment>(
85 args&: AddrDelta));
86 }
87 } else {
88 // Emit the GUID of the split function that the sentinel probe represents.
89 MCOS->emitInt64(Value: Guid);
90 }
91
92 if (Discriminator)
93 MCOS->emitULEB128IntValue(Value: Discriminator);
94
95 LLVM_DEBUG({
96 dbgs().indent(MCPseudoProbeTable::DdgPrintIndent);
97 dbgs() << "Probe: " << Index << "\n";
98 });
99}
100
101void MCPseudoProbeInlineTree::addPseudoProbe(
102 const MCPseudoProbe &Probe, const MCPseudoProbeInlineStack &InlineStack) {
103 // The function should not be called on the root.
104 assert(isRoot() && "Should only be called on root");
105
106 // When it comes here, the input look like:
107 // Probe: GUID of C, ...
108 // InlineStack: [88, A], [66, B]
109 // which means, Function A inlines function B at call site with a probe id of
110 // 88, and B inlines C at probe 66. The tri-tree expects a tree path like {[0,
111 // A], [88, B], [66, C]} to locate the tree node where the probe should be
112 // added. Note that the edge [0, A] means A is the top-level function we are
113 // emitting probes for.
114
115 // Make a [0, A] edge.
116 // An empty inline stack means the function that the probe originates from
117 // is a top-level function.
118 InlineSite Top;
119 if (InlineStack.empty()) {
120 Top = InlineSite(Probe.getGuid(), 0);
121 } else {
122 Top = InlineSite(std::get<0>(t: InlineStack.front()), 0);
123 }
124
125 auto *Cur = getOrAddNode(Site: Top);
126
127 // Make interior edges by walking the inline stack. Once it's done, Cur should
128 // point to the node that the probe originates from.
129 if (!InlineStack.empty()) {
130 auto Iter = InlineStack.begin();
131 auto Index = std::get<1>(t: *Iter);
132 Iter++;
133 for (; Iter != InlineStack.end(); Iter++) {
134 // Make an edge by using the previous probe id and current GUID.
135 Cur = Cur->getOrAddNode(Site: InlineSite(std::get<0>(t: *Iter), Index));
136 Index = std::get<1>(t: *Iter);
137 }
138 Cur = Cur->getOrAddNode(Site: InlineSite(Probe.getGuid(), Index));
139 }
140
141 Cur->Probes.push_back(x: Probe);
142}
143
144void MCPseudoProbeInlineTree::emit(MCObjectStreamer *MCOS,
145 const MCPseudoProbe *&LastProbe) {
146 LLVM_DEBUG({
147 dbgs().indent(MCPseudoProbeTable::DdgPrintIndent);
148 dbgs() << "Group [\n";
149 MCPseudoProbeTable::DdgPrintIndent += 2;
150 });
151 assert(!isRoot() && "Root should be handled separately");
152
153 // Emit probes grouped by GUID.
154 LLVM_DEBUG({
155 dbgs().indent(MCPseudoProbeTable::DdgPrintIndent);
156 dbgs() << "GUID: " << Guid << "\n";
157 });
158 // Emit Guid
159 MCOS->emitInt64(Value: Guid);
160 // Emit number of probes in this node, including a sentinel probe for
161 // top-level functions if needed.
162 bool NeedSentinel = false;
163 if (Parent->isRoot()) {
164 assert(isSentinelProbe(LastProbe->getAttributes()) &&
165 "Starting probe of a top-level function should be a sentinel probe");
166 // The main body of a split function doesn't need a sentinel probe.
167 if (LastProbe->getGuid() != Guid)
168 NeedSentinel = true;
169 }
170
171 MCOS->emitULEB128IntValue(Value: Probes.size() + NeedSentinel);
172 // Emit number of direct inlinees
173 MCOS->emitULEB128IntValue(Value: Children.size());
174 // Emit sentinel probe for top-level functions
175 if (NeedSentinel)
176 LastProbe->emit(MCOS, LastProbe: nullptr);
177
178 // Emit probes in this group
179 for (const auto &Probe : Probes) {
180 Probe.emit(MCOS, LastProbe);
181 LastProbe = &Probe;
182 }
183
184 // Emit sorted descendant. InlineSite is unique for each pair, so there will
185 // be no ordering of Inlinee based on MCPseudoProbeInlineTree*
186 using InlineeType = std::pair<InlineSite, MCPseudoProbeInlineTree *>;
187 std::vector<InlineeType> Inlinees;
188 for (const auto &Child : Children)
189 Inlinees.emplace_back(args: Child.first, args: Child.second.get());
190 llvm::sort(C&: Inlinees, Comp: llvm::less_first());
191
192 for (const auto &Inlinee : Inlinees) {
193 // Emit probe index
194 MCOS->emitULEB128IntValue(Value: std::get<1>(t: Inlinee.first));
195 LLVM_DEBUG({
196 dbgs().indent(MCPseudoProbeTable::DdgPrintIndent);
197 dbgs() << "InlineSite: " << std::get<1>(Inlinee.first) << "\n";
198 });
199 // Emit the group
200 Inlinee.second->emit(MCOS, LastProbe);
201 }
202
203 LLVM_DEBUG({
204 MCPseudoProbeTable::DdgPrintIndent -= 2;
205 dbgs().indent(MCPseudoProbeTable::DdgPrintIndent);
206 dbgs() << "]\n";
207 });
208}
209
210void MCPseudoProbeSections::emit(MCObjectStreamer *MCOS) {
211 MCContext &Ctx = MCOS->getContext();
212 SmallVector<std::pair<MCSymbol *, MCPseudoProbeInlineTree *>> Vec;
213 Vec.reserve(N: MCProbeDivisions.size());
214 for (auto &ProbeSec : MCProbeDivisions)
215 Vec.emplace_back(Args: ProbeSec.first, Args: &ProbeSec.second);
216 for (auto I : llvm::enumerate(First&: MCOS->getAssembler()))
217 I.value().setOrdinal(I.index());
218 llvm::sort(C&: Vec, Comp: [](auto A, auto B) {
219 return A.first->getSection().getOrdinal() <
220 B.first->getSection().getOrdinal();
221 });
222 for (auto [FuncSym, RootPtr] : Vec) {
223 const auto &Root = *RootPtr;
224 if (auto *S = Ctx.getObjectFileInfo()->getPseudoProbeSection(
225 TextSec: FuncSym->getSection())) {
226 // Switch to the .pseudoprobe section or a comdat group.
227 MCOS->switchSection(Section: S);
228 // Emit probes grouped by GUID.
229 // Emit sorted descendant. InlineSite is unique for each pair, so there
230 // will be no ordering of Inlinee based on MCPseudoProbeInlineTree*
231 using InlineeType = std::pair<InlineSite, MCPseudoProbeInlineTree *>;
232 std::vector<InlineeType> Inlinees;
233 for (const auto &Child : Root.getChildren())
234 Inlinees.emplace_back(args: Child.first, args: Child.second.get());
235 llvm::sort(C&: Inlinees, Comp: llvm::less_first());
236
237 for (const auto &Inlinee : Inlinees) {
238 // Emit the group guarded by a sentinel probe.
239 MCPseudoProbe SentinelProbe(
240 const_cast<MCSymbol *>(FuncSym), MD5Hash(Str: FuncSym->getName()),
241 (uint32_t)PseudoProbeReservedId::Invalid,
242 (uint32_t)PseudoProbeType::Block,
243 (uint32_t)PseudoProbeAttributes::Sentinel, 0);
244 const MCPseudoProbe *Probe = &SentinelProbe;
245 Inlinee.second->emit(MCOS, LastProbe&: Probe);
246 }
247 }
248 }
249}
250
251//
252// This emits the pseudo probe tables.
253//
254void MCPseudoProbeTable::emit(MCObjectStreamer *MCOS) {
255 MCContext &Ctx = MCOS->getContext();
256 auto &ProbeTable = Ctx.getMCPseudoProbeTable();
257
258 // Bail out early so we don't switch to the pseudo_probe section needlessly
259 // and in doing so create an unnecessary (if empty) section.
260 auto &ProbeSections = ProbeTable.getProbeSections();
261 if (ProbeSections.empty())
262 return;
263
264 LLVM_DEBUG(MCPseudoProbeTable::DdgPrintIndent = 0);
265
266 // Put out the probe.
267 ProbeSections.emit(MCOS);
268}
269
270static StringRef getProbeFNameForGUID(const GUIDProbeFunctionMap &GUID2FuncMAP,
271 uint64_t GUID) {
272 auto It = GUID2FuncMAP.find(GUID);
273 assert(It != GUID2FuncMAP.end() &&
274 "Probe function must exist for a valid GUID");
275 return It->FuncName;
276}
277
278void MCPseudoProbeFuncDesc::print(raw_ostream &OS) {
279 OS << "GUID: " << FuncGUID << " Name: " << FuncName << "\n";
280 OS << "Hash: " << FuncHash << "\n";
281}
282
283void MCDecodedPseudoProbe::getInlineContext(
284 SmallVectorImpl<MCPseudoProbeFrameLocation> &ContextStack,
285 const GUIDProbeFunctionMap &GUID2FuncMAP) const {
286 uint32_t Begin = ContextStack.size();
287 MCDecodedPseudoProbeInlineTree *Cur = InlineTree;
288 // It will add the string of each node's inline site during iteration.
289 // Note that it won't include the probe's belonging function(leaf location)
290 while (Cur->hasInlineSite()) {
291 StringRef FuncName = getProbeFNameForGUID(GUID2FuncMAP, GUID: Cur->Parent->Guid);
292 ContextStack.emplace_back(Args: MCPseudoProbeFrameLocation(
293 FuncName, std::get<1>(t: Cur->getInlineSite())));
294 Cur = static_cast<MCDecodedPseudoProbeInlineTree *>(Cur->Parent);
295 }
296 // Make the ContextStack in caller-callee order
297 std::reverse(first: ContextStack.begin() + Begin, last: ContextStack.end());
298}
299
300std::string MCDecodedPseudoProbe::getInlineContextStr(
301 const GUIDProbeFunctionMap &GUID2FuncMAP) const {
302 std::ostringstream OContextStr;
303 SmallVector<MCPseudoProbeFrameLocation, 16> ContextStack;
304 getInlineContext(ContextStack, GUID2FuncMAP);
305 for (auto &Cxt : ContextStack) {
306 if (OContextStr.str().size())
307 OContextStr << " @ ";
308 OContextStr << Cxt.first.str() << ":" << Cxt.second;
309 }
310 return OContextStr.str();
311}
312
313static const char *PseudoProbeTypeStr[3] = {"Block", "IndirectCall",
314 "DirectCall"};
315
316void MCDecodedPseudoProbe::print(raw_ostream &OS,
317 const GUIDProbeFunctionMap &GUID2FuncMAP,
318 bool ShowName) const {
319 OS << "FUNC: ";
320 if (ShowName) {
321 StringRef FuncName = getProbeFNameForGUID(GUID2FuncMAP, GUID: getGuid());
322 OS << FuncName.str() << " ";
323 } else {
324 OS << getGuid() << " ";
325 }
326 OS << "Index: " << Index << " ";
327 if (Discriminator)
328 OS << "Discriminator: " << Discriminator << " ";
329 OS << "Type: " << PseudoProbeTypeStr[static_cast<uint8_t>(Type)] << " ";
330 std::string InlineContextStr = getInlineContextStr(GUID2FuncMAP);
331 if (InlineContextStr.size()) {
332 OS << "Inlined: @ ";
333 OS << InlineContextStr;
334 }
335 OS << "\n";
336}
337
338template <typename T> ErrorOr<T> MCPseudoProbeDecoder::readUnencodedNumber() {
339 if (Data + sizeof(T) > End) {
340 return std::error_code();
341 }
342 T Val = endian::readNext<T, llvm::endianness::little>(Data);
343 return ErrorOr<T>(Val);
344}
345
346template <typename T> ErrorOr<T> MCPseudoProbeDecoder::readUnsignedNumber() {
347 unsigned NumBytesRead = 0;
348 uint64_t Val = decodeULEB128(p: Data, n: &NumBytesRead);
349 if (Val > std::numeric_limits<T>::max() || (Data + NumBytesRead > End)) {
350 return std::error_code();
351 }
352 Data += NumBytesRead;
353 return ErrorOr<T>(static_cast<T>(Val));
354}
355
356template <typename T> ErrorOr<T> MCPseudoProbeDecoder::readSignedNumber() {
357 unsigned NumBytesRead = 0;
358 int64_t Val = decodeSLEB128(p: Data, n: &NumBytesRead);
359 if (Val > std::numeric_limits<T>::max() || (Data + NumBytesRead > End)) {
360 return std::error_code();
361 }
362 Data += NumBytesRead;
363 return ErrorOr<T>(static_cast<T>(Val));
364}
365
366ErrorOr<StringRef> MCPseudoProbeDecoder::readString(uint32_t Size) {
367 StringRef Str(reinterpret_cast<const char *>(Data), Size);
368 if (Data + Size > End) {
369 return std::error_code();
370 }
371 Data += Size;
372 return ErrorOr<StringRef>(Str);
373}
374
375bool MCPseudoProbeDecoder::buildGUID2FuncDescMap(const uint8_t *Start,
376 std::size_t Size,
377 bool IsMMapped) {
378 // The pseudo_probe_desc section has a format like:
379 // .section .pseudo_probe_desc,"",@progbits
380 // .quad -5182264717993193164 // GUID
381 // .quad 4294967295 // Hash
382 // .uleb 3 // Name size
383 // .ascii "foo" // Name
384 // .quad -2624081020897602054
385 // .quad 174696971957
386 // .uleb 34
387 // .ascii "main"
388
389 Data = Start;
390 End = Data + Size;
391
392 uint32_t FuncDescCount = 0;
393 while (Data < End) {
394 // GUID
395 if (!readUnencodedNumber<uint64_t>())
396 return false;
397 // Hash
398 if (!readUnencodedNumber<uint64_t>())
399 return false;
400
401 auto ErrorOrNameSize = readUnsignedNumber<uint32_t>();
402 if (!ErrorOrNameSize)
403 return false;
404 // Function name
405 if (!readString(Size: *ErrorOrNameSize))
406 return false;
407 ++FuncDescCount;
408 }
409 assert(Data == End && "Have unprocessed data in pseudo_probe_desc section");
410 GUID2FuncDescMap.reserve(n: FuncDescCount);
411
412 Data = Start;
413 End = Data + Size;
414 while (Data < End) {
415 uint64_t GUID =
416 cantFail(ValOrErr: errorOrToExpected(EO: readUnencodedNumber<uint64_t>()));
417 uint64_t Hash =
418 cantFail(ValOrErr: errorOrToExpected(EO: readUnencodedNumber<uint64_t>()));
419 uint32_t NameSize =
420 cantFail(ValOrErr: errorOrToExpected(EO: readUnsignedNumber<uint32_t>()));
421 StringRef Name = cantFail(ValOrErr: errorOrToExpected(EO: readString(Size: NameSize)));
422
423 // Initialize PseudoProbeFuncDesc and populate it into GUID2FuncDescMap
424 GUID2FuncDescMap.emplace_back(
425 args&: GUID, args&: Hash, args: IsMMapped ? Name : Name.copy(A&: FuncNameAllocator));
426 }
427 assert(Data == End && "Have unprocessed data in pseudo_probe_desc section");
428 assert(GUID2FuncDescMap.size() == FuncDescCount &&
429 "Mismatching function description count pre- and post-parsing");
430 llvm::sort(C&: GUID2FuncDescMap, Comp: [](const auto &LHS, const auto &RHS) {
431 return LHS.FuncGUID < RHS.FuncGUID;
432 });
433 return true;
434}
435
436template <bool IsTopLevelFunc>
437bool MCPseudoProbeDecoder::buildAddress2ProbeMap(
438 MCDecodedPseudoProbeInlineTree *Cur, uint64_t &LastAddr,
439 const Uint64Set &GuidFilter, const Uint64Map &FuncStartAddrs,
440 const uint32_t CurChildIndex) {
441 // The pseudo_probe section encodes an inline forest and each tree has a
442 // format defined in MCPseudoProbe.h
443
444 uint32_t Index = 0;
445 if (IsTopLevelFunc) {
446 // Use a sequential id for top level inliner.
447 Index = CurChildIndex;
448 } else {
449 // Read inline site for inlinees
450 Index = cantFail(ValOrErr: errorOrToExpected(EO: readUnsignedNumber<uint32_t>()));
451 }
452
453 // Read guid
454 uint64_t Guid = cantFail(ValOrErr: errorOrToExpected(EO: readUnencodedNumber<uint64_t>()));
455
456 // Decide if top-level node should be disgarded.
457 if (IsTopLevelFunc && !GuidFilter.empty() && !GuidFilter.count(V: Guid))
458 Cur = nullptr;
459
460 // If the incoming node is null, all its children nodes should be disgarded.
461 if (Cur) {
462 // Switch/add to a new tree node(inlinee)
463 Cur->getChildren()[CurChildIndex] =
464 MCDecodedPseudoProbeInlineTree(InlineSite(Guid, Index), Cur);
465 Cur = &Cur->getChildren()[CurChildIndex];
466 if (IsTopLevelFunc && !EncodingIsAddrBased) {
467 if (auto V = FuncStartAddrs.lookup(Val: Guid))
468 LastAddr = V;
469 }
470 }
471
472 // Read number of probes in the current node.
473 uint32_t NodeCount =
474 cantFail(ValOrErr: errorOrToExpected(EO: readUnsignedNumber<uint32_t>()));
475 uint32_t CurrentProbeCount = 0;
476 // Read number of direct inlinees
477 uint32_t ChildrenToProcess =
478 cantFail(ValOrErr: errorOrToExpected(EO: readUnsignedNumber<uint32_t>()));
479 // Read all probes in this node
480 for (std::size_t I = 0; I < NodeCount; I++) {
481 // Read index
482 uint32_t Index =
483 cantFail(ValOrErr: errorOrToExpected(EO: readUnsignedNumber<uint32_t>()));
484 // Read type | flag.
485 uint8_t Value = cantFail(ValOrErr: errorOrToExpected(EO: readUnencodedNumber<uint8_t>()));
486 uint8_t Kind = Value & 0xf;
487 uint8_t Attr = (Value & 0x70) >> 4;
488 // Read address
489 uint64_t Addr = 0;
490 if (Value & 0x80) {
491 int64_t Offset = cantFail(ValOrErr: errorOrToExpected(EO: readSignedNumber<int64_t>()));
492 Addr = LastAddr + Offset;
493 } else {
494 Addr = cantFail(ValOrErr: errorOrToExpected(EO: readUnencodedNumber<int64_t>()));
495 if (isSentinelProbe(Flags: Attr)) {
496 // For sentinel probe, the addr field actually stores the GUID of the
497 // split function. Convert it to the real address.
498 if (auto V = FuncStartAddrs.lookup(Val: Addr))
499 Addr = V;
500 } else {
501 // For now we assume all probe encoding should be either based on
502 // leading probe address or function start address.
503 // The scheme is for downwards compatibility.
504 // TODO: retire this scheme once compatibility is no longer an issue.
505 EncodingIsAddrBased = true;
506 }
507 }
508
509 uint32_t Discriminator = 0;
510 if (hasDiscriminator(Flags: Attr)) {
511 Discriminator =
512 cantFail(ValOrErr: errorOrToExpected(EO: readUnsignedNumber<uint32_t>()));
513 }
514
515 if (Cur && !isSentinelProbe(Flags: Attr)) {
516 PseudoProbeVec.emplace_back(args&: Addr, args&: Index, args: PseudoProbeType(Kind), args&: Attr,
517 args&: Discriminator, args&: Cur);
518 ++CurrentProbeCount;
519 }
520 LastAddr = Addr;
521 }
522
523 if (Cur) {
524 Cur->setProbes(
525 MutableArrayRef(PseudoProbeVec).take_back(N: CurrentProbeCount));
526 InlineTreeVec.resize(new_size: InlineTreeVec.size() + ChildrenToProcess);
527 Cur->getChildren() =
528 MutableArrayRef(InlineTreeVec).take_back(N: ChildrenToProcess);
529 }
530 for (uint32_t I = 0; I < ChildrenToProcess; I++) {
531 buildAddress2ProbeMap<false>(Cur, LastAddr, GuidFilter, FuncStartAddrs, CurChildIndex: I);
532 }
533 return Cur;
534}
535
536template <bool IsTopLevelFunc>
537bool MCPseudoProbeDecoder::countRecords(bool &Discard, uint32_t &ProbeCount,
538 uint32_t &InlinedCount,
539 const Uint64Set &GuidFilter) {
540 if (!IsTopLevelFunc)
541 // Read inline site for inlinees
542 if (!readUnsignedNumber<uint32_t>())
543 return false;
544
545 // Read guid
546 auto ErrorOrCurGuid = readUnencodedNumber<uint64_t>();
547 if (!ErrorOrCurGuid)
548 return false;
549 uint64_t Guid = std::move(*ErrorOrCurGuid);
550
551 // Decide if top-level node should be disgarded.
552 if (IsTopLevelFunc) {
553 Discard = !GuidFilter.empty() && !GuidFilter.count(V: Guid);
554 if (!Discard)
555 // Allocate an entry for top-level function record.
556 ++InlinedCount;
557 }
558
559 // Read number of probes in the current node.
560 auto ErrorOrNodeCount = readUnsignedNumber<uint32_t>();
561 if (!ErrorOrNodeCount)
562 return false;
563 uint32_t NodeCount = std::move(*ErrorOrNodeCount);
564 uint32_t CurrentProbeCount = 0;
565
566 // Read number of direct inlinees
567 auto ErrorOrCurChildrenToProcess = readUnsignedNumber<uint32_t>();
568 if (!ErrorOrCurChildrenToProcess)
569 return false;
570 uint32_t ChildrenToProcess = std::move(*ErrorOrCurChildrenToProcess);
571
572 // Read all probes in this node
573 for (std::size_t I = 0; I < NodeCount; I++) {
574 // Read index
575 if (!readUnsignedNumber<uint32_t>())
576 return false;
577
578 // Read type | flag.
579 auto ErrorOrValue = readUnencodedNumber<uint8_t>();
580 if (!ErrorOrValue)
581 return false;
582 uint8_t Value = std::move(*ErrorOrValue);
583
584 uint8_t Attr = (Value & 0x70) >> 4;
585 if (Value & 0x80) {
586 // Offset
587 if (!readSignedNumber<int64_t>())
588 return false;
589 } else {
590 // Addr
591 if (!readUnencodedNumber<int64_t>())
592 return false;
593 }
594
595 if (hasDiscriminator(Flags: Attr))
596 // Discriminator
597 if (!readUnsignedNumber<uint32_t>())
598 return false;
599
600 if (!Discard && !isSentinelProbe(Flags: Attr))
601 ++CurrentProbeCount;
602 }
603
604 if (!Discard) {
605 ProbeCount += CurrentProbeCount;
606 InlinedCount += ChildrenToProcess;
607 }
608
609 for (uint32_t I = 0; I < ChildrenToProcess; I++)
610 if (!countRecords<false>(Discard, ProbeCount, InlinedCount, GuidFilter))
611 return false;
612 return true;
613}
614
615bool MCPseudoProbeDecoder::buildAddress2ProbeMap(
616 const uint8_t *Start, std::size_t Size, const Uint64Set &GuidFilter,
617 const Uint64Map &FuncStartAddrs) {
618 // For function records in the order of their appearance in the encoded data
619 // (DFS), count the number of contained probes and inlined function records.
620 uint32_t ProbeCount = 0;
621 uint32_t InlinedCount = 0;
622 uint32_t TopLevelFuncs = 0;
623 Data = Start;
624 End = Data + Size;
625 bool Discard = false;
626 while (Data < End) {
627 if (!countRecords<true>(Discard, ProbeCount, InlinedCount, GuidFilter))
628 return false;
629 TopLevelFuncs += !Discard;
630 }
631 assert(Data == End && "Have unprocessed data in pseudo_probe section");
632 PseudoProbeVec.reserve(n: ProbeCount);
633 InlineTreeVec.reserve(n: InlinedCount);
634
635 // Allocate top-level function records as children of DummyInlineRoot.
636 InlineTreeVec.resize(new_size: TopLevelFuncs);
637 DummyInlineRoot.getChildren() = MutableArrayRef(InlineTreeVec);
638
639 Data = Start;
640 End = Data + Size;
641 uint64_t LastAddr = 0;
642 uint32_t CurChildIndex = 0;
643 while (Data < End)
644 CurChildIndex += buildAddress2ProbeMap<true>(
645 Cur: &DummyInlineRoot, LastAddr, GuidFilter, FuncStartAddrs, CurChildIndex);
646 assert(Data == End && "Have unprocessed data in pseudo_probe section");
647 assert(PseudoProbeVec.size() == ProbeCount &&
648 "Mismatching probe count pre- and post-parsing");
649 assert(InlineTreeVec.size() == InlinedCount &&
650 "Mismatching function records count pre- and post-parsing");
651
652 std::vector<std::pair<uint64_t, uint32_t>> SortedA2P(ProbeCount);
653 for (const auto &[I, Probe] : llvm::enumerate(First&: PseudoProbeVec))
654 SortedA2P[I] = {Probe.getAddress(), I};
655 llvm::sort(C&: SortedA2P);
656 Address2ProbesMap.reserve(n: ProbeCount);
657 for (const uint32_t I : llvm::make_second_range(c&: SortedA2P))
658 Address2ProbesMap.emplace_back(args&: PseudoProbeVec[I]);
659 SortedA2P.clear();
660 return true;
661}
662
663void MCPseudoProbeDecoder::printGUID2FuncDescMap(raw_ostream &OS) {
664 OS << "Pseudo Probe Desc:\n";
665 for (auto &I : GUID2FuncDescMap)
666 I.print(OS);
667}
668
669void MCPseudoProbeDecoder::printProbeForAddress(raw_ostream &OS,
670 uint64_t Address) {
671 for (const MCDecodedPseudoProbe &Probe : Address2ProbesMap.find(Address)) {
672 OS << " [Probe]:\t";
673 Probe.print(OS, GUID2FuncMAP: GUID2FuncDescMap, ShowName: true);
674 }
675}
676
677void MCPseudoProbeDecoder::printProbesForAllAddresses(raw_ostream &OS) {
678 uint64_t PrevAddress = INT64_MAX;
679 for (MCDecodedPseudoProbe &Probe : Address2ProbesMap) {
680 uint64_t Address = Probe.getAddress();
681 if (Address != PrevAddress) {
682 PrevAddress = Address;
683 OS << "Address:\t" << Address << '\n';
684 }
685 OS << " [Probe]:\t";
686 Probe.print(OS, GUID2FuncMAP: GUID2FuncDescMap, ShowName: true);
687 }
688}
689
690const MCDecodedPseudoProbe *
691MCPseudoProbeDecoder::getCallProbeForAddr(uint64_t Address) const {
692 const MCDecodedPseudoProbe *CallProbe = nullptr;
693 for (const MCDecodedPseudoProbe &Probe : Address2ProbesMap.find(Address)) {
694 if (Probe.isCall()) {
695 // Disabling the assert and returning first call probe seen so far.
696 // Subsequent call probes, if any, are ignored. Due to the the way
697 // .pseudo_probe section is decoded, probes of the same-named independent
698 // static functions are merged thus multiple call probes may be seen for a
699 // callsite. This should only happen to compiler-generated statics, with
700 // -funique-internal-linkage-names where user statics get unique names.
701 //
702 // TODO: re-enable or narrow down the assert to static functions only.
703 //
704 // assert(!CallProbe &&
705 // "There should be only one call probe corresponding to address "
706 // "which is a callsite.");
707 CallProbe = &Probe;
708 break;
709 }
710 }
711 return CallProbe;
712}
713
714const MCPseudoProbeFuncDesc *
715MCPseudoProbeDecoder::getFuncDescForGUID(uint64_t GUID) const {
716 auto It = GUID2FuncDescMap.find(GUID);
717 assert(It != GUID2FuncDescMap.end() && "Function descriptor doesn't exist");
718 return &*It;
719}
720
721void MCPseudoProbeDecoder::getInlineContextForProbe(
722 const MCDecodedPseudoProbe *Probe,
723 SmallVectorImpl<MCPseudoProbeFrameLocation> &InlineContextStack,
724 bool IncludeLeaf) const {
725 Probe->getInlineContext(ContextStack&: InlineContextStack, GUID2FuncMAP: GUID2FuncDescMap);
726 if (!IncludeLeaf)
727 return;
728 // Note that the context from probe doesn't include leaf frame,
729 // hence we need to retrieve and prepend leaf if requested.
730 const auto *FuncDesc = getFuncDescForGUID(GUID: Probe->getGuid());
731 InlineContextStack.emplace_back(
732 Args: MCPseudoProbeFrameLocation(FuncDesc->FuncName, Probe->getIndex()));
733}
734
735const MCPseudoProbeFuncDesc *MCPseudoProbeDecoder::getInlinerDescForProbe(
736 const MCDecodedPseudoProbe *Probe) const {
737 MCDecodedPseudoProbeInlineTree *InlinerNode = Probe->getInlineTreeNode();
738 if (!InlinerNode->hasInlineSite())
739 return nullptr;
740 return getFuncDescForGUID(GUID: InlinerNode->Parent->Guid);
741}
742