1//===-- PerfReader.h - perfscript reader -----------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#ifndef LLVM_TOOLS_LLVM_PROFGEN_PERFREADER_H
10#define LLVM_TOOLS_LLVM_PROFGEN_PERFREADER_H
11#include "ErrorHandling.h"
12#include "ProfiledBinary.h"
13#include "llvm/ADT/DenseMap.h"
14#include "llvm/ADT/StringSet.h"
15#include "llvm/Support/Casting.h"
16#include "llvm/Support/CommandLine.h"
17#include "llvm/Support/Error.h"
18#include "llvm/Support/Regex.h"
19#include <cstdint>
20#include <fstream>
21#include <map>
22
23namespace llvm {
24
25class CleanupInstaller;
26
27namespace sampleprof {
28
29// Stream based trace line iterator
30class TraceStream {
31 std::string CurrentLine;
32 std::ifstream Fin;
33 bool IsAtEoF = false;
34 uint64_t LineNumber = 0;
35
36public:
37 TraceStream(StringRef Filename) : Fin(Filename.str()) {
38 if (!Fin.good())
39 exitWithError(Message: "Error read input perf script file", Whence: Filename);
40 advance();
41 }
42
43 StringRef getCurrentLine() {
44 assert(!IsAtEoF && "Line iterator reaches the End-of-File!");
45 return CurrentLine;
46 }
47
48 uint64_t getLineNumber() { return LineNumber; }
49
50 bool isAtEoF() { return IsAtEoF; }
51
52 // Read the next line
53 void advance() {
54 if (!std::getline(is&: Fin, str&: CurrentLine)) {
55 IsAtEoF = true;
56 return;
57 }
58 LineNumber++;
59 }
60};
61
62// The type of input format.
63enum InputFormat {
64 UnknownFormat = 0,
65 PerfData = 1, // Raw linux perf.data.
66 PerfScript = 2, // Perf script create by `perf script` command.
67 UnsymbolizedProfile = 3, // Unsymbolized profile generated by llvm-profgen.
68 ETMFormat = 4, // Raw ETM format.
69};
70
71// The type of perfscript content.
72enum PerfContent {
73 UnknownContent = 0,
74 LBR = 1, // Only LBR sample.
75 LBRStack = 2, // Hybrid sample including call stack and LBR stack.
76};
77
78struct InputFile {
79 std::string InputFilePath;
80 InputFormat Format = InputFormat::UnknownFormat;
81 PerfContent Content = PerfContent::UnknownContent;
82};
83
84// The parsed LBR sample entry.
85struct LBREntry {
86 uint64_t Source = 0;
87 uint64_t Target = 0;
88 LBREntry(uint64_t S, uint64_t T) : Source(S), Target(T) {}
89
90#ifndef NDEBUG
91 void print() const {
92 dbgs() << "from " << format("%#010x", Source) << " to "
93 << format("%#010x", Target);
94 }
95#endif
96};
97
98#ifndef NDEBUG
99static inline void printLBRStack(const SmallVectorImpl<LBREntry> &LBRStack) {
100 for (size_t I = 0; I < LBRStack.size(); I++) {
101 dbgs() << "[" << I << "] ";
102 LBRStack[I].print();
103 dbgs() << "\n";
104 }
105}
106
107static inline void printCallStack(const SmallVectorImpl<uint64_t> &CallStack) {
108 for (size_t I = 0; I < CallStack.size(); I++) {
109 dbgs() << "[" << I << "] " << format("%#010x", CallStack[I]) << "\n";
110 }
111}
112#endif
113
114// Hash interface for generic data of type T
115// Data should implement a \fn getHashCode and a \fn isEqual
116// Currently getHashCode is non-virtual to avoid the overhead of calling vtable,
117// i.e we explicitly calculate hash of derived class, assign to base class's
118// HashCode. This also provides the flexibility for calculating the hash code
119// incrementally(like rolling hash) during frame stack unwinding since unwinding
120// only changes the leaf of frame stack. \fn isEqual is a virtual function,
121// which will have perf overhead. In the future, if we redesign a better hash
122// function, then we can just skip this or switch to non-virtual function(like
123// just ignore comparison if hash conflicts probabilities is low)
124template <class T> class Hashable {
125public:
126 std::shared_ptr<T> Data;
127 Hashable(const std::shared_ptr<T> &D) : Data(D) {}
128
129 T *getPtr() const { return Data.get(); }
130};
131
132} // end namespace sampleprof
133
134template <typename T> struct DenseMapInfo<sampleprof::Hashable<T>> {
135 static unsigned getHashValue(const sampleprof::Hashable<T> &Key) {
136 // Don't make it virtual for getHashCode
137 uint64_t Hash = Key.Data->getHashCode();
138 assert(Hash && "Should generate HashCode for it!");
139 return DenseMapInfo<uint64_t>::getHashValue(Val: Hash);
140 }
141
142 static bool isEqual(const sampleprof::Hashable<T> &LHS,
143 const sampleprof::Hashable<T> &RHS) {
144 // Precisely compare the data, vtable will have overhead.
145 return LHS.Data->isEqual(RHS.Data.get());
146 }
147};
148
149namespace sampleprof {
150
151struct PerfSample {
152 // LBR stack recorded in FIFO order.
153 SmallVector<LBREntry, 16> LBRStack;
154 // Call stack recorded in FILO(leaf to root) order, it's used for CS-profile
155 // generation
156 SmallVector<uint64_t, 16> CallStack;
157
158 virtual ~PerfSample() = default;
159 uint64_t getHashCode() const {
160 // Use simple DJB2 hash
161 auto HashCombine = [](uint64_t H, uint64_t V) {
162 return ((H << 5) + H) + V;
163 };
164 uint64_t Hash = 5381;
165 for (const auto &Value : CallStack) {
166 Hash = HashCombine(Hash, Value);
167 }
168 for (const auto &Entry : LBRStack) {
169 Hash = HashCombine(Hash, Entry.Source);
170 Hash = HashCombine(Hash, Entry.Target);
171 }
172 return Hash;
173 }
174
175 bool isEqual(const PerfSample *Other) const {
176 const SmallVector<uint64_t, 16> &OtherCallStack = Other->CallStack;
177 const SmallVector<LBREntry, 16> &OtherLBRStack = Other->LBRStack;
178
179 if (CallStack.size() != OtherCallStack.size() ||
180 LBRStack.size() != OtherLBRStack.size())
181 return false;
182
183 if (!std::equal(first1: CallStack.begin(), last1: CallStack.end(), first2: OtherCallStack.begin()))
184 return false;
185
186 for (size_t I = 0; I < OtherLBRStack.size(); I++) {
187 if (LBRStack[I].Source != OtherLBRStack[I].Source ||
188 LBRStack[I].Target != OtherLBRStack[I].Target)
189 return false;
190 }
191 return true;
192 }
193
194#ifndef NDEBUG
195 uint64_t Linenum = 0;
196
197 void print() const {
198 dbgs() << "Line " << Linenum << "\n";
199 dbgs() << "LBR stack\n";
200 printLBRStack(LBRStack);
201 dbgs() << "Call stack\n";
202 printCallStack(CallStack);
203 }
204#endif
205};
206// After parsing the sample, we record the samples by aggregating them
207// into this counter. The key stores the sample data and the value is
208// the sample repeat times.
209using AggregatedCounter = DenseMap<Hashable<PerfSample>, uint64_t>;
210
211using SampleVector = SmallVector<std::tuple<uint64_t, uint64_t, uint64_t>, 16>;
212
213inline bool isValidFallThroughRange(uint64_t Start, uint64_t End,
214 ProfiledBinary *Binary) {
215 // Start bigger than End is considered invalid.
216 // LBR ranges cross the unconditional jmp are also assumed invalid.
217 // It's found that perf data may contain duplicate LBR entries that could form
218 // a range that does not reflect real execution flow on some Intel targets,
219 // e.g. Skylake. Such ranges are ususally very long. Exclude them since there
220 // cannot be a linear execution range that spans over unconditional jmp.
221 return Start <= End && !Binary->rangeCrossUncondBranch(Start, End);
222}
223
224// The state for the unwinder, it doesn't hold the data but only keep the
225// pointer/index of the data, While unwinding, the CallStack is changed
226// dynamicially and will be recorded as the context of the sample
227struct UnwindState {
228 // Profiled binary that current frame address belongs to
229 const ProfiledBinary *Binary;
230 // Call stack trie node
231 struct ProfiledFrame {
232 const uint64_t Address = DummyRoot;
233 ProfiledFrame *Parent;
234 SampleVector RangeSamples;
235 SampleVector BranchSamples;
236 DenseMap<uint64_t, std::unique_ptr<ProfiledFrame>> Children;
237
238 ProfiledFrame(uint64_t Addr = 0, ProfiledFrame *P = nullptr)
239 : Address(Addr), Parent(P) {}
240 ProfiledFrame *getOrCreateChildFrame(uint64_t Address) {
241 assert(Address && "Address can't be zero!");
242 auto [It, Inserted] = Children.try_emplace(Key: Address);
243 if (Inserted)
244 It->second = std::make_unique<ProfiledFrame>(args&: Address, args: this);
245 return It->second.get();
246 }
247 void recordRangeCount(uint64_t Start, uint64_t End, uint64_t Count) {
248 RangeSamples.emplace_back(Args: std::make_tuple(args&: Start, args&: End, args&: Count));
249 }
250 void recordBranchCount(uint64_t Source, uint64_t Target, uint64_t Count) {
251 BranchSamples.emplace_back(Args: std::make_tuple(args&: Source, args&: Target, args&: Count));
252 }
253 bool isDummyRoot() { return Address == DummyRoot; }
254 bool isExternalFrame() { return Address == ExternalAddr; }
255 bool isLeafFrame() { return Children.empty(); }
256 };
257
258 ProfiledFrame DummyTrieRoot;
259 ProfiledFrame *CurrentLeafFrame;
260 // Used to fall through the LBR stack
261 uint32_t LBRIndex = 0;
262 // Reference to PerfSample.LBRStack
263 const SmallVector<LBREntry, 16> &LBRStack;
264 // Used to iterate the address range
265 InstructionPointer InstPtr;
266 // Indicate whether unwinding is currently in a bad state which requires to
267 // skip all subsequent unwinding.
268 bool Invalid = false;
269 UnwindState(const PerfSample *Sample, const ProfiledBinary *Binary)
270 : Binary(Binary), LBRStack(Sample->LBRStack),
271 InstPtr(Binary, Sample->CallStack.front()) {
272 initFrameTrie(CallStack: Sample->CallStack);
273 }
274
275 void checkStateConsistency() {
276 assert(InstPtr.Address == CurrentLeafFrame->Address &&
277 "IP should align with context leaf");
278 }
279
280 void setInvalid() { Invalid = true; }
281 bool hasNextLBR() const { return LBRIndex < LBRStack.size(); }
282 uint64_t getCurrentLBRSource() const { return LBRStack[LBRIndex].Source; }
283 uint64_t getCurrentLBRTarget() const { return LBRStack[LBRIndex].Target; }
284 const LBREntry &getCurrentLBR() const { return LBRStack[LBRIndex]; }
285 bool IsLastLBR() const { return LBRIndex == 0; }
286 size_t getLBRStackSize() const { return LBRStack.size(); }
287 void advanceLBR() { LBRIndex++; }
288 ProfiledFrame *getParentFrame() { return CurrentLeafFrame->Parent; }
289
290 void pushFrame(uint64_t Address) {
291 CurrentLeafFrame = CurrentLeafFrame->getOrCreateChildFrame(Address);
292 }
293
294 void switchToFrame(uint64_t Address) {
295 if (CurrentLeafFrame->Address == Address)
296 return;
297 CurrentLeafFrame = CurrentLeafFrame->Parent->getOrCreateChildFrame(Address);
298 }
299
300 void popFrame() { CurrentLeafFrame = CurrentLeafFrame->Parent; }
301
302 void clearCallStack() { CurrentLeafFrame = &DummyTrieRoot; }
303
304 void initFrameTrie(const SmallVectorImpl<uint64_t> &CallStack) {
305 ProfiledFrame *Cur = &DummyTrieRoot;
306 for (auto Address : reverse(C: CallStack)) {
307 Cur = Cur->getOrCreateChildFrame(Address);
308 }
309 CurrentLeafFrame = Cur;
310 }
311
312 ProfiledFrame *getDummyRootPtr() { return &DummyTrieRoot; }
313};
314
315// Base class for sample counter key with context
316struct ContextKey {
317 uint64_t HashCode = 0;
318 virtual ~ContextKey() = default;
319 uint64_t getHashCode() {
320 if (HashCode == 0)
321 genHashCode();
322 return HashCode;
323 }
324 virtual void genHashCode() = 0;
325 virtual bool isEqual(const ContextKey *K) const {
326 return HashCode == K->HashCode;
327 };
328
329 // Utilities for LLVM-style RTTI
330 enum ContextKind { CK_StringBased, CK_AddrBased };
331 const ContextKind Kind;
332 ContextKind getKind() const { return Kind; }
333 ContextKey(ContextKind K) : Kind(K){};
334};
335
336// String based context id
337struct StringBasedCtxKey : public ContextKey {
338 SampleContextFrameVector Context;
339
340 bool WasLeafInlined;
341 StringBasedCtxKey() : ContextKey(CK_StringBased), WasLeafInlined(false){};
342 static bool classof(const ContextKey *K) {
343 return K->getKind() == CK_StringBased;
344 }
345
346 bool isEqual(const ContextKey *K) const override {
347 const StringBasedCtxKey *Other = dyn_cast<StringBasedCtxKey>(Val: K);
348 return Context == Other->Context;
349 }
350
351 void genHashCode() override {
352 HashCode = hash_value(S: SampleContextFrames(Context));
353 }
354};
355
356// Address-based context id
357struct AddrBasedCtxKey : public ContextKey {
358 SmallVector<uint64_t, 16> Context;
359
360 bool WasLeafInlined;
361 AddrBasedCtxKey() : ContextKey(CK_AddrBased), WasLeafInlined(false){};
362 static bool classof(const ContextKey *K) {
363 return K->getKind() == CK_AddrBased;
364 }
365
366 bool isEqual(const ContextKey *K) const override {
367 const AddrBasedCtxKey *Other = dyn_cast<AddrBasedCtxKey>(Val: K);
368 return Context == Other->Context;
369 }
370
371 void genHashCode() override { HashCode = hash_combine_range(R&: Context); }
372};
373
374// The counter of branch samples for one function indexed by the branch,
375// which is represented as the source and target offset pair.
376using BranchSample = std::map<std::pair<uint64_t, uint64_t>, uint64_t>;
377// The counter of range samples for one function indexed by the range,
378// which is represented as the start and end offset pair.
379using RangeSample = std::map<std::pair<uint64_t, uint64_t>, uint64_t>;
380// <<inst-addr, vtable-data-symbol>, count> map for data access samples.
381// The instruction address is the virtual address in the binary.
382using DataAccessSample = std::map<std::pair<uint64_t, StringRef>, uint64_t>;
383// Wrapper for sample counters including range counter and branch counter
384struct SampleCounter {
385 RangeSample RangeCounter;
386 BranchSample BranchCounter;
387 DataAccessSample DataAccessCounter;
388
389 void recordRangeCount(uint64_t Start, uint64_t End, uint64_t Repeat) {
390 assert(Start <= End && "Invalid instruction range");
391 RangeCounter[{Start, End}] += Repeat;
392 }
393 void recordBranchCount(uint64_t Source, uint64_t Target, uint64_t Repeat) {
394 BranchCounter[{Source, Target}] += Repeat;
395 }
396 void recordDataAccessCount(uint64_t InstAddr, StringRef DataSymbol,
397 uint64_t Repeat) {
398 DataAccessCounter[{InstAddr, DataSymbol}] += Repeat;
399 }
400};
401
402// Sample counter with context to support context-sensitive profile
403using ContextSampleCounterMap = DenseMap<Hashable<ContextKey>, SampleCounter>;
404
405struct FrameStack {
406 SmallVector<uint64_t, 16> Stack;
407 ProfiledBinary *Binary;
408 FrameStack(ProfiledBinary *B) : Binary(B) {}
409 bool pushFrame(UnwindState::ProfiledFrame *Cur) {
410 assert(!Cur->isExternalFrame() &&
411 "External frame's not expected for context stack.");
412 Stack.push_back(Elt: Cur->Address);
413 return true;
414 }
415
416 void popFrame() {
417 if (!Stack.empty())
418 Stack.pop_back();
419 }
420 std::shared_ptr<StringBasedCtxKey> getContextKey();
421};
422
423struct AddressStack {
424 SmallVector<uint64_t, 16> Stack;
425 ProfiledBinary *Binary;
426 AddressStack(ProfiledBinary *B) : Binary(B) {}
427 bool pushFrame(UnwindState::ProfiledFrame *Cur) {
428 assert(!Cur->isExternalFrame() &&
429 "External frame's not expected for context stack.");
430 Stack.push_back(Elt: Cur->Address);
431 return true;
432 }
433
434 void popFrame() {
435 if (!Stack.empty())
436 Stack.pop_back();
437 }
438 std::shared_ptr<AddrBasedCtxKey> getContextKey();
439};
440
441/*
442As in hybrid sample we have a group of LBRs and the most recent sampling call
443stack, we can walk through those LBRs to infer more call stacks which would be
444used as context for profile. VirtualUnwinder is the class to do the call stack
445unwinding based on LBR state. Two types of unwinding are processd here:
4461) LBR unwinding and 2) linear range unwinding.
447Specifically, for each LBR entry(can be classified into call, return, regular
448branch), LBR unwinding will replay the operation by pushing, popping or
449switching leaf frame towards the call stack and since the initial call stack
450is most recently sampled, the replay should be in anti-execution order, i.e. for
451the regular case, pop the call stack when LBR is call, push frame on call stack
452when LBR is return. After each LBR processed, it also needs to align with the
453next LBR by going through instructions from previous LBR's target to current
454LBR's source, which is the linear unwinding. As instruction from linear range
455can come from different function by inlining, linear unwinding will do the range
456splitting and record counters by the range with same inline context. Over those
457unwinding process we will record each call stack as context id and LBR/linear
458range as sample counter for further CS profile generation.
459*/
460class VirtualUnwinder {
461public:
462 VirtualUnwinder(ContextSampleCounterMap *Counter, ProfiledBinary *B)
463 : CtxCounterMap(Counter), Binary(B) {}
464 bool unwind(const PerfSample *Sample, uint64_t Repeat);
465 std::set<uint64_t> &getUntrackedCallsites() { return UntrackedCallsites; }
466
467 uint64_t NumTotalBranches = 0;
468 uint64_t NumExtCallBranch = 0;
469 uint64_t NumMissingExternalFrame = 0;
470 uint64_t NumMismatchedProEpiBranch = 0;
471 uint64_t NumMismatchedExtCallBranch = 0;
472 uint64_t NumUnpairedExtAddr = 0;
473 uint64_t NumPairedExtAddr = 0;
474
475private:
476 bool isSourceExternal(UnwindState &State) const {
477 return State.getCurrentLBRSource() == ExternalAddr;
478 }
479
480 bool isTargetExternal(UnwindState &State) const {
481 return State.getCurrentLBRTarget() == ExternalAddr;
482 }
483
484 // Determine whether the return source is from external code by checking if
485 // the target's the next inst is a call inst.
486 bool isReturnFromExternal(UnwindState &State) const {
487 return isSourceExternal(State) &&
488 (Binary->getCallAddrFromFrameAddr(FrameAddr: State.getCurrentLBRTarget()) != 0);
489 }
490
491 // If the source is external address but it's not the `return` case, treat it
492 // as a call from external.
493 bool isCallFromExternal(UnwindState &State) const {
494 return isSourceExternal(State) &&
495 Binary->getCallAddrFromFrameAddr(FrameAddr: State.getCurrentLBRTarget()) == 0;
496 }
497
498 bool isCallState(UnwindState &State) const {
499 // The tail call frame is always missing here in stack sample, we will
500 // use a specific tail call tracker to infer it.
501 if (!isValidState(State))
502 return false;
503
504 if (Binary->addressIsCall(Address: State.getCurrentLBRSource()))
505 return true;
506
507 return isCallFromExternal(State);
508 }
509
510 bool isReturnState(UnwindState &State) const {
511 if (!isValidState(State))
512 return false;
513
514 // Simply check addressIsReturn, as ret is always reliable, both for
515 // regular call and tail call.
516 if (Binary->addressIsReturn(Address: State.getCurrentLBRSource()))
517 return true;
518
519 return isReturnFromExternal(State);
520 }
521
522 bool isValidState(UnwindState &State) const { return !State.Invalid; }
523
524 void unwindCall(UnwindState &State);
525 void unwindLinear(UnwindState &State, uint64_t Repeat);
526 void unwindReturn(UnwindState &State);
527 void unwindBranch(UnwindState &State);
528
529 template <typename T>
530 void collectSamplesFromFrame(UnwindState::ProfiledFrame *Cur, T &Stack);
531 // Collect each samples on trie node by DFS traversal
532 template <typename T>
533 void collectSamplesFromFrameTrie(UnwindState::ProfiledFrame *Cur, T &Stack);
534 void collectSamplesFromFrameTrie(UnwindState::ProfiledFrame *Cur);
535
536 void recordBranchCount(const LBREntry &Branch, UnwindState &State,
537 uint64_t Repeat);
538
539 ContextSampleCounterMap *CtxCounterMap;
540 // Profiled binary that current frame address belongs to
541 ProfiledBinary *Binary;
542 // Keep track of all untracked callsites
543 std::set<uint64_t> UntrackedCallsites;
544};
545
546// Read perf trace to parse the events and samples.
547class PerfReaderBase {
548public:
549 PerfReaderBase(ProfiledBinary *B, StringRef PerfTrace)
550 : Binary(B), PerfTraceFile(PerfTrace) {
551 // Initialize the base address to preferred address.
552 Binary->setBaseAddress(Binary->getPreferredBaseAddress());
553 };
554 virtual ~PerfReaderBase() = default;
555 static std::unique_ptr<PerfReaderBase>
556 create(ProfiledBinary *Binary, InputFile &Input,
557 std::optional<int32_t> PIDFilter);
558
559 // Entry of the reader to parse multiple perf traces
560 virtual void parsePerfTraces() = 0;
561
562 // Parse the <ip, vtable-data-symbol> from the data access perf trace file,
563 // and accumulate the data access count for each <ip, data-symbol> pair.
564 Error
565 parseDataAccessPerfTraces(StringRef DataAccessPerfFile,
566 std::optional<int32_t> PIDFilter = std::nullopt);
567
568 const ContextSampleCounterMap &getSampleCounters() const {
569 return SampleCounters;
570 }
571 bool profileIsCS() { return ProfileIsCS; }
572
573protected:
574 ProfiledBinary *Binary = nullptr;
575 StringRef PerfTraceFile;
576
577 ContextSampleCounterMap SampleCounters;
578 bool ProfileIsCS = false;
579
580 uint64_t NumTotalSample = 0;
581 uint64_t NumLeafExternalFrame = 0;
582 uint64_t NumLeadingOutgoingLBR = 0;
583};
584
585// Read perf script to parse the events and samples.
586class PerfScriptReader : public PerfReaderBase {
587public:
588 PerfScriptReader(ProfiledBinary *B, StringRef PerfTrace,
589 std::optional<int32_t> PID)
590 : PerfReaderBase(B, PerfTrace), PIDFilter(PID) {};
591
592 // Entry of the reader to parse multiple perf traces
593 void parsePerfTraces() override;
594
595 // Parse a single line of a PERF_RECORD_MMAP event looking for a
596 // mapping between the binary name and its memory layout.
597 // TODO: Move this static method from PerScriptReader (subclass) to
598 // PerfReaderBase (superclass).
599 static bool extractMMapEventForBinary(ProfiledBinary *Binary, StringRef Line,
600 MMapEvent &MMap);
601
602 // Generate perf script from perf data
603 static InputFile convertPerfDataToTrace(ProfiledBinary *Binary, bool SkipPID,
604 InputFile &File,
605 std::optional<int32_t> PIDFilter);
606 // Extract perf script type by peaking at the input
607 static PerfContent checkPerfScriptType(StringRef FileName);
608
609 // Cleanup installers for temporary files created by perf script command.
610 // Those files will be automatically removed when running destructor or
611 // receiving signals.
612 static SmallVector<CleanupInstaller, 2> TempFileCleanups;
613
614protected:
615 // Check whether a given line is LBR sample
616 static bool isLBRSample(StringRef Line, bool CheckLineStart);
617 // Check whether a given line is MMAP event
618 static bool isMMapEvent(StringRef Line);
619 // Update base address based on mmap events
620 void updateBinaryAddress(const MMapEvent &Event);
621 // Parse mmap event and update binary address
622 void parseMMapEvent(TraceStream &TraceIt);
623 // Parse perf events/samples and do aggregation
624 void parseAndAggregateTrace();
625 // Parse either an MMAP event or a perf sample
626 void parseEventOrSample(TraceStream &TraceIt);
627 // Warn if the relevant mmap event is missing.
628 void warnIfMissingMMap();
629 // Emit accumulate warnings.
630 void warnTruncatedStack();
631 // Warn if range is invalid.
632 void warnInvalidRange();
633 // Warn if sampled branch/target addresses don't match the binary.
634 void warnIfBranchTargetMismatch();
635 // Extract call stack from the perf trace lines
636 bool extractCallstack(TraceStream &TraceIt,
637 SmallVectorImpl<uint64_t> &CallStack);
638 // Extract LBR stack from one perf trace line
639 bool extractLBRStack(TraceStream &TraceIt,
640 SmallVectorImpl<LBREntry> &LBRStack);
641 uint64_t parseAggregatedCount(TraceStream &TraceIt);
642 // Parse one sample from multiple perf lines, override this for different
643 // sample type
644 void parseSample(TraceStream &TraceIt);
645 // An aggregated count is given to indicate how many times the sample is
646 // repeated.
647 virtual void parseSample(TraceStream &TraceIt, uint64_t Count){};
648 void computeCounterFromLBR(const PerfSample *Sample, uint64_t Repeat);
649 // Post process the profile after trace aggregation, we will do simple range
650 // overlap computation for AutoFDO, or unwind for CSSPGO(hybrid sample).
651 virtual void generateUnsymbolizedProfile();
652 void writeUnsymbolizedProfile(StringRef Filename);
653 void writeUnsymbolizedProfile(raw_fd_ostream &OS);
654
655 // Samples with the repeating time generated by the perf reader
656 AggregatedCounter AggregatedSamples;
657 // Keep track of all invalid return addresses
658 std::set<uint64_t> InvalidReturnAddresses;
659 // PID for the process of interest
660 std::optional<int32_t> PIDFilter;
661};
662
663/*
664 The reader of LBR only perf script.
665 A typical LBR sample is like:
666 40062f 0x4005c8/0x4005dc/P/-/-/0 0x40062f/0x4005b0/P/-/-/0 ...
667 ... 0x4005c8/0x4005dc/P/-/-/0
668*/
669class LBRPerfReader : public PerfScriptReader {
670public:
671 LBRPerfReader(ProfiledBinary *Binary, StringRef PerfTrace,
672 std::optional<int32_t> PID)
673 : PerfScriptReader(Binary, PerfTrace, PID) {};
674 // Parse the LBR only sample.
675 void parseSample(TraceStream &TraceIt, uint64_t Count) override;
676};
677
678/*
679 Hybrid perf script includes a group of hybrid samples(LBRs + call stack),
680 which is used to generate CS profile. An example of hybrid sample:
681 4005dc # call stack leaf
682 400634
683 400684 # call stack root
684 0x4005c8/0x4005dc/P/-/-/0 0x40062f/0x4005b0/P/-/-/0 ...
685 ... 0x4005c8/0x4005dc/P/-/-/0 # LBR Entries
686*/
687class HybridPerfReader : public PerfScriptReader {
688public:
689 HybridPerfReader(ProfiledBinary *Binary, StringRef PerfTrace,
690 std::optional<int32_t> PID)
691 : PerfScriptReader(Binary, PerfTrace, PID) {};
692 // Parse the hybrid sample including the call and LBR line
693 void parseSample(TraceStream &TraceIt, uint64_t Count) override;
694 void generateUnsymbolizedProfile() override;
695
696private:
697 // Unwind the hybrid samples after aggregration
698 void unwindSamples();
699
700 uint64_t NumBogusTrace = 0;
701 uint64_t NumTotalHybridSample = 0;
702};
703
704/*
705 Format of unsymbolized profile:
706
707 [frame1 @ frame2 @ ...] # If it's a CS profile
708 number of entries in RangeCounter
709 from_1-to_1:count_1
710 from_2-to_2:count_2
711 ......
712 from_n-to_n:count_n
713 number of entries in BranchCounter
714 src_1->dst_1:count_1
715 src_2->dst_2:count_2
716 ......
717 src_n->dst_n:count_n
718 [frame1 @ frame2 @ ...] # Next context
719 ......
720
721Note that non-CS profile doesn't have the empty `[]` context.
722*/
723class UnsymbolizedProfileReader : public PerfReaderBase {
724public:
725 UnsymbolizedProfileReader(ProfiledBinary *Binary, StringRef PerfTrace)
726 : PerfReaderBase(Binary, PerfTrace){};
727 void parsePerfTraces() override;
728
729private:
730 void readSampleCounters(TraceStream &TraceIt, SampleCounter &SCounters);
731 void readUnsymbolizedProfile(StringRef Filename);
732
733 StringSet<> ContextStrSet;
734};
735
736class ETMReader {
737public:
738 ETMReader(ProfiledBinary *Binary, StringRef TraceFile, uint8_t TraceID)
739 : Binary(Binary), TraceFile(TraceFile), TraceID(TraceID) {}
740 void parseETMTraces();
741 void recordProcessedRange(uint64_t Start, uint64_t End, uint64_t Count);
742 const ContextSampleCounterMap &getSampleCounters() const { return Counters; }
743
744private:
745 ProfiledBinary *Binary = nullptr;
746 StringRef TraceFile;
747 uint8_t TraceID;
748 ContextSampleCounterMap Counters;
749};
750
751} // end namespace sampleprof
752} // end namespace llvm
753
754#endif
755