1//===-- PerfReader.cpp - perfscript reader ---------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8#include "PerfReader.h"
9#include "ErrorHandling.h"
10#include "Options.h"
11#include "ProfileGenerator.h"
12#include "llvm/ADT/SmallString.h"
13#include "llvm/DebugInfo/Symbolize/SymbolizableModule.h"
14#include "llvm/ProfileData/ETMTraceDecoder.h"
15#include "llvm/Support/FileSystem.h"
16#include "llvm/Support/LineIterator.h"
17#include "llvm/Support/MemoryBuffer.h"
18#include "llvm/Support/Process.h"
19#include "llvm/Support/Timer.h"
20#include "llvm/Support/ToolOutputFile.h"
21#include "llvm/Support/raw_ostream.h"
22#include "llvm/TargetParser/Triple.h"
23#include <cctype>
24
25#define DEBUG_TYPE "perf-reader"
26
27namespace llvm {
28
29cl::opt<bool> SkipSymbolization("skip-symbolization",
30 cl::desc("Dump the unsymbolized profile to the "
31 "output file. It will show unwinder "
32 "output for CS profile generation."),
33 cl::cat(ProfGenCategory));
34
35static cl::opt<bool> ShowMmapEvents("show-mmap-events",
36 cl::desc("Print binary load events."),
37 cl::cat(ProfGenCategory));
38
39static cl::opt<bool>
40 UseOffset("use-offset", cl::init(Val: true),
41 cl::desc("Work with `--skip-symbolization` or "
42 "`--unsymbolized-profile` to write/read the "
43 "offset instead of virtual address."),
44 cl::cat(ProfGenCategory));
45
46static cl::opt<bool> UseLoadableSegmentAsBase(
47 "use-first-loadable-segment-as-base",
48 cl::desc("Use first loadable segment address as base address "
49 "for offsets in unsymbolized profile. By default "
50 "first executable segment address is used"),
51 cl::cat(ProfGenCategory));
52
53static cl::opt<bool>
54 IgnoreStackSamples("ignore-stack-samples",
55 cl::desc("Ignore call stack samples for hybrid samples "
56 "and produce context-insensitive profile."),
57 cl::cat(ProfGenCategory));
58cl::opt<bool> ShowDetailedWarning("show-detailed-warning",
59 cl::desc("Show detailed warning message."),
60 cl::cat(ProfGenCategory));
61
62static cl::opt<int> CSProfMaxUnsymbolizedCtxDepth(
63 "csprof-max-unsymbolized-context-depth", cl::init(Val: -1),
64 cl::desc("Keep the last K contexts while merging unsymbolized profile. -1 "
65 "means no depth limit."),
66 cl::cat(ProfGenCategory));
67
68cl::opt<bool> TimeProfGen("time-profgen", cl::desc("Time llvm-profgen phases"),
69 cl::init(Val: false), cl::cat(ProfGenCategory));
70
71static const char *TimerGroupName = "profgen";
72static const char *TimerGroupDesc = "llvm-profgen";
73
74namespace sampleprof {
75
76void VirtualUnwinder::unwindCall(UnwindState &State) {
77 uint64_t Source = State.getCurrentLBRSource();
78 auto *ParentFrame = State.getParentFrame();
79 // The 2nd frame after leaf could be missing if stack sample is
80 // taken when IP is within prolog/epilog, as frame chain isn't
81 // setup yet. Fill in the missing frame in that case.
82 // TODO: Currently we just assume all the addr that can't match the
83 // 2nd frame is in prolog/epilog. In the future, we will switch to
84 // pro/epi tracker(Dwarf CFI) for the precise check.
85 if (ParentFrame == State.getDummyRootPtr() ||
86 ParentFrame->Address != Source) {
87 State.switchToFrame(Address: Source);
88 if (ParentFrame != State.getDummyRootPtr()) {
89 if (Source == ExternalAddr)
90 NumMismatchedExtCallBranch++;
91 else
92 NumMismatchedProEpiBranch++;
93 }
94 } else {
95 State.popFrame();
96 }
97 State.InstPtr.update(Addr: Source);
98}
99
100void VirtualUnwinder::unwindLinear(UnwindState &State, uint64_t Repeat) {
101 InstructionPointer &IP = State.InstPtr;
102 uint64_t Target = State.getCurrentLBRTarget();
103 uint64_t End = IP.Address;
104
105 if (End == ExternalAddr && Target == ExternalAddr) {
106 // Filter out the case when leaf external frame matches the external LBR
107 // target, this is a valid state, it happens that the code run into external
108 // address then return back. The call frame under the external frame
109 // remains valid and can be unwound later, just skip recording this range.
110 NumPairedExtAddr++;
111 return;
112 }
113
114 if (End == ExternalAddr || Target == ExternalAddr) {
115 // Range is invalid if only one point is external address. This means LBR
116 // traces contains a standalone external address failing to pair another
117 // one, likely due to interrupt jmp or broken perf script. Set the
118 // state to invalid.
119 NumUnpairedExtAddr++;
120 State.setInvalid();
121 return;
122 }
123
124 if (!isValidFallThroughRange(Start: Target, End, Binary)) {
125 // Skip unwinding the rest of LBR trace when a bogus range is seen.
126 State.setInvalid();
127 return;
128 }
129
130 if (Binary->usePseudoProbes()) {
131 // We don't need to top frame probe since it should be extracted
132 // from the range.
133 // The outcome of the virtual unwinding with pseudo probes is a
134 // map from a context key to the address range being unwound.
135 // This means basically linear unwinding is not needed for pseudo
136 // probes. The range will be simply recorded here and will be
137 // converted to a list of pseudo probes to report in ProfileGenerator.
138 State.getParentFrame()->recordRangeCount(Start: Target, End, Count: Repeat);
139 } else {
140 // Unwind linear execution part.
141 // Split and record the range by different inline context. For example:
142 // [0x01] ... main:1 # Target
143 // [0x02] ... main:2
144 // [0x03] ... main:3 @ foo:1
145 // [0x04] ... main:3 @ foo:2
146 // [0x05] ... main:3 @ foo:3
147 // [0x06] ... main:4
148 // [0x07] ... main:5 # End
149 // It will be recorded:
150 // [main:*] : [0x06, 0x07], [0x01, 0x02]
151 // [main:3 @ foo:*] : [0x03, 0x05]
152 while (IP.Address > Target) {
153 uint64_t PrevIP = IP.Address;
154 IP.backward();
155 // Break into segments for implicit call/return due to inlining
156 bool SameInlinee = Binary->inlineContextEqual(Add1: PrevIP, Add2: IP.Address);
157 if (!SameInlinee) {
158 State.switchToFrame(Address: PrevIP);
159 State.CurrentLeafFrame->recordRangeCount(Start: PrevIP, End, Count: Repeat);
160 End = IP.Address;
161 }
162 }
163 assert(IP.Address == Target && "The last one must be the target address.");
164 // Record the remaining range, [0x01, 0x02] in the example
165 State.switchToFrame(Address: IP.Address);
166 State.CurrentLeafFrame->recordRangeCount(Start: IP.Address, End, Count: Repeat);
167 }
168}
169
170void VirtualUnwinder::unwindReturn(UnwindState &State) {
171 // Add extra frame as we unwind through the return
172 const LBREntry &LBR = State.getCurrentLBR();
173 uint64_t CallAddr = Binary->getCallAddrFromFrameAddr(FrameAddr: LBR.Target);
174 State.switchToFrame(Address: CallAddr);
175 State.pushFrame(Address: LBR.Source);
176 State.InstPtr.update(Addr: LBR.Source);
177}
178
179void VirtualUnwinder::unwindBranch(UnwindState &State) {
180 // TODO: Tolerate tail call for now, as we may see tail call from libraries.
181 // This is only for intra function branches, excluding tail calls.
182 uint64_t Source = State.getCurrentLBRSource();
183 State.switchToFrame(Address: Source);
184 State.InstPtr.update(Addr: Source);
185}
186
187std::shared_ptr<StringBasedCtxKey> FrameStack::getContextKey() {
188 std::shared_ptr<StringBasedCtxKey> KeyStr =
189 std::make_shared<StringBasedCtxKey>();
190 KeyStr->Context = Binary->getExpandedContext(Stack, WasLeafInlined&: KeyStr->WasLeafInlined);
191 return KeyStr;
192}
193
194std::shared_ptr<AddrBasedCtxKey> AddressStack::getContextKey() {
195 std::shared_ptr<AddrBasedCtxKey> KeyStr = std::make_shared<AddrBasedCtxKey>();
196 KeyStr->Context = Stack;
197 CSProfileGenerator::compressRecursionContext<uint64_t>(Context&: KeyStr->Context);
198 // MaxContextDepth(--csprof-max-context-depth) is used to trim both symbolized
199 // and unsymbolized profile context. Sometimes we want to at least preserve
200 // the inlinings for the leaf frame(the profiled binary inlining),
201 // --csprof-max-context-depth may not be flexible enough, in this case,
202 // --csprof-max-unsymbolized-context-depth is used to limit the context for
203 // unsymbolized profile. If both are set, use the minimum of them.
204 int Depth = CSProfileGenerator::MaxContextDepth != -1
205 ? CSProfileGenerator::MaxContextDepth
206 : KeyStr->Context.size();
207 Depth = CSProfMaxUnsymbolizedCtxDepth != -1
208 ? std::min(a: static_cast<int>(CSProfMaxUnsymbolizedCtxDepth), b: Depth)
209 : Depth;
210 CSProfileGenerator::trimContext<uint64_t>(S&: KeyStr->Context, Depth);
211 return KeyStr;
212}
213
214template <typename T>
215void VirtualUnwinder::collectSamplesFromFrame(UnwindState::ProfiledFrame *Cur,
216 T &Stack) {
217 if (Cur->RangeSamples.empty() && Cur->BranchSamples.empty())
218 return;
219
220 std::shared_ptr<ContextKey> Key = Stack.getContextKey();
221 if (Key == nullptr)
222 return;
223 auto Ret = CtxCounterMap->try_emplace(Key: Hashable<ContextKey>(Key));
224 SampleCounter &SCounter = Ret.first->second;
225 for (auto &I : Cur->RangeSamples)
226 SCounter.recordRangeCount(Start: std::get<0>(t&: I), End: std::get<1>(t&: I), Repeat: std::get<2>(t&: I));
227
228 for (auto &I : Cur->BranchSamples)
229 SCounter.recordBranchCount(Source: std::get<0>(t&: I), Target: std::get<1>(t&: I), Repeat: std::get<2>(t&: I));
230}
231
232template <typename T>
233void VirtualUnwinder::collectSamplesFromFrameTrie(
234 UnwindState::ProfiledFrame *Cur, T &Stack) {
235 if (!Cur->isDummyRoot()) {
236 // Truncate the context for external frame since this isn't a real call
237 // context the compiler will see.
238 if (Cur->isExternalFrame() || !Stack.pushFrame(Cur)) {
239 // Process truncated context
240 // Start a new traversal ignoring its bottom context
241 T EmptyStack(Binary);
242 collectSamplesFromFrame(Cur, EmptyStack);
243 for (const auto &Item : Cur->Children) {
244 collectSamplesFromFrameTrie(Item.second.get(), EmptyStack);
245 }
246
247 // Keep note of untracked call site and deduplicate them
248 // for warning later.
249 if (!Cur->isLeafFrame())
250 UntrackedCallsites.insert(x: Cur->Address);
251
252 return;
253 }
254 }
255
256 collectSamplesFromFrame(Cur, Stack);
257 // Process children frame
258 for (const auto &Item : Cur->Children) {
259 collectSamplesFromFrameTrie(Item.second.get(), Stack);
260 }
261 // Recover the call stack
262 Stack.popFrame();
263}
264
265void VirtualUnwinder::collectSamplesFromFrameTrie(
266 UnwindState::ProfiledFrame *Cur) {
267 if (Binary->usePseudoProbes()) {
268 AddressStack Stack(Binary);
269 collectSamplesFromFrameTrie<AddressStack>(Cur, Stack);
270 } else {
271 FrameStack Stack(Binary);
272 collectSamplesFromFrameTrie<FrameStack>(Cur, Stack);
273 }
274}
275
276void VirtualUnwinder::recordBranchCount(const LBREntry &Branch,
277 UnwindState &State, uint64_t Repeat) {
278 if (Branch.Target == ExternalAddr)
279 return;
280
281 // Record external-to-internal pattern on the trie root, it later can be
282 // used for generating head samples.
283 if (Branch.Source == ExternalAddr) {
284 State.getDummyRootPtr()->recordBranchCount(Source: Branch.Source, Target: Branch.Target,
285 Count: Repeat);
286 return;
287 }
288
289 if (Binary->usePseudoProbes()) {
290 // Same as recordRangeCount, We don't need to top frame probe since we will
291 // extract it from branch's source address
292 State.getParentFrame()->recordBranchCount(Source: Branch.Source, Target: Branch.Target,
293 Count: Repeat);
294 } else {
295 State.CurrentLeafFrame->recordBranchCount(Source: Branch.Source, Target: Branch.Target,
296 Count: Repeat);
297 }
298}
299
300bool VirtualUnwinder::unwind(const PerfSample *Sample, uint64_t Repeat) {
301 // Capture initial state as starting point for unwinding.
302 UnwindState State(Sample, Binary);
303
304 NumTotalBranches += State.LBRStack.size();
305 // Now process the LBR samples in parrallel with stack sample
306 // Note that we do not reverse the LBR entry order so we can
307 // unwind the sample stack as we walk through LBR entries.
308 while (State.hasNextLBR()) {
309 State.checkStateConsistency();
310
311 // Do not attempt linear unwind for the leaf range as it's incomplete.
312 if (!State.IsLastLBR()) {
313 // Unwind implicit calls/returns from inlining, along the linear path,
314 // break into smaller sub section each with its own calling context.
315 unwindLinear(State, Repeat);
316 }
317
318 // Save the LBR branch before it gets unwound.
319 const LBREntry &Branch = State.getCurrentLBR();
320 if (isCallState(State)) {
321 // Unwind calls - we know we encountered call if LBR overlaps with
322 // transition between leaf the 2nd frame. Note that for calls that
323 // were not in the original stack sample, we should have added the
324 // extra frame when processing the return paired with this call.
325 unwindCall(State);
326 } else if (isReturnState(State)) {
327 // Unwind returns - check whether the IP is indeed at a return
328 // instruction
329 unwindReturn(State);
330 } else if (isValidState(State)) {
331 // Unwind branches
332 unwindBranch(State);
333 } else {
334 // Skip unwinding the rest of LBR trace. Reset the stack and update the
335 // state so that the rest of the trace can still be processed as if they
336 // do not have stack samples.
337 State.clearCallStack();
338 State.InstPtr.update(Addr: State.getCurrentLBRSource());
339 State.pushFrame(Address: State.InstPtr.Address);
340 }
341
342 State.advanceLBR();
343 // Record `branch` with calling context after unwinding.
344 recordBranchCount(Branch, State, Repeat);
345 }
346 // As samples are aggregated on trie, record them into counter map
347 collectSamplesFromFrameTrie(Cur: State.getDummyRootPtr());
348
349 return true;
350}
351
352std::unique_ptr<PerfReaderBase>
353PerfReaderBase::create(ProfiledBinary *Binary, InputFile &Input,
354 std::optional<int32_t> PIDFilter) {
355 std::unique_ptr<PerfReaderBase> PerfReader;
356
357 if (Input.Format == InputFormat::UnsymbolizedProfile) {
358 PerfReader.reset(
359 p: new UnsymbolizedProfileReader(Binary, Input.InputFilePath));
360 return PerfReader;
361 }
362
363 // For perf data input, we need to convert them into perf script first.
364 // If this is a kernel perf file, there is no need for retrieving PIDs.
365 if (Input.Format == InputFormat::PerfData)
366 Input = PerfScriptReader::convertPerfDataToTrace(Binary, SkipPID: Binary->isKernel(),
367 File&: Input, PIDFilter);
368
369 assert((Input.Format == InputFormat::PerfScript) &&
370 "Should be a perfscript!");
371
372 Input.Content = PerfScriptReader::checkPerfScriptType(FileName: Input.InputFilePath);
373 if (Input.Content == PerfContent::LBRStack) {
374 PerfReader.reset(
375 p: new HybridPerfReader(Binary, Input.InputFilePath, PIDFilter));
376 } else if (Input.Content == PerfContent::LBR) {
377 PerfReader.reset(p: new LBRPerfReader(Binary, Input.InputFilePath, PIDFilter));
378 } else {
379 exitWithError(Message: "Unsupported perfscript!");
380 }
381
382 return PerfReader;
383}
384
385Error PerfReaderBase::parseDataAccessPerfTraces(
386 StringRef DataAccessPerfTraceFile, std::optional<int32_t> PIDFilter) {
387 // A perf_record_sample line is like
388 // . 1282514022939813 0x87b0 [0x60]: PERF_RECORD_SAMPLE(IP, 0x4002):
389 // 3446532/3446532: 0x2608a2 period: 233 addr: 0x3b3fb0
390 constexpr static StringRef DataAccessSamplePattern =
391 "PERF_RECORD_SAMPLE\\([A-Za-z]+, 0x[0-9a-fA-F]+\\): "
392 "([0-9]+)\\/[0-9]+: 0x([0-9a-fA-F]+) period: [0-9]+ addr: "
393 "0x([0-9a-fA-F]+)";
394
395 llvm::Regex LogRegex(DataAccessSamplePattern);
396
397 auto BufferOrErr = MemoryBuffer::getFile(Filename: DataAccessPerfTraceFile);
398 std::error_code EC = BufferOrErr.getError();
399 if (EC)
400 return make_error<StringError>(Args: "Failed to open perf trace file: " +
401 DataAccessPerfTraceFile,
402 Args: inconvertibleErrorCode());
403
404 assert(!SampleCounters.empty() && "Sample counters should not be empty!");
405 SampleCounter &Counter = SampleCounters.begin()->second;
406 line_iterator LineIt(*BufferOrErr.get(), true);
407
408 for (; !LineIt.is_at_eof(); ++LineIt) {
409 StringRef Line = *LineIt;
410
411 MMapEvent MMap;
412 if (Line.contains(Other: "PERF_RECORD_MMAP2")) {
413 if (PerfScriptReader::extractMMapEventForBinary(Binary, Line, MMap)) {
414 if (!MMap.MemProtectionFlag.contains(Other: "x")) {
415 if (Error E = Binary->addMMapNonTextEvent(Event: MMap)) {
416 return E;
417 }
418 }
419 }
420 continue;
421 }
422
423 SmallVector<StringRef> Fields;
424 if (LogRegex.match(String: Line, Matches: &Fields)) {
425 int32_t PID = 0;
426 if (Fields[1].getAsInteger(Radix: 10, Result&: PID))
427 return make_error<StringError>(
428 Args: "Failed to parse PID from perf trace line: " + Line,
429 Args: inconvertibleErrorCode());
430
431 if (PIDFilter.has_value() && *PIDFilter != PID) {
432 continue;
433 }
434
435 uint64_t DataAddress = 0;
436 if (Fields[3].getAsInteger(Radix: 16, Result&: DataAddress))
437 return make_error<StringError>(
438 Args: "Failed to parse data address from perf trace line: " + Line,
439 Args: inconvertibleErrorCode());
440 // Out of all the memory access events, the vtable accesses are used to
441 // construct type profiles. We assume that this is under the Itanium
442 // C++ ABI so we can use `_ZTV` prefix to identify vtable.
443 StringRef DataSymbol = Binary->symbolizeDataAddress(
444 Address: Binary->CanonicalizeNonTextAddress(Address: DataAddress));
445 if (DataSymbol.starts_with(Prefix: "_ZTV")) {
446 uint64_t IP = 0;
447 Fields[2].getAsInteger(Radix: 16, Result&: IP);
448 Counter.recordDataAccessCount(InstAddr: Binary->canonicalizeVirtualAddress(Address: IP),
449 DataSymbol, Repeat: 1);
450 }
451 }
452 }
453 return Error::success();
454}
455
456InputFile
457PerfScriptReader::convertPerfDataToTrace(ProfiledBinary *Binary, bool SkipPID,
458 InputFile &File,
459 std::optional<int32_t> PIDFilter) {
460 StringRef PerfData = File.InputFilePath;
461 // Run perf script to retrieve PIDs matching binary we're interested in.
462 auto PerfExecutable = sys::Process::FindInEnvPath(EnvName: "PATH", FileName: "perf");
463 if (!PerfExecutable) {
464 exitWithError(Message: "Perf not found.");
465 }
466 std::string PerfExecutablePath = *PerfExecutable;
467 SmallString<128> PerfTraceFile;
468 sys::fs::createUniquePath(Model: "perf-script-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%.tmp",
469 ResultPath&: PerfTraceFile, /*MakeAbsolute=*/true);
470 std::string ErrorFile = std::string(PerfTraceFile) + ".err";
471 std::optional<StringRef> Redirects[] = {std::nullopt, // Stdin
472 StringRef(PerfTraceFile), // Stdout
473 StringRef(ErrorFile)}; // Stderr
474 PerfScriptReader::TempFileCleanups.emplace_back(Args&: PerfTraceFile);
475 PerfScriptReader::TempFileCleanups.emplace_back(Args&: ErrorFile);
476
477 auto RunPerfScript = [&](ArrayRef<StringRef> Args) {
478 // ExecuteAndWait does not truncate redirected output files on Unix. Remove
479 // both files so a shorter invocation cannot retain output from the
480 // previous perf script invocation.
481 for (StringRef Path : {StringRef(PerfTraceFile), StringRef(ErrorFile)}) {
482 if (std::error_code EC = sys::fs::remove(path: Path))
483 exitWithError(EC, Whence: Path);
484 }
485
486 std::string ExecutionError;
487 bool ExecutionFailed = false;
488 int ExitCode =
489 sys::ExecuteAndWait(Program: PerfExecutablePath, Args, Env: std::nullopt, Redirects,
490 /*SecondsToWait=*/0, /*MemoryLimit=*/0,
491 ErrMsg: &ExecutionError, ExecutionFailed: &ExecutionFailed);
492 if (!ExecutionFailed && ExitCode == 0)
493 return;
494
495 std::string Message;
496 raw_string_ostream OS(Message);
497 if (ExecutionFailed || ExitCode == -1)
498 OS << "Failed to execute perf script";
499 else if (ExitCode == -2)
500 OS << "Perf script terminated abnormally";
501 else
502 OS << "Perf script failed with exit code " << ExitCode;
503 if (!ExecutionError.empty())
504 OS << ": " << ExecutionError;
505
506 if (auto ErrorBuffer = MemoryBuffer::getFile(Filename: ErrorFile)) {
507 StringRef Stderr = ErrorBuffer.get()->getBuffer().trim();
508 if (!Stderr.empty())
509 OS << "\n" << Stderr;
510 }
511 exitWithError(Message: OS.str());
512 };
513
514 std::string PIDs;
515 if (!SkipPID) {
516 StringRef ScriptMMapArgs[] = {PerfExecutablePath,
517 "script",
518 "--show-mmap-events",
519 "-F",
520 "comm,pid",
521 "-i",
522 PerfData};
523 RunPerfScript(ScriptMMapArgs);
524
525 // Collect the PIDs
526 TraceStream TraceIt(PerfTraceFile);
527 DenseSet<int32_t> PIDSet;
528 while (!TraceIt.isAtEoF()) {
529 MMapEvent MMap;
530 if (isMMapEvent(Line: TraceIt.getCurrentLine()) &&
531 extractMMapEventForBinary(Binary, Line: TraceIt.getCurrentLine(), MMap)) {
532 auto It = PIDSet.insert(V: MMap.PID);
533 if (It.second && (!PIDFilter || MMap.PID == *PIDFilter)) {
534 if (!PIDs.empty()) {
535 PIDs.append(s: ",");
536 }
537 PIDs.append(str: utostr(X: MMap.PID));
538 }
539 }
540 TraceIt.advance();
541 }
542
543 if (PIDs.empty()) {
544 exitWithError(Message: "No relevant mmap event is found in perf data.");
545 }
546 }
547
548 // Run perf script again to retrieve events for PIDs collected above
549 SmallVector<StringRef, 8> ScriptSampleArgs;
550 ScriptSampleArgs.push_back(Elt: PerfExecutablePath);
551 ScriptSampleArgs.push_back(Elt: "script");
552 ScriptSampleArgs.push_back(Elt: "--show-mmap-events");
553 ScriptSampleArgs.push_back(Elt: "-F");
554 ScriptSampleArgs.push_back(Elt: "ip,brstack");
555 ScriptSampleArgs.push_back(Elt: "-i");
556 ScriptSampleArgs.push_back(Elt: PerfData);
557 if (!PIDs.empty()) {
558 ScriptSampleArgs.push_back(Elt: "--pid");
559 ScriptSampleArgs.push_back(Elt: PIDs);
560 }
561 RunPerfScript(ScriptSampleArgs);
562
563 return {.InputFilePath: std::string(PerfTraceFile), .Format: InputFormat::PerfScript,
564 .Content: PerfContent::UnknownContent};
565}
566
567static StringRef filename(StringRef Path, bool UseBackSlash) {
568 llvm::sys::path::Style PathStyle =
569 UseBackSlash ? llvm::sys::path::Style::windows_backslash
570 : llvm::sys::path::Style::native;
571 StringRef FileName = llvm::sys::path::filename(path: Path, style: PathStyle);
572
573 // In case this file use \r\n as newline.
574 if (UseBackSlash && FileName.back() == '\r')
575 return FileName.drop_back();
576
577 return FileName;
578}
579
580void PerfScriptReader::updateBinaryAddress(const MMapEvent &Event) {
581 // Drop the event which doesn't belong to user-provided binary
582 StringRef BinaryName = filename(Path: Event.BinaryPath, UseBackSlash: Binary->isCOFF());
583 bool IsKernel = Binary->isKernel();
584 if (!IsKernel && Binary->getName() != BinaryName)
585 return;
586 if (IsKernel && !Binary->isKernelImageName(BinaryName))
587 return;
588
589 // Drop the event if process does not match pid filter
590 if (PIDFilter && Event.PID != *PIDFilter)
591 return;
592
593 Binary->addMMapRange(Address: Event.Address, Size: Event.Size);
594
595 // Check if the FileOffset falls within the [Event.Offset, Event.Offset +
596 // Event.Size) range.
597 auto MMapContainsFileOffset = [&](uint64_t FileOffset) {
598 return Event.Offset <= FileOffset &&
599 (FileOffset - Event.Offset) < Event.Size;
600 };
601
602 if (IsKernel || MMapContainsFileOffset(Binary->getTextSegmentOffset())) {
603 // For ELF, subtract the file offset to get the runtime address
604 // corresponding to file offset zero. Kernel mmap events report the text
605 // address as Event.Offset, so use the text segment offset from the ELF
606 // instead.
607 const uint64_t RuntimeBaseAddress =
608 Binary->isCOFF()
609 ? Event.Address
610 : Event.Address -
611 (IsKernel ? Binary->getTextSegmentOffset() : Event.Offset);
612 // A binary image could be unloaded and then reloaded at different
613 // place, so update binary load address.
614 // Only update for the first executable segment and assume all other
615 // segments are loaded at consecutive memory addresses, which is the case on
616 // X64.
617 Binary->setBaseAddress(RuntimeBaseAddress);
618 Binary->setIsLoadedByMMap(true);
619 } else {
620 // Verify segments are loaded consecutively.
621 const auto &Offsets = Binary->getTextSegmentOffsets();
622 auto IsContiguousMMapForSegment = [&](auto SegmentIt, uint64_t FileOffset,
623 uint64_t RuntimeAddress) {
624 auto I = std::distance(Offsets.begin(), SegmentIt);
625 const auto &PreferredAddrs = Binary->getPreferredTextSegmentAddresses();
626 return PreferredAddrs[I] + (FileOffset - *SegmentIt) ==
627 Binary->canonicalizeVirtualAddress(Address: RuntimeAddress);
628 };
629
630 auto It = llvm::lower_bound(Range: Offsets, Value: Event.Offset);
631 if (It != Offsets.end() && MMapContainsFileOffset(*It)) {
632 // The event is for loading a separate executable segment.
633 uint64_t RuntimeSegmentAddress = Event.Address + (*It - Event.Offset);
634 if (!IsContiguousMMapForSegment(It, *It, RuntimeSegmentAddress))
635 exitWithError(Message: "Executable segments not loaded consecutively");
636 } else {
637 if (It == Offsets.begin())
638 exitWithError(Message: "File offset not found");
639 else {
640 // Find the segment the event falls in. A large segment could be loaded
641 // via multiple mmap calls with consecutive memory addresses.
642 --It;
643 assert(*It < Event.Offset);
644 if (!IsContiguousMMapForSegment(It, Event.Offset, Event.Address))
645 exitWithError(Message: "Segment not loaded by consecutive mmaps");
646 }
647 }
648 }
649}
650
651static std::string getContextKeyStr(ContextKey *K,
652 const ProfiledBinary *Binary) {
653 if (const auto *CtxKey = dyn_cast<StringBasedCtxKey>(Val: K)) {
654 return SampleContext::getContextString(Context: CtxKey->Context);
655 } else if (const auto *CtxKey = dyn_cast<AddrBasedCtxKey>(Val: K)) {
656 std::ostringstream OContextStr;
657 for (uint32_t I = 0; I < CtxKey->Context.size(); I++) {
658 if (OContextStr.str().size())
659 OContextStr << " @ ";
660 uint64_t Address = CtxKey->Context[I];
661 if (UseOffset) {
662 if (UseLoadableSegmentAsBase)
663 Address -= Binary->getFirstLoadableAddress();
664 else
665 Address -= Binary->getPreferredBaseAddress();
666 }
667 OContextStr << "0x"
668 << utohexstr(X: Address,
669 /*LowerCase=*/true);
670 }
671 return OContextStr.str();
672 } else {
673 llvm_unreachable("unexpected key type");
674 }
675}
676
677void HybridPerfReader::unwindSamples() {
678 NamedRegionTimer T("unwind", "Unwind samples", TimerGroupName, TimerGroupDesc,
679 TimeProfGen);
680 VirtualUnwinder Unwinder(&SampleCounters, Binary);
681 for (const auto &Item : AggregatedSamples) {
682 const PerfSample *Sample = Item.first.getPtr();
683 Unwinder.unwind(Sample, Repeat: Item.second);
684 }
685
686 // Warn about untracked frames due to missing probes.
687 if (ShowDetailedWarning) {
688 for (auto Address : Unwinder.getUntrackedCallsites())
689 WithColor::warning() << "Profile context truncated due to missing probe "
690 << "for call instruction at "
691 << format(Fmt: "0x%" PRIx64, Vals: Address) << "\n";
692 }
693
694 emitWarningSummary(Num: Unwinder.getUntrackedCallsites().size(),
695 Total: SampleCounters.size(),
696 Msg: "of profiled contexts are truncated due to missing probe "
697 "for call instruction.");
698
699 emitWarningSummary(
700 Num: Unwinder.NumMismatchedExtCallBranch, Total: Unwinder.NumTotalBranches,
701 Msg: "of branches'source is a call instruction but doesn't match call frame "
702 "stack, likely due to unwinding error of external frame.");
703
704 emitWarningSummary(Num: Unwinder.NumPairedExtAddr * 2, Total: Unwinder.NumTotalBranches,
705 Msg: "of branches containing paired external address.");
706
707 emitWarningSummary(Num: Unwinder.NumUnpairedExtAddr, Total: Unwinder.NumTotalBranches,
708 Msg: "of branches containing external address but doesn't have "
709 "another external address to pair, likely due to "
710 "interrupt jmp or broken perf script.");
711
712 emitWarningSummary(
713 Num: Unwinder.NumMismatchedProEpiBranch, Total: Unwinder.NumTotalBranches,
714 Msg: "of branches'source is a call instruction but doesn't match call frame "
715 "stack, likely due to frame in prolog/epilog.");
716
717 emitWarningSummary(Num: Unwinder.NumMissingExternalFrame,
718 Total: Unwinder.NumExtCallBranch,
719 Msg: "of artificial call branches but doesn't have an external "
720 "frame to match.");
721
722 emitWarningSummary(Num: NumBogusTrace, Total: NumTotalHybridSample,
723 Msg: "of hybrid samples had a callchain leaf that disagreed "
724 "with the newest LBR target (bogus trace).");
725 if (NumBogusTrace * 100 > NumTotalHybridSample)
726 WithColor::warning() << "Bogus trace rate exceeds 1%: the profile has high "
727 "sample skid and may not be suitable for "
728 "optimization.\n";
729}
730
731/// Parse a hex address from \p Str.
732static bool parseAddress(StringRef Str, uint64_t &Addr, bool HasPrefix) {
733 Str = Str.take_while(F: [](char C) { return !isspace(C); });
734 if (Str.consume_front(Prefix: "0x") != HasPrefix)
735 return true;
736 return Str.getAsInteger(Radix: 16, Result&: Addr);
737}
738
739bool PerfScriptReader::extractLBRStack(TraceStream &TraceIt,
740 SmallVectorImpl<LBREntry> &LBRStack) {
741 // The raw format of LBR stack is like:
742 // 0x4005c8/0x4005dc/P/-/-/0 0x40062f/0x4005b0/P/-/-/0 ...
743 // ... 0x4005c8/0x4005dc/P/-/-/0
744 // It's in FIFO order and separated by whitespace.
745 SmallVector<StringRef, 32> Records;
746 TraceIt.getCurrentLine().rtrim().split(A&: Records, Separator: " ", MaxSplit: -1, KeepEmpty: false);
747 auto WarnInvalidLBR = [](TraceStream &TraceIt) {
748 WithColor::warning() << "Invalid address in LBR record at line "
749 << TraceIt.getLineNumber() << ": "
750 << TraceIt.getCurrentLine() << "\n";
751 };
752
753 // Skip the leading instruction pointer.
754 size_t Index = 0;
755 uint64_t LeadingAddr;
756 if (!Records.empty() && !Records[0].contains(C: '/')) {
757 if (parseAddress(Str: Records[0], Addr&: LeadingAddr, HasPrefix: false)) {
758 WarnInvalidLBR(TraceIt);
759 TraceIt.advance();
760 return false;
761 }
762 Index = 1;
763 }
764
765 // Now extract LBR samples - note that we do not reverse the
766 // LBR entry order so we can unwind the sample stack as we walk
767 // through LBR entries.
768 while (Index < Records.size()) {
769 auto &Token = Records[Index++];
770 if (Token.size() == 0)
771 continue;
772
773 SmallVector<StringRef, 8> Addresses;
774 Token.split(A&: Addresses, Separator: "/");
775 uint64_t Src;
776 uint64_t Dst;
777
778 // Stop at broken LBR records.
779 if (Addresses.size() < 2 || parseAddress(Str: Addresses[0], Addr&: Src, HasPrefix: true) ||
780 parseAddress(Str: Addresses[1], Addr&: Dst, HasPrefix: true)) {
781 WarnInvalidLBR(TraceIt);
782 break;
783 }
784
785 // Canonicalize to use preferred load address as base address.
786 Src = Binary->canonicalizeVirtualAddress(Address: Src);
787 Dst = Binary->canonicalizeVirtualAddress(Address: Dst);
788 bool SrcIsInternal = Binary->addressIsCode(Address: Src);
789 bool DstIsInternal = Binary->addressIsCode(Address: Dst);
790 if (!SrcIsInternal)
791 Src = ExternalAddr;
792 if (!DstIsInternal)
793 Dst = ExternalAddr;
794 // Filter external-to-external case to reduce LBR trace size.
795 if (!SrcIsInternal && !DstIsInternal)
796 continue;
797
798 LBRStack.emplace_back(Args: LBREntry(Src, Dst));
799 }
800 TraceIt.advance();
801 return !LBRStack.empty();
802}
803
804bool PerfScriptReader::extractCallstack(TraceStream &TraceIt,
805 SmallVectorImpl<uint64_t> &CallStack) {
806 // The raw format of call stack is like:
807 // 4005dc # leaf frame
808 // 400634
809 // 400684 # root frame
810 // It's in bottom-up order with each frame in one line.
811
812 // Extract stack frames from sample
813 while (!TraceIt.isAtEoF() && !isLBRSample(Line: TraceIt.getCurrentLine(), CheckLineStart: true)) {
814 StringRef FrameStr = TraceIt.getCurrentLine().ltrim();
815 uint64_t FrameAddr = 0;
816 if (parseAddress(Str: FrameStr, Addr&: FrameAddr, HasPrefix: false)) {
817 // We might parse a non-perf sample line like empty line and comments,
818 // skip it
819 TraceIt.advance();
820 return false;
821 }
822 TraceIt.advance();
823
824 FrameAddr = Binary->canonicalizeVirtualAddress(Address: FrameAddr);
825 // Currently intermixed frame from different binaries is not supported.
826 if (!Binary->addressIsCode(Address: FrameAddr)) {
827 if (CallStack.empty())
828 NumLeafExternalFrame++;
829 // Push a special value(ExternalAddr) for the external frames so that
830 // unwinder can still work on this with artificial Call/Return branch.
831 // After unwinding, the context will be truncated for external frame.
832 // Also deduplicate the consecutive external addresses.
833 if (CallStack.empty() || CallStack.back() != ExternalAddr)
834 CallStack.emplace_back(Args: ExternalAddr);
835 continue;
836 }
837
838 // We need to translate return address to call address for non-leaf frames.
839 if (!CallStack.empty()) {
840 auto CallAddr = Binary->getCallAddrFromFrameAddr(FrameAddr);
841 if (!CallAddr) {
842 // Stop at an invalid return address caused by bad unwinding. This could
843 // happen to frame-pointer-based unwinding and the callee functions that
844 // do not have the frame pointer chain set up.
845 InvalidReturnAddresses.insert(x: FrameAddr);
846 break;
847 }
848 FrameAddr = CallAddr;
849 }
850
851 CallStack.emplace_back(Args&: FrameAddr);
852 }
853
854 // Strip out the bottom external addr.
855 if (CallStack.size() > 1 && CallStack.back() == ExternalAddr)
856 CallStack.pop_back();
857
858 // Skip other unrelated line, find the next valid LBR line
859 // Note that even for empty call stack, we should skip the address at the
860 // bottom, otherwise the following pass may generate a truncated callstack
861 while (!TraceIt.isAtEoF() && !isLBRSample(Line: TraceIt.getCurrentLine(), CheckLineStart: true)) {
862 TraceIt.advance();
863 }
864 // Filter out broken stack sample. We may not have complete frame info
865 // if sample end up in prolog/epilog, the result is dangling context not
866 // connected to entry point. This should be relatively rare thus not much
867 // impact on overall profile quality. However we do want to filter them
868 // out to reduce the number of different calling contexts. One instance
869 // of such case - when sample landed in prolog/epilog, somehow stack
870 // walking will be broken in an unexpected way that higher frames will be
871 // missing.
872 return !CallStack.empty() &&
873 !Binary->addressInPrologEpilog(Address: CallStack.front());
874}
875
876void PerfScriptReader::warnIfMissingMMap() {
877 if (!Binary->getMissingMMapWarned() && !Binary->getIsLoadedByMMap()) {
878 WithColor::warning() << "No relevant mmap event is matched for "
879 << Binary->getName()
880 << ", will use preferred address ("
881 << format(Fmt: "0x%" PRIx64,
882 Vals: Binary->getPreferredBaseAddress())
883 << ") as the base loading address!\n";
884 // Avoid redundant warning, only warn at the first unmatched sample.
885 Binary->setMissingMMapWarned(true);
886 }
887}
888
889// The unwinder requires that LBR tip belong to the leaf frame.
890// External addresses are not checked.
891static bool isValidTrace(ProfiledBinary *Binary, uint64_t StackLeaf,
892 uint64_t LBRLeaf) {
893 if (StackLeaf == ExternalAddr || LBRLeaf == ExternalAddr)
894 return true;
895 return Binary->findFuncRange(Address: LBRLeaf) == Binary->findFuncRange(Address: StackLeaf);
896}
897
898void HybridPerfReader::parseSample(TraceStream &TraceIt, uint64_t Count) {
899 // The raw hybird sample started with call stack in FILO order and followed
900 // intermediately by LBR sample
901 // e.g.
902 // 4005dc # call stack leaf
903 // 400634
904 // 400684 # call stack root
905 // 0x4005c8/0x4005dc/P/-/-/0 0x40062f/0x4005b0/P/-/-/0 ...
906 // ... 0x4005c8/0x4005dc/P/-/-/0 # LBR Entries
907 //
908 std::shared_ptr<PerfSample> Sample = std::make_shared<PerfSample>();
909#ifndef NDEBUG
910 Sample->Linenum = TraceIt.getLineNumber();
911#endif
912 // Parsing call stack and populate into PerfSample.CallStack
913 if (!extractCallstack(TraceIt, CallStack&: Sample->CallStack)) {
914 // Skip the next LBR line matched current call stack
915 if (!TraceIt.isAtEoF() && isLBRSample(Line: TraceIt.getCurrentLine(), CheckLineStart: true))
916 TraceIt.advance();
917 return;
918 }
919
920 warnIfMissingMMap();
921
922 if (!TraceIt.isAtEoF() && isLBRSample(Line: TraceIt.getCurrentLine(), CheckLineStart: true)) {
923 // Parsing LBR stack and populate into PerfSample.LBRStack
924 if (extractLBRStack(TraceIt, LBRStack&: Sample->LBRStack)) {
925 if (IgnoreStackSamples) {
926 Sample->CallStack.clear();
927 } else {
928 NumTotalHybridSample++;
929 // Drop samples whose callchain and LBR disagree before the
930 // canonicalization below hides the disagreement.
931 uint64_t StackLeaf = Sample->CallStack.front();
932 uint64_t LBRLeaf = Sample->LBRStack[0].Target;
933 if (!isValidTrace(Binary, StackLeaf, LBRLeaf)) {
934 NumBogusTrace++;
935 if (ShowDetailedWarning)
936 WithColor::warning()
937 << "Bogus trace: stack tip = " << format_hex(N: StackLeaf, Width: 10)
938 << ", LBR tip = " << format_hex(N: LBRLeaf, Width: 10) << "\n";
939 return;
940 }
941 // Canonicalize stack leaf to avoid 'random' IP from leaf frame skew LBR
942 // ranges
943 Sample->CallStack.front() = Sample->LBRStack[0].Target;
944 }
945 // Record samples by aggregation
946 AggregatedSamples[Hashable<PerfSample>(Sample)] += Count;
947 }
948 } else {
949 // LBR sample is encoded in single line after stack sample
950 exitWithError(Message: "'Hybrid perf sample is corrupted, No LBR sample line");
951 }
952}
953
954void PerfScriptReader::writeUnsymbolizedProfile(StringRef Filename) {
955 std::error_code EC;
956 raw_fd_ostream OS(Filename, EC, llvm::sys::fs::OF_TextWithCRLF);
957 if (EC)
958 exitWithError(EC, Whence: Filename);
959 writeUnsymbolizedProfile(OS);
960}
961
962// Use ordered map to make the output deterministic
963using OrderedCounterForPrint = std::map<std::string, SampleCounter *>;
964
965void PerfScriptReader::writeUnsymbolizedProfile(raw_fd_ostream &OS) {
966 OrderedCounterForPrint OrderedCounters;
967 for (auto &CI : SampleCounters) {
968 OrderedCounters[getContextKeyStr(K: CI.first.getPtr(), Binary)] = &CI.second;
969 }
970
971 auto SCounterPrinter = [&](RangeSample &Counter, StringRef Separator,
972 uint32_t Indent) {
973 OS.indent(NumSpaces: Indent);
974 OS << Counter.size() << "\n";
975 for (auto &I : Counter) {
976 uint64_t Start = I.first.first;
977 uint64_t End = I.first.second;
978
979 if (UseOffset) {
980 if (UseLoadableSegmentAsBase) {
981 Start -= Binary->getFirstLoadableAddress();
982 End -= Binary->getFirstLoadableAddress();
983 } else {
984 Start -= Binary->getPreferredBaseAddress();
985 End -= Binary->getPreferredBaseAddress();
986 }
987 }
988
989 OS.indent(NumSpaces: Indent);
990 OS << Twine::utohexstr(Val: Start) << Separator << Twine::utohexstr(Val: End) << ":"
991 << I.second << "\n";
992 }
993 };
994
995 for (auto &CI : OrderedCounters) {
996 uint32_t Indent = 0;
997 if (ProfileIsCS) {
998 // Context string key
999 OS << "[" << CI.first << "]\n";
1000 Indent = 2;
1001 }
1002
1003 SampleCounter &Counter = *CI.second;
1004 SCounterPrinter(Counter.RangeCounter, "-", Indent);
1005 SCounterPrinter(Counter.BranchCounter, "->", Indent);
1006 }
1007}
1008
1009// Format of input:
1010// number of entries in RangeCounter
1011// from_1-to_1:count_1
1012// from_2-to_2:count_2
1013// ......
1014// from_n-to_n:count_n
1015// number of entries in BranchCounter
1016// src_1->dst_1:count_1
1017// src_2->dst_2:count_2
1018// ......
1019// src_n->dst_n:count_n
1020void UnsymbolizedProfileReader::readSampleCounters(TraceStream &TraceIt,
1021 SampleCounter &SCounters) {
1022 auto exitWithErrorForTraceLine = [](TraceStream &TraceIt) {
1023 std::string Msg = TraceIt.isAtEoF()
1024 ? "Invalid raw profile!"
1025 : "Invalid raw profile at line " +
1026 Twine(TraceIt.getLineNumber()).str() + ": " +
1027 TraceIt.getCurrentLine().str();
1028 exitWithError(Message: Msg);
1029 };
1030 auto ReadNumber = [&](uint64_t &Num) {
1031 if (TraceIt.isAtEoF())
1032 exitWithErrorForTraceLine(TraceIt);
1033 if (TraceIt.getCurrentLine().ltrim().getAsInteger(Radix: 10, Result&: Num))
1034 exitWithErrorForTraceLine(TraceIt);
1035 TraceIt.advance();
1036 };
1037
1038 auto ReadCounter = [&](RangeSample &Counter, StringRef Separator) {
1039 uint64_t Num = 0;
1040 ReadNumber(Num);
1041 while (Num--) {
1042 if (TraceIt.isAtEoF())
1043 exitWithErrorForTraceLine(TraceIt);
1044 StringRef Line = TraceIt.getCurrentLine().ltrim();
1045
1046 uint64_t Count = 0;
1047 auto LineSplit = Line.split(Separator: ":");
1048 if (LineSplit.second.empty() || LineSplit.second.getAsInteger(Radix: 10, Result&: Count))
1049 exitWithErrorForTraceLine(TraceIt);
1050
1051 uint64_t Source = 0;
1052 uint64_t Target = 0;
1053 auto Range = LineSplit.first.split(Separator);
1054 if (Range.second.empty() || Range.first.getAsInteger(Radix: 16, Result&: Source) ||
1055 Range.second.getAsInteger(Radix: 16, Result&: Target))
1056 exitWithErrorForTraceLine(TraceIt);
1057
1058 if (UseOffset) {
1059 if (UseLoadableSegmentAsBase) {
1060 Source += Binary->getFirstLoadableAddress();
1061 Target += Binary->getFirstLoadableAddress();
1062 } else {
1063 Source += Binary->getPreferredBaseAddress();
1064 Target += Binary->getPreferredBaseAddress();
1065 }
1066 }
1067
1068 Counter[{Source, Target}] += Count;
1069 TraceIt.advance();
1070 }
1071 };
1072
1073 ReadCounter(SCounters.RangeCounter, "-");
1074 ReadCounter(SCounters.BranchCounter, "->");
1075}
1076
1077void UnsymbolizedProfileReader::readUnsymbolizedProfile(StringRef FileName) {
1078 TraceStream TraceIt(FileName);
1079 while (!TraceIt.isAtEoF()) {
1080 std::shared_ptr<StringBasedCtxKey> Key =
1081 std::make_shared<StringBasedCtxKey>();
1082 StringRef Line = TraceIt.getCurrentLine();
1083 // Read context stack for CS profile.
1084 if (Line.starts_with(Prefix: "[")) {
1085 ProfileIsCS = true;
1086 auto I = ContextStrSet.insert(key: Line);
1087 SampleContext::createCtxVectorFromStr(ContextStr: I.first->getKey(), Context&: Key->Context);
1088 TraceIt.advance();
1089 }
1090 auto Ret = SampleCounters.try_emplace(Key: Hashable<ContextKey>(Key));
1091 readSampleCounters(TraceIt, SCounters&: Ret.first->second);
1092 }
1093}
1094
1095void UnsymbolizedProfileReader::parsePerfTraces() {
1096 readUnsymbolizedProfile(FileName: PerfTraceFile);
1097}
1098
1099void PerfScriptReader::computeCounterFromLBR(const PerfSample *Sample,
1100 uint64_t Repeat) {
1101 SampleCounter &Counter = SampleCounters.begin()->second;
1102 uint64_t EndAddress = 0;
1103 for (const LBREntry &LBR : Sample->LBRStack) {
1104 uint64_t SourceAddress = LBR.Source;
1105 uint64_t TargetAddress = LBR.Target;
1106
1107 // Record the branch if its SourceAddress is external. It can be the case an
1108 // external source call an internal function, later this branch will be used
1109 // to generate the function's head sample.
1110 if (Binary->addressIsCode(Address: TargetAddress)) {
1111 Counter.recordBranchCount(Source: SourceAddress, Target: TargetAddress, Repeat);
1112 }
1113
1114 // If this not the first LBR, update the range count between TO of current
1115 // LBR and FROM of next LBR.
1116 uint64_t StartAddress = TargetAddress;
1117 if (Binary->addressIsCode(Address: StartAddress) &&
1118 Binary->addressIsCode(Address: EndAddress) &&
1119 isValidFallThroughRange(Start: StartAddress, End: EndAddress, Binary))
1120 Counter.recordRangeCount(Start: StartAddress, End: EndAddress, Repeat);
1121 EndAddress = SourceAddress;
1122 }
1123}
1124
1125void LBRPerfReader::parseSample(TraceStream &TraceIt, uint64_t Count) {
1126 std::shared_ptr<PerfSample> Sample = std::make_shared<PerfSample>();
1127 // Parsing LBR stack and populate into PerfSample.LBRStack
1128 if (extractLBRStack(TraceIt, LBRStack&: Sample->LBRStack)) {
1129 warnIfMissingMMap();
1130 // Record LBR only samples by aggregation
1131 AggregatedSamples[Hashable<PerfSample>(Sample)] += Count;
1132 }
1133}
1134
1135void PerfScriptReader::generateUnsymbolizedProfile() {
1136 // There is no context for LBR only sample, so initialize one entry with
1137 // fake "empty" context key.
1138 assert(SampleCounters.empty() &&
1139 "Sample counter map should be empty before raw profile generation");
1140 std::shared_ptr<StringBasedCtxKey> Key =
1141 std::make_shared<StringBasedCtxKey>();
1142 SampleCounters.try_emplace(Key: Hashable<ContextKey>(Key));
1143 for (const auto &Item : AggregatedSamples) {
1144 const PerfSample *Sample = Item.first.getPtr();
1145 computeCounterFromLBR(Sample, Repeat: Item.second);
1146 }
1147}
1148
1149uint64_t PerfScriptReader::parseAggregatedCount(TraceStream &TraceIt) {
1150 // The aggregated count is optional, so do not skip the line and return 1 if
1151 // it's unmatched
1152 uint64_t Count = 1;
1153 if (!TraceIt.getCurrentLine().getAsInteger(Radix: 10, Result&: Count))
1154 TraceIt.advance();
1155 return Count;
1156}
1157
1158void PerfScriptReader::parseSample(TraceStream &TraceIt) {
1159 NumTotalSample++;
1160 uint64_t Count = parseAggregatedCount(TraceIt);
1161 assert(Count >= 1 && "Aggregated count should be >= 1!");
1162 parseSample(TraceIt, Count);
1163}
1164
1165bool PerfScriptReader::extractMMapEventForBinary(ProfiledBinary *Binary,
1166 StringRef Line,
1167 MMapEvent &MMap) {
1168 if (!Binary->isKernel() && !Line.contains(Other: Binary->getName()) &&
1169 !ShowMmapEvents)
1170 return false;
1171 // Parse a MMap2 line like:
1172 // PERF_RECORD_MMAP2 2113428/2113428: [0x7fd4efb57000(0x204000) @ 0
1173 // 08:04 19532229 3585508847]: r-xp /usr/lib64/libdl-2.17.so
1174 constexpr static const char *const MMap2Pattern =
1175 "PERF_RECORD_MMAP2 (-?[0-9]+)/[0-9]+: "
1176 "\\[(0x[a-f0-9]+)\\((0x[a-f0-9]+)\\) @ "
1177 "(0x[a-f0-9]+|0) .*\\]: ([-a-z]+) (.*)";
1178 // Parse a MMap line like
1179 // PERF_RECORD_MMAP -1/0: [0xffffffff81e00000(0x3e8fa000) @ \
1180 // 0xffffffff81e00000]: x [kernel.kallsyms]_text
1181 constexpr static const char *const MMapPattern =
1182 "PERF_RECORD_MMAP (-?[0-9]+)/[0-9]+: "
1183 "\\[(0x[a-f0-9]+)\\((0x[a-f0-9]+)\\) @ "
1184 "(0x[a-f0-9]+|0)\\]: ([-a-z]+) (.*)";
1185 // Field 0 - whole line
1186 // Field 1 - PID
1187 // Field 2 - base address
1188 // Field 3 - mmapped size
1189 // Field 4 - page offset
1190 // Field 5 - binary path
1191 enum EventIndex {
1192 WHOLE_LINE = 0,
1193 PID = 1,
1194 MMAPPED_ADDRESS = 2,
1195 MMAPPED_SIZE = 3,
1196 PAGE_OFFSET = 4,
1197 MEM_PROTECTION_FLAG = 5,
1198 BINARY_PATH = 6,
1199 };
1200
1201 bool R = false;
1202 SmallVector<StringRef, 7> Fields;
1203 if (Line.contains(Other: "PERF_RECORD_MMAP2 ")) {
1204 Regex RegMmap2(MMap2Pattern);
1205 R = RegMmap2.match(String: Line, Matches: &Fields);
1206 } else if (Line.contains(Other: "PERF_RECORD_MMAP ")) {
1207 Regex RegMmap(MMapPattern);
1208 R = RegMmap.match(String: Line, Matches: &Fields);
1209 } else
1210 llvm_unreachable("unexpected MMAP event entry");
1211
1212 if (!R) {
1213 std::string WarningMsg = "Cannot parse mmap event: " + Line.str() + " \n";
1214 WithColor::warning() << WarningMsg;
1215 return false;
1216 }
1217 long long MMapPID = 0;
1218 getAsSignedInteger(Str: Fields[PID], Radix: 10, Result&: MMapPID);
1219 MMap.PID = MMapPID;
1220 Fields[MMAPPED_ADDRESS].getAsInteger(Radix: 0, Result&: MMap.Address);
1221 Fields[MMAPPED_SIZE].getAsInteger(Radix: 0, Result&: MMap.Size);
1222 Fields[PAGE_OFFSET].getAsInteger(Radix: 0, Result&: MMap.Offset);
1223 MMap.MemProtectionFlag = Fields[MEM_PROTECTION_FLAG];
1224 MMap.BinaryPath = Fields[BINARY_PATH];
1225 if (ShowMmapEvents) {
1226 outs() << "Mmap: Binary " << MMap.BinaryPath << " loaded at "
1227 << format(Fmt: "0x%" PRIx64 ":", Vals: MMap.Address) << " \n";
1228 }
1229
1230 StringRef BinaryName = filename(Path: MMap.BinaryPath, UseBackSlash: Binary->isCOFF());
1231 if (Binary->isKernel()) {
1232 return Binary->isKernelImageName(BinaryName);
1233 }
1234 return Binary->getName() == BinaryName;
1235}
1236
1237void PerfScriptReader::parseMMapEvent(TraceStream &TraceIt) {
1238 MMapEvent MMap;
1239 if (extractMMapEventForBinary(Binary, Line: TraceIt.getCurrentLine(), MMap))
1240 updateBinaryAddress(Event: MMap);
1241 TraceIt.advance();
1242}
1243
1244void PerfScriptReader::parseEventOrSample(TraceStream &TraceIt) {
1245 if (isMMapEvent(Line: TraceIt.getCurrentLine()))
1246 parseMMapEvent(TraceIt);
1247 else
1248 parseSample(TraceIt);
1249}
1250
1251void PerfScriptReader::parseAndAggregateTrace() {
1252 NamedRegionTimer T("parseTrace", "Parse and aggregate trace", TimerGroupName,
1253 TimerGroupDesc, TimeProfGen);
1254 // Trace line iterator
1255 TraceStream TraceIt(PerfTraceFile);
1256 while (!TraceIt.isAtEoF())
1257 parseEventOrSample(TraceIt);
1258}
1259
1260// A LBR sample is like:
1261// 40062f 0x5c6313f/0x5c63170/P/-/-/0 0x5c630e7/0x5c63130/P/-/-/0 ...
1262// A heuristic for fast detection by checking whether a
1263// leading " 0x" and the '/' exist.
1264bool PerfScriptReader::isLBRSample(StringRef Line, bool CheckLineStart) {
1265 // Skip the leading instruction pointer
1266 SmallVector<StringRef, 32> Records;
1267 if (!CheckLineStart)
1268 Line = Line.trim();
1269 // Line might start with IP or only contain brstack. Check first two records
1270 // and fail if no record exists.
1271 Line.split(A&: Records, Separator: " ", MaxSplit: 2, KeepEmpty: CheckLineStart);
1272 for (StringRef Record : Records)
1273 if (Record.starts_with(Prefix: "0x") && Record.contains(C: '/'))
1274 return true;
1275 return false;
1276}
1277
1278bool PerfScriptReader::isMMapEvent(StringRef Line) {
1279 // Short cut to avoid string find is possible.
1280 if (Line.empty() || Line.size() < 50)
1281 return false;
1282
1283 if (std::isdigit(Line[0]))
1284 return false;
1285
1286 // PERF_RECORD_MMAP2 or PERF_RECORD_MMAP does not appear at the beginning of
1287 // the line for ` perf script --show-mmap-events -i ...`
1288 return Line.contains(Other: "PERF_RECORD_MMAP");
1289}
1290
1291// The raw hybird sample is like
1292// e.g.
1293// 4005dc # call stack leaf
1294// 400634
1295// 400684 # call stack root
1296// 0x4005c8/0x4005dc/P/-/-/0 0x40062f/0x4005b0/P/-/-/0 ...
1297// ... 0x4005c8/0x4005dc/P/-/-/0 # LBR Entries
1298// Determine the perfscript contains hybrid samples(call stack + LBRs) by
1299// checking whether there is a non-empty call stack immediately followed by
1300// a LBR sample
1301PerfContent PerfScriptReader::checkPerfScriptType(StringRef FileName) {
1302 TraceStream TraceIt(FileName);
1303 uint64_t FrameAddr = 0;
1304 while (!TraceIt.isAtEoF()) {
1305 // Skip the aggregated count
1306 if (!TraceIt.getCurrentLine().getAsInteger(Radix: 10, Result&: FrameAddr))
1307 TraceIt.advance();
1308
1309 // Detect sample with call stack
1310 int32_t Count = 0;
1311 while (!TraceIt.isAtEoF() &&
1312 !isLBRSample(Line: TraceIt.getCurrentLine(), CheckLineStart: false) &&
1313 !parseAddress(Str: TraceIt.getCurrentLine().ltrim(), Addr&: FrameAddr, HasPrefix: false)) {
1314 Count++;
1315 TraceIt.advance();
1316 }
1317 if (!TraceIt.isAtEoF()) {
1318 if (isLBRSample(Line: TraceIt.getCurrentLine(), CheckLineStart: false)) {
1319 if (Count > 0)
1320 return PerfContent::LBRStack;
1321 else
1322 return PerfContent::LBR;
1323 }
1324 TraceIt.advance();
1325 }
1326 }
1327
1328 exitWithError(Message: "Invalid perf script input!");
1329 return PerfContent::UnknownContent;
1330}
1331
1332void HybridPerfReader::generateUnsymbolizedProfile() {
1333 ProfileIsCS = !IgnoreStackSamples;
1334 if (ProfileIsCS)
1335 unwindSamples();
1336 else
1337 PerfScriptReader::generateUnsymbolizedProfile();
1338}
1339
1340void PerfScriptReader::warnTruncatedStack() {
1341 if (ShowDetailedWarning) {
1342 for (auto Address : InvalidReturnAddresses) {
1343 WithColor::warning()
1344 << "Truncated stack sample due to invalid return address at "
1345 << format(Fmt: "0x%" PRIx64, Vals: Address)
1346 << ", likely caused by frame pointer omission\n";
1347 }
1348 }
1349 emitWarningSummary(
1350 Num: InvalidReturnAddresses.size(), Total: AggregatedSamples.size(),
1351 Msg: "of truncated stack samples due to invalid return address, "
1352 "likely caused by frame pointer omission.");
1353}
1354
1355void PerfScriptReader::warnInvalidRange() {
1356 DenseMap<std::pair<uint64_t, uint64_t>, uint64_t> Ranges;
1357
1358 for (const auto &Item : AggregatedSamples) {
1359 const PerfSample *Sample = Item.first.getPtr();
1360 uint64_t Count = Item.second;
1361 uint64_t EndAddress = 0;
1362 for (const LBREntry &LBR : Sample->LBRStack) {
1363 uint64_t SourceAddress = LBR.Source;
1364 uint64_t StartAddress = LBR.Target;
1365 if (EndAddress != 0)
1366 Ranges[{StartAddress, EndAddress}] += Count;
1367 EndAddress = SourceAddress;
1368 }
1369 }
1370
1371 if (Ranges.empty()) {
1372 WithColor::warning() << "No samples in perf script!\n";
1373 return;
1374 }
1375
1376 auto WarnInvalidRange = [&](uint64_t StartAddress, uint64_t EndAddress,
1377 StringRef Msg) {
1378 if (!ShowDetailedWarning)
1379 return;
1380 WithColor::warning() << "[" << format(Fmt: "%8" PRIx64, Vals: StartAddress) << ","
1381 << format(Fmt: "%8" PRIx64, Vals: EndAddress) << "]: " << Msg
1382 << "\n";
1383 };
1384
1385 const char *EndNotBoundaryMsg = "Range is not on instruction boundary, "
1386 "likely due to profile and binary mismatch.";
1387 const char *DanglingRangeMsg = "Range does not belong to any functions, "
1388 "likely from PLT, .init or .fini section.";
1389 const char *RangeCrossFuncMsg =
1390 "Fall through range should not cross function boundaries, likely due to "
1391 "profile and binary mismatch.";
1392 const char *BogusRangeMsg = "Range start is after or too far from range end.";
1393
1394 uint64_t TotalRangeNum = 0;
1395 uint64_t InstNotBoundary = 0;
1396 uint64_t UnmatchedRange = 0;
1397 uint64_t RecoveredRange = 0;
1398 uint64_t RangeCrossFunc = 0;
1399 uint64_t BogusRange = 0;
1400
1401 for (auto &I : Ranges) {
1402 uint64_t StartAddress = I.first.first;
1403 uint64_t EndAddress = I.first.second;
1404 TotalRangeNum += I.second;
1405
1406 if (!Binary->addressIsCode(Address: StartAddress) &&
1407 !Binary->addressIsCode(Address: EndAddress))
1408 continue;
1409
1410 if (!Binary->addressIsCode(Address: StartAddress) ||
1411 !Binary->addressIsTransfer(Address: EndAddress)) {
1412 InstNotBoundary += I.second;
1413 WarnInvalidRange(StartAddress, EndAddress, EndNotBoundaryMsg);
1414 }
1415
1416 auto *FRange = Binary->findFuncRange(Address: StartAddress);
1417 if (!FRange) {
1418 UnmatchedRange += I.second;
1419 WarnInvalidRange(StartAddress, EndAddress, DanglingRangeMsg);
1420 continue;
1421 }
1422
1423 if (FRange->Func->NameStatus != DwarfNameStatus::Matched)
1424 RecoveredRange += I.second;
1425
1426 if (EndAddress >= FRange->EndAddress) {
1427 RangeCrossFunc += I.second;
1428 WarnInvalidRange(StartAddress, EndAddress, RangeCrossFuncMsg);
1429 }
1430
1431 if (Binary->addressIsCode(Address: StartAddress) &&
1432 Binary->addressIsCode(Address: EndAddress) &&
1433 !isValidFallThroughRange(Start: StartAddress, End: EndAddress, Binary)) {
1434 BogusRange += I.second;
1435 WarnInvalidRange(StartAddress, EndAddress, BogusRangeMsg);
1436 }
1437 }
1438
1439 emitWarningSummary(
1440 Num: InstNotBoundary, Total: TotalRangeNum,
1441 Msg: "of samples are from ranges that are not on instruction boundary.");
1442 emitWarningSummary(
1443 Num: UnmatchedRange, Total: TotalRangeNum,
1444 Msg: "of samples are from ranges that do not belong to any functions.");
1445 emitWarningSummary(Num: RecoveredRange, Total: TotalRangeNum,
1446 Msg: "of samples are from ranges that belong to functions "
1447 "recovered from symbol table.");
1448 emitWarningSummary(
1449 Num: RangeCrossFunc, Total: TotalRangeNum,
1450 Msg: "of samples are from ranges that do cross function boundaries.");
1451 emitWarningSummary(
1452 Num: BogusRange, Total: TotalRangeNum,
1453 Msg: "of samples are from ranges that have range start after or too far from "
1454 "range end acrossing the unconditinal jmp.");
1455}
1456
1457void PerfScriptReader::warnIfBranchTargetMismatch() {
1458 // Collect unique branch source and target addresses from LBR samples,
1459 // then check what percentage don't match known instructions in the binary.
1460
1461 uint64_t MismatchedBranches = 0;
1462 uint64_t MismatchedIndirectTargets = 0;
1463 uint64_t MismatchedTargets = 0;
1464 uint64_t TotalSamples = 0;
1465
1466 for (const auto &Item : AggregatedSamples) {
1467 const PerfSample *Sample = Item.first.getPtr();
1468 for (const LBREntry &LBR : Sample->LBRStack) {
1469 uint64_t Source = LBR.Source;
1470 uint64_t Target = LBR.Target;
1471 if (Source == ExternalAddr || Target == ExternalAddr)
1472 continue;
1473 TotalSamples++;
1474
1475 // Validate Branch sources are Call/Branch/Indirect Branch
1476 if (!Binary->addressIsTransfer(Address: Source))
1477 MismatchedBranches++;
1478
1479 // Validate Indirect Branch targets landed in code. This may over estimate
1480 // the vaid targets only because there's no good way to determine jump
1481 // table targets
1482 if (Binary->addressIsIndirectBranch(Address: Source)) {
1483 if (!Binary->addressIsCode(Address: Target))
1484 MismatchedIndirectTargets++;
1485 } else if (!Binary->addressIsBranchTarget(Address: Target) &&
1486 !Binary->findFuncRangeForStartAddr(Address: Target))
1487 MismatchedTargets++;
1488 }
1489 }
1490
1491 emitWarningSummary(Num: MismatchedBranches, Total: TotalSamples,
1492 Msg: "of branch samples do not match the binary.");
1493 emitWarningSummary(Num: MismatchedTargets, Total: TotalSamples,
1494 Msg: "of branch targets do not match the binary.");
1495 emitWarningSummary(Num: MismatchedIndirectTargets, Total: TotalSamples,
1496 Msg: "of indirect branch targets do not match the binary.");
1497}
1498
1499void PerfScriptReader::parsePerfTraces() {
1500 // Parse perf traces and do aggregation.
1501 parseAndAggregateTrace();
1502 if (Binary->isKernel() && !Binary->getIsLoadedByMMap()) {
1503 exitWithError(
1504 Message: "Kernel is requested, but no kernel is found in mmap events.");
1505 }
1506
1507 emitWarningSummary(Num: NumLeafExternalFrame, Total: NumTotalSample,
1508 Msg: "of samples have leaf external frame in call stack.");
1509 emitWarningSummary(Num: NumLeadingOutgoingLBR, Total: NumTotalSample,
1510 Msg: "of samples have leading external LBR.");
1511
1512 // Generate unsymbolized profile.
1513 warnTruncatedStack();
1514 warnInvalidRange();
1515 warnIfBranchTargetMismatch();
1516 generateUnsymbolizedProfile();
1517 AggregatedSamples.clear();
1518
1519 if (SkipSymbolization)
1520 writeUnsymbolizedProfile(Filename: OutputFilename);
1521}
1522
1523SmallVector<CleanupInstaller, 2> PerfScriptReader::TempFileCleanups;
1524
1525void ETMReader::recordProcessedRange(uint64_t Start, uint64_t End,
1526 uint64_t Count) {
1527 assert(!Counters.empty() && "Counters should not be empty!");
1528 auto &Counter = Counters.begin()->second;
1529 Counter.recordRangeCount(Start, End, Repeat: Count);
1530}
1531
1532class ETMCallback : public ETMDecoder::Callback {
1533 ETMReader *Reader;
1534
1535public:
1536 ETMCallback(ETMReader *R) : Reader(R) {}
1537 void processInstructionRange(uint64_t Start, uint64_t End) override {
1538 Reader->recordProcessedRange(Start, End, Count: 1);
1539 }
1540};
1541
1542void ETMReader::parseETMTraces() {
1543 auto BufferOrErr = MemoryBuffer::getFile(Filename: TraceFile);
1544 if (std::error_code EC = BufferOrErr.getError())
1545 exitWithError(Message: "Could not open ETM trace file: " + EC.message());
1546
1547 ArrayRef<uint8_t> Data(
1548 reinterpret_cast<const uint8_t *>((*BufferOrErr)->getBufferStart()),
1549 (*BufferOrErr)->getBufferSize());
1550
1551 // There is no context for ETM instruction traces.
1552 // Initialize the SampleCounters map with a single empty context key
1553 // to aggregate all instruction hits into a global bucket.
1554 auto Key = std::make_shared<StringBasedCtxKey>();
1555 Counters.try_emplace(Key: Hashable<ContextKey>(Key));
1556
1557 // The protocol utilizes a 0x80 byte as an initial synchronization header.
1558 // Perform a manual search for this sync point to discard any leading
1559 // padding or truncated packets before decoding begins.
1560 size_t StartIdx = 0;
1561 while (StartIdx < Data.size() && Data[StartIdx] != 0x80)
1562 StartIdx++;
1563 if (StartIdx >= Data.size())
1564 exitWithError(Message: "No synchronization header (0x80) found in the bitstream.");
1565 ArrayRef<uint8_t> TraceSlice = Data.slice(N: StartIdx);
1566
1567 auto DecoderOrErr = ETMDecoder::create(
1568 Binary: Binary->getBinary(), TargetTriple: Binary->getTriple(), TraceID: static_cast<uint8_t>(TraceID));
1569
1570 if (!DecoderOrErr)
1571 exitWithError(Message: toString(E: DecoderOrErr.takeError()));
1572 auto Decoder = std::move(*DecoderOrErr);
1573
1574 ETMCallback CB(this);
1575 if (Error E = Decoder->processTrace(TraceData: TraceSlice, TraceCallback&: CB))
1576 exitWithError(Message: toString(E: std::move(E)));
1577}
1578
1579} // end namespace sampleprof
1580} // end namespace llvm
1581