1//===-- PerfReader.cpp - perfscript reader ---------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8#include "PerfReader.h"
9#include "ErrorHandling.h"
10#include "Options.h"
11#include "ProfileGenerator.h"
12#include "llvm/ADT/SmallString.h"
13#include "llvm/ProfileData/ETMTraceDecoder.h"
14#include "llvm/Support/FileSystem.h"
15#include "llvm/Support/LineIterator.h"
16#include "llvm/Support/MemoryBuffer.h"
17#include "llvm/Support/Process.h"
18#include "llvm/Support/Timer.h"
19#include "llvm/Support/ToolOutputFile.h"
20#include "llvm/Support/raw_ostream.h"
21#include "llvm/TargetParser/Triple.h"
22#include <cctype>
23
24#define DEBUG_TYPE "perf-reader"
25
26namespace llvm {
27
28cl::opt<bool> SkipSymbolization("skip-symbolization",
29 cl::desc("Dump the unsymbolized profile to the "
30 "output file. It will show unwinder "
31 "output for CS profile generation."),
32 cl::cat(ProfGenCategory));
33
34static cl::opt<bool> ShowMmapEvents("show-mmap-events",
35 cl::desc("Print binary load events."),
36 cl::cat(ProfGenCategory));
37
38static cl::opt<bool>
39 UseOffset("use-offset", cl::init(Val: true),
40 cl::desc("Work with `--skip-symbolization` or "
41 "`--unsymbolized-profile` to write/read the "
42 "offset instead of virtual address."),
43 cl::cat(ProfGenCategory));
44
45static cl::opt<bool> UseLoadableSegmentAsBase(
46 "use-first-loadable-segment-as-base",
47 cl::desc("Use first loadable segment address as base address "
48 "for offsets in unsymbolized profile. By default "
49 "first executable segment address is used"),
50 cl::cat(ProfGenCategory));
51
52static cl::opt<bool>
53 IgnoreStackSamples("ignore-stack-samples",
54 cl::desc("Ignore call stack samples for hybrid samples "
55 "and produce context-insensitive profile."),
56 cl::cat(ProfGenCategory));
57cl::opt<bool> ShowDetailedWarning("show-detailed-warning",
58 cl::desc("Show detailed warning message."),
59 cl::cat(ProfGenCategory));
60
61static cl::opt<int> CSProfMaxUnsymbolizedCtxDepth(
62 "csprof-max-unsymbolized-context-depth", cl::init(Val: -1),
63 cl::desc("Keep the last K contexts while merging unsymbolized profile. -1 "
64 "means no depth limit."),
65 cl::cat(ProfGenCategory));
66
67cl::opt<bool> TimeProfGen("time-profgen", cl::desc("Time llvm-profgen phases"),
68 cl::init(Val: false), cl::cat(ProfGenCategory));
69
70static const char *TimerGroupName = "profgen";
71static const char *TimerGroupDesc = "llvm-profgen";
72
73namespace sampleprof {
74
75void VirtualUnwinder::unwindCall(UnwindState &State) {
76 uint64_t Source = State.getCurrentLBRSource();
77 auto *ParentFrame = State.getParentFrame();
78 // The 2nd frame after leaf could be missing if stack sample is
79 // taken when IP is within prolog/epilog, as frame chain isn't
80 // setup yet. Fill in the missing frame in that case.
81 // TODO: Currently we just assume all the addr that can't match the
82 // 2nd frame is in prolog/epilog. In the future, we will switch to
83 // pro/epi tracker(Dwarf CFI) for the precise check.
84 if (ParentFrame == State.getDummyRootPtr() ||
85 ParentFrame->Address != Source) {
86 State.switchToFrame(Address: Source);
87 if (ParentFrame != State.getDummyRootPtr()) {
88 if (Source == ExternalAddr)
89 NumMismatchedExtCallBranch++;
90 else
91 NumMismatchedProEpiBranch++;
92 }
93 } else {
94 State.popFrame();
95 }
96 State.InstPtr.update(Addr: Source);
97}
98
99void VirtualUnwinder::unwindLinear(UnwindState &State, uint64_t Repeat) {
100 InstructionPointer &IP = State.InstPtr;
101 uint64_t Target = State.getCurrentLBRTarget();
102 uint64_t End = IP.Address;
103
104 if (End == ExternalAddr && Target == ExternalAddr) {
105 // Filter out the case when leaf external frame matches the external LBR
106 // target, this is a valid state, it happens that the code run into external
107 // address then return back. The call frame under the external frame
108 // remains valid and can be unwound later, just skip recording this range.
109 NumPairedExtAddr++;
110 return;
111 }
112
113 if (End == ExternalAddr || Target == ExternalAddr) {
114 // Range is invalid if only one point is external address. This means LBR
115 // traces contains a standalone external address failing to pair another
116 // one, likely due to interrupt jmp or broken perf script. Set the
117 // state to invalid.
118 NumUnpairedExtAddr++;
119 State.setInvalid();
120 return;
121 }
122
123 if (!isValidFallThroughRange(Start: Target, End, Binary)) {
124 // Skip unwinding the rest of LBR trace when a bogus range is seen.
125 State.setInvalid();
126 return;
127 }
128
129 if (Binary->usePseudoProbes()) {
130 // We don't need to top frame probe since it should be extracted
131 // from the range.
132 // The outcome of the virtual unwinding with pseudo probes is a
133 // map from a context key to the address range being unwound.
134 // This means basically linear unwinding is not needed for pseudo
135 // probes. The range will be simply recorded here and will be
136 // converted to a list of pseudo probes to report in ProfileGenerator.
137 State.getParentFrame()->recordRangeCount(Start: Target, End, Count: Repeat);
138 } else {
139 // Unwind linear execution part.
140 // Split and record the range by different inline context. For example:
141 // [0x01] ... main:1 # Target
142 // [0x02] ... main:2
143 // [0x03] ... main:3 @ foo:1
144 // [0x04] ... main:3 @ foo:2
145 // [0x05] ... main:3 @ foo:3
146 // [0x06] ... main:4
147 // [0x07] ... main:5 # End
148 // It will be recorded:
149 // [main:*] : [0x06, 0x07], [0x01, 0x02]
150 // [main:3 @ foo:*] : [0x03, 0x05]
151 while (IP.Address > Target) {
152 uint64_t PrevIP = IP.Address;
153 IP.backward();
154 // Break into segments for implicit call/return due to inlining
155 bool SameInlinee = Binary->inlineContextEqual(Add1: PrevIP, Add2: IP.Address);
156 if (!SameInlinee) {
157 State.switchToFrame(Address: PrevIP);
158 State.CurrentLeafFrame->recordRangeCount(Start: PrevIP, End, Count: Repeat);
159 End = IP.Address;
160 }
161 }
162 assert(IP.Address == Target && "The last one must be the target address.");
163 // Record the remaining range, [0x01, 0x02] in the example
164 State.switchToFrame(Address: IP.Address);
165 State.CurrentLeafFrame->recordRangeCount(Start: IP.Address, End, Count: Repeat);
166 }
167}
168
169void VirtualUnwinder::unwindReturn(UnwindState &State) {
170 // Add extra frame as we unwind through the return
171 const LBREntry &LBR = State.getCurrentLBR();
172 uint64_t CallAddr = Binary->getCallAddrFromFrameAddr(FrameAddr: LBR.Target);
173 State.switchToFrame(Address: CallAddr);
174 State.pushFrame(Address: LBR.Source);
175 State.InstPtr.update(Addr: LBR.Source);
176}
177
178void VirtualUnwinder::unwindBranch(UnwindState &State) {
179 // TODO: Tolerate tail call for now, as we may see tail call from libraries.
180 // This is only for intra function branches, excluding tail calls.
181 uint64_t Source = State.getCurrentLBRSource();
182 State.switchToFrame(Address: Source);
183 State.InstPtr.update(Addr: Source);
184}
185
186std::shared_ptr<StringBasedCtxKey> FrameStack::getContextKey() {
187 std::shared_ptr<StringBasedCtxKey> KeyStr =
188 std::make_shared<StringBasedCtxKey>();
189 KeyStr->Context = Binary->getExpandedContext(Stack, WasLeafInlined&: KeyStr->WasLeafInlined);
190 return KeyStr;
191}
192
193std::shared_ptr<AddrBasedCtxKey> AddressStack::getContextKey() {
194 std::shared_ptr<AddrBasedCtxKey> KeyStr = std::make_shared<AddrBasedCtxKey>();
195 KeyStr->Context = Stack;
196 CSProfileGenerator::compressRecursionContext<uint64_t>(Context&: KeyStr->Context);
197 // MaxContextDepth(--csprof-max-context-depth) is used to trim both symbolized
198 // and unsymbolized profile context. Sometimes we want to at least preserve
199 // the inlinings for the leaf frame(the profiled binary inlining),
200 // --csprof-max-context-depth may not be flexible enough, in this case,
201 // --csprof-max-unsymbolized-context-depth is used to limit the context for
202 // unsymbolized profile. If both are set, use the minimum of them.
203 int Depth = CSProfileGenerator::MaxContextDepth != -1
204 ? CSProfileGenerator::MaxContextDepth
205 : KeyStr->Context.size();
206 Depth = CSProfMaxUnsymbolizedCtxDepth != -1
207 ? std::min(a: static_cast<int>(CSProfMaxUnsymbolizedCtxDepth), b: Depth)
208 : Depth;
209 CSProfileGenerator::trimContext<uint64_t>(S&: KeyStr->Context, Depth);
210 return KeyStr;
211}
212
213template <typename T>
214void VirtualUnwinder::collectSamplesFromFrame(UnwindState::ProfiledFrame *Cur,
215 T &Stack) {
216 if (Cur->RangeSamples.empty() && Cur->BranchSamples.empty())
217 return;
218
219 std::shared_ptr<ContextKey> Key = Stack.getContextKey();
220 if (Key == nullptr)
221 return;
222 auto Ret = CtxCounterMap->try_emplace(Key: Hashable<ContextKey>(Key));
223 SampleCounter &SCounter = Ret.first->second;
224 for (auto &I : Cur->RangeSamples)
225 SCounter.recordRangeCount(Start: std::get<0>(t&: I), End: std::get<1>(t&: I), Repeat: std::get<2>(t&: I));
226
227 for (auto &I : Cur->BranchSamples)
228 SCounter.recordBranchCount(Source: std::get<0>(t&: I), Target: std::get<1>(t&: I), Repeat: std::get<2>(t&: I));
229}
230
231template <typename T>
232void VirtualUnwinder::collectSamplesFromFrameTrie(
233 UnwindState::ProfiledFrame *Cur, T &Stack) {
234 if (!Cur->isDummyRoot()) {
235 // Truncate the context for external frame since this isn't a real call
236 // context the compiler will see.
237 if (Cur->isExternalFrame() || !Stack.pushFrame(Cur)) {
238 // Process truncated context
239 // Start a new traversal ignoring its bottom context
240 T EmptyStack(Binary);
241 collectSamplesFromFrame(Cur, EmptyStack);
242 for (const auto &Item : Cur->Children) {
243 collectSamplesFromFrameTrie(Item.second.get(), EmptyStack);
244 }
245
246 // Keep note of untracked call site and deduplicate them
247 // for warning later.
248 if (!Cur->isLeafFrame())
249 UntrackedCallsites.insert(x: Cur->Address);
250
251 return;
252 }
253 }
254
255 collectSamplesFromFrame(Cur, Stack);
256 // Process children frame
257 for (const auto &Item : Cur->Children) {
258 collectSamplesFromFrameTrie(Item.second.get(), Stack);
259 }
260 // Recover the call stack
261 Stack.popFrame();
262}
263
264void VirtualUnwinder::collectSamplesFromFrameTrie(
265 UnwindState::ProfiledFrame *Cur) {
266 if (Binary->usePseudoProbes()) {
267 AddressStack Stack(Binary);
268 collectSamplesFromFrameTrie<AddressStack>(Cur, Stack);
269 } else {
270 FrameStack Stack(Binary);
271 collectSamplesFromFrameTrie<FrameStack>(Cur, Stack);
272 }
273}
274
275void VirtualUnwinder::recordBranchCount(const LBREntry &Branch,
276 UnwindState &State, uint64_t Repeat) {
277 if (Branch.Target == ExternalAddr)
278 return;
279
280 // Record external-to-internal pattern on the trie root, it later can be
281 // used for generating head samples.
282 if (Branch.Source == ExternalAddr) {
283 State.getDummyRootPtr()->recordBranchCount(Source: Branch.Source, Target: Branch.Target,
284 Count: Repeat);
285 return;
286 }
287
288 if (Binary->usePseudoProbes()) {
289 // Same as recordRangeCount, We don't need to top frame probe since we will
290 // extract it from branch's source address
291 State.getParentFrame()->recordBranchCount(Source: Branch.Source, Target: Branch.Target,
292 Count: Repeat);
293 } else {
294 State.CurrentLeafFrame->recordBranchCount(Source: Branch.Source, Target: Branch.Target,
295 Count: Repeat);
296 }
297}
298
299bool VirtualUnwinder::unwind(const PerfSample *Sample, uint64_t Repeat) {
300 // Capture initial state as starting point for unwinding.
301 UnwindState State(Sample, Binary);
302
303 NumTotalBranches += State.LBRStack.size();
304 // Now process the LBR samples in parrallel with stack sample
305 // Note that we do not reverse the LBR entry order so we can
306 // unwind the sample stack as we walk through LBR entries.
307 while (State.hasNextLBR()) {
308 State.checkStateConsistency();
309
310 // Do not attempt linear unwind for the leaf range as it's incomplete.
311 if (!State.IsLastLBR()) {
312 // Unwind implicit calls/returns from inlining, along the linear path,
313 // break into smaller sub section each with its own calling context.
314 unwindLinear(State, Repeat);
315 }
316
317 // Save the LBR branch before it gets unwound.
318 const LBREntry &Branch = State.getCurrentLBR();
319 if (isCallState(State)) {
320 // Unwind calls - we know we encountered call if LBR overlaps with
321 // transition between leaf the 2nd frame. Note that for calls that
322 // were not in the original stack sample, we should have added the
323 // extra frame when processing the return paired with this call.
324 unwindCall(State);
325 } else if (isReturnState(State)) {
326 // Unwind returns - check whether the IP is indeed at a return
327 // instruction
328 unwindReturn(State);
329 } else if (isValidState(State)) {
330 // Unwind branches
331 unwindBranch(State);
332 } else {
333 // Skip unwinding the rest of LBR trace. Reset the stack and update the
334 // state so that the rest of the trace can still be processed as if they
335 // do not have stack samples.
336 State.clearCallStack();
337 State.InstPtr.update(Addr: State.getCurrentLBRSource());
338 State.pushFrame(Address: State.InstPtr.Address);
339 }
340
341 State.advanceLBR();
342 // Record `branch` with calling context after unwinding.
343 recordBranchCount(Branch, State, Repeat);
344 }
345 // As samples are aggregated on trie, record them into counter map
346 collectSamplesFromFrameTrie(Cur: State.getDummyRootPtr());
347
348 return true;
349}
350
351std::unique_ptr<PerfReaderBase>
352PerfReaderBase::create(ProfiledBinary *Binary, InputFile &Input,
353 std::optional<int32_t> PIDFilter) {
354 std::unique_ptr<PerfReaderBase> PerfReader;
355
356 if (Input.Format == InputFormat::UnsymbolizedProfile) {
357 PerfReader.reset(
358 p: new UnsymbolizedProfileReader(Binary, Input.InputFilePath));
359 return PerfReader;
360 }
361
362 // For perf data input, we need to convert them into perf script first.
363 // If this is a kernel perf file, there is no need for retrieving PIDs.
364 if (Input.Format == InputFormat::PerfData)
365 Input = PerfScriptReader::convertPerfDataToTrace(Binary, SkipPID: Binary->isKernel(),
366 File&: Input, PIDFilter);
367
368 assert((Input.Format == InputFormat::PerfScript) &&
369 "Should be a perfscript!");
370
371 Input.Content = PerfScriptReader::checkPerfScriptType(FileName: Input.InputFilePath);
372 if (Input.Content == PerfContent::LBRStack) {
373 PerfReader.reset(
374 p: new HybridPerfReader(Binary, Input.InputFilePath, PIDFilter));
375 } else if (Input.Content == PerfContent::LBR) {
376 PerfReader.reset(p: new LBRPerfReader(Binary, Input.InputFilePath, PIDFilter));
377 } else {
378 exitWithError(Message: "Unsupported perfscript!");
379 }
380
381 return PerfReader;
382}
383
384Error PerfReaderBase::parseDataAccessPerfTraces(
385 StringRef DataAccessPerfTraceFile, std::optional<int32_t> PIDFilter) {
386 // A perf_record_sample line is like
387 // . 1282514022939813 0x87b0 [0x60]: PERF_RECORD_SAMPLE(IP, 0x4002):
388 // 3446532/3446532: 0x2608a2 period: 233 addr: 0x3b3fb0
389 constexpr static StringRef DataAccessSamplePattern =
390 "PERF_RECORD_SAMPLE\\([A-Za-z]+, 0x[0-9a-fA-F]+\\): "
391 "([0-9]+)\\/[0-9]+: 0x([0-9a-fA-F]+) period: [0-9]+ addr: "
392 "0x([0-9a-fA-F]+)";
393
394 llvm::Regex LogRegex(DataAccessSamplePattern);
395
396 auto BufferOrErr = MemoryBuffer::getFile(Filename: DataAccessPerfTraceFile);
397 std::error_code EC = BufferOrErr.getError();
398 if (EC)
399 return make_error<StringError>(Args: "Failed to open perf trace file: " +
400 DataAccessPerfTraceFile,
401 Args: inconvertibleErrorCode());
402
403 assert(!SampleCounters.empty() && "Sample counters should not be empty!");
404 SampleCounter &Counter = SampleCounters.begin()->second;
405 line_iterator LineIt(*BufferOrErr.get(), true);
406
407 for (; !LineIt.is_at_eof(); ++LineIt) {
408 StringRef Line = *LineIt;
409
410 MMapEvent MMap;
411 if (Line.contains(Other: "PERF_RECORD_MMAP2")) {
412 if (PerfScriptReader::extractMMapEventForBinary(Binary, Line, MMap)) {
413 if (!MMap.MemProtectionFlag.contains(Other: "x")) {
414 if (Error E = Binary->addMMapNonTextEvent(Event: MMap)) {
415 return E;
416 }
417 }
418 }
419 continue;
420 }
421
422 SmallVector<StringRef> Fields;
423 if (LogRegex.match(String: Line, Matches: &Fields)) {
424 int32_t PID = 0;
425 if (Fields[1].getAsInteger(Radix: 10, Result&: PID))
426 return make_error<StringError>(
427 Args: "Failed to parse PID from perf trace line: " + Line,
428 Args: inconvertibleErrorCode());
429
430 if (PIDFilter.has_value() && *PIDFilter != PID) {
431 continue;
432 }
433
434 uint64_t DataAddress = 0;
435 if (Fields[3].getAsInteger(Radix: 16, Result&: DataAddress))
436 return make_error<StringError>(
437 Args: "Failed to parse data address from perf trace line: " + Line,
438 Args: inconvertibleErrorCode());
439 // Out of all the memory access events, the vtable accesses are used to
440 // construct type profiles. We assume that this is under the Itanium
441 // C++ ABI so we can use `_ZTV` prefix to identify vtable.
442 StringRef DataSymbol = Binary->symbolizeDataAddress(
443 Address: Binary->CanonicalizeNonTextAddress(Address: DataAddress));
444 if (DataSymbol.starts_with(Prefix: "_ZTV")) {
445 uint64_t IP = 0;
446 Fields[2].getAsInteger(Radix: 16, Result&: IP);
447 Counter.recordDataAccessCount(InstAddr: Binary->canonicalizeVirtualAddress(Address: IP),
448 DataSymbol, Repeat: 1);
449 }
450 }
451 }
452 return Error::success();
453}
454
455InputFile
456PerfScriptReader::convertPerfDataToTrace(ProfiledBinary *Binary, bool SkipPID,
457 InputFile &File,
458 std::optional<int32_t> PIDFilter) {
459 StringRef PerfData = File.InputFilePath;
460 // Run perf script to retrieve PIDs matching binary we're interested in.
461 auto PerfExecutable = sys::Process::FindInEnvPath(EnvName: "PATH", FileName: "perf");
462 if (!PerfExecutable) {
463 exitWithError(Message: "Perf not found.");
464 }
465 std::string PerfExecutablePath = *PerfExecutable;
466 SmallString<128> PerfTraceFile;
467 sys::fs::createUniquePath(Model: "perf-script-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%.tmp",
468 ResultPath&: PerfTraceFile, /*MakeAbsolute=*/true);
469 std::string ErrorFile = std::string(PerfTraceFile) + ".err";
470 std::optional<StringRef> Redirects[] = {std::nullopt, // Stdin
471 StringRef(PerfTraceFile), // Stdout
472 StringRef(ErrorFile)}; // Stderr
473 PerfScriptReader::TempFileCleanups.emplace_back(Args&: PerfTraceFile);
474 PerfScriptReader::TempFileCleanups.emplace_back(Args&: ErrorFile);
475
476 auto RunPerfScript = [&](ArrayRef<StringRef> Args) {
477 // ExecuteAndWait does not truncate redirected output files on Unix. Remove
478 // both files so a shorter invocation cannot retain output from the
479 // previous perf script invocation.
480 for (StringRef Path : {StringRef(PerfTraceFile), StringRef(ErrorFile)}) {
481 if (std::error_code EC = sys::fs::remove(path: Path))
482 exitWithError(EC, Whence: Path);
483 }
484
485 std::string ExecutionError;
486 bool ExecutionFailed = false;
487 int ExitCode =
488 sys::ExecuteAndWait(Program: PerfExecutablePath, Args, Env: std::nullopt, Redirects,
489 /*SecondsToWait=*/0, /*MemoryLimit=*/0,
490 ErrMsg: &ExecutionError, ExecutionFailed: &ExecutionFailed);
491 if (!ExecutionFailed && ExitCode == 0)
492 return;
493
494 std::string Message;
495 raw_string_ostream OS(Message);
496 if (ExecutionFailed || ExitCode == -1)
497 OS << "Failed to execute perf script";
498 else if (ExitCode == -2)
499 OS << "Perf script terminated abnormally";
500 else
501 OS << "Perf script failed with exit code " << ExitCode;
502 if (!ExecutionError.empty())
503 OS << ": " << ExecutionError;
504
505 if (auto ErrorBuffer = MemoryBuffer::getFile(Filename: ErrorFile)) {
506 StringRef Stderr = ErrorBuffer.get()->getBuffer().trim();
507 if (!Stderr.empty())
508 OS << "\n" << Stderr;
509 }
510 exitWithError(Message: OS.str());
511 };
512
513 std::string PIDs;
514 if (!SkipPID) {
515 StringRef ScriptMMapArgs[] = {PerfExecutablePath,
516 "script",
517 "--show-mmap-events",
518 "-F",
519 "comm,pid",
520 "-i",
521 PerfData};
522 RunPerfScript(ScriptMMapArgs);
523
524 // Collect the PIDs
525 TraceStream TraceIt(PerfTraceFile);
526 DenseSet<int32_t> PIDSet;
527 while (!TraceIt.isAtEoF()) {
528 MMapEvent MMap;
529 if (isMMapEvent(Line: TraceIt.getCurrentLine()) &&
530 extractMMapEventForBinary(Binary, Line: TraceIt.getCurrentLine(), MMap)) {
531 auto It = PIDSet.insert(V: MMap.PID);
532 if (It.second && (!PIDFilter || MMap.PID == *PIDFilter)) {
533 if (!PIDs.empty()) {
534 PIDs.append(s: ",");
535 }
536 PIDs.append(str: utostr(X: MMap.PID));
537 }
538 }
539 TraceIt.advance();
540 }
541
542 if (PIDs.empty()) {
543 exitWithError(Message: "No relevant mmap event is found in perf data.");
544 }
545 }
546
547 // Run perf script again to retrieve events for PIDs collected above
548 SmallVector<StringRef, 8> ScriptSampleArgs;
549 ScriptSampleArgs.push_back(Elt: PerfExecutablePath);
550 ScriptSampleArgs.push_back(Elt: "script");
551 ScriptSampleArgs.push_back(Elt: "--show-mmap-events");
552 ScriptSampleArgs.push_back(Elt: "-F");
553 ScriptSampleArgs.push_back(Elt: "ip,brstack");
554 ScriptSampleArgs.push_back(Elt: "-i");
555 ScriptSampleArgs.push_back(Elt: PerfData);
556 if (!PIDs.empty()) {
557 ScriptSampleArgs.push_back(Elt: "--pid");
558 ScriptSampleArgs.push_back(Elt: PIDs);
559 }
560 RunPerfScript(ScriptSampleArgs);
561
562 return {.InputFilePath: std::string(PerfTraceFile), .Format: InputFormat::PerfScript,
563 .Content: PerfContent::UnknownContent};
564}
565
566static StringRef filename(StringRef Path, bool UseBackSlash) {
567 llvm::sys::path::Style PathStyle =
568 UseBackSlash ? llvm::sys::path::Style::windows_backslash
569 : llvm::sys::path::Style::native;
570 StringRef FileName = llvm::sys::path::filename(path: Path, style: PathStyle);
571
572 // In case this file use \r\n as newline.
573 if (UseBackSlash && FileName.back() == '\r')
574 return FileName.drop_back();
575
576 return FileName;
577}
578
579void PerfScriptReader::updateBinaryAddress(const MMapEvent &Event) {
580 // Drop the event which doesn't belong to user-provided binary
581 StringRef BinaryName = filename(Path: Event.BinaryPath, UseBackSlash: Binary->isCOFF());
582 bool IsKernel = Binary->isKernel();
583 if (!IsKernel && Binary->getName() != BinaryName)
584 return;
585 if (IsKernel && !Binary->isKernelImageName(BinaryName))
586 return;
587
588 // Drop the event if process does not match pid filter
589 if (PIDFilter && Event.PID != *PIDFilter)
590 return;
591
592 Binary->addMMapRange(Address: Event.Address, Size: Event.Size);
593
594 // Check if the FileOffset falls within the [Event.Offset, Event.Offset +
595 // Event.Size) range.
596 auto MMapContainsFileOffset = [&](uint64_t FileOffset) {
597 return Event.Offset <= FileOffset &&
598 (FileOffset - Event.Offset) < Event.Size;
599 };
600
601 if (IsKernel || MMapContainsFileOffset(Binary->getTextSegmentOffset())) {
602 // For ELF, subtract the file offset to get the runtime address
603 // corresponding to file offset zero. Kernel mmap events report the text
604 // address as Event.Offset, so use the text segment offset from the ELF
605 // instead.
606 const uint64_t RuntimeBaseAddress =
607 Binary->isCOFF()
608 ? Event.Address
609 : Event.Address -
610 (IsKernel ? Binary->getTextSegmentOffset() : Event.Offset);
611 // A binary image could be unloaded and then reloaded at different
612 // place, so update binary load address.
613 // Only update for the first executable segment and assume all other
614 // segments are loaded at consecutive memory addresses, which is the case on
615 // X64.
616 Binary->setBaseAddress(RuntimeBaseAddress);
617 Binary->setIsLoadedByMMap(true);
618 } else {
619 // Verify segments are loaded consecutively.
620 const auto &Offsets = Binary->getTextSegmentOffsets();
621 auto IsContiguousMMapForSegment = [&](auto SegmentIt, uint64_t FileOffset,
622 uint64_t RuntimeAddress) {
623 auto I = std::distance(Offsets.begin(), SegmentIt);
624 const auto &PreferredAddrs = Binary->getPreferredTextSegmentAddresses();
625 return PreferredAddrs[I] + (FileOffset - *SegmentIt) ==
626 Binary->canonicalizeVirtualAddress(Address: RuntimeAddress);
627 };
628
629 auto It = llvm::lower_bound(Range: Offsets, Value: Event.Offset);
630 if (It != Offsets.end() && MMapContainsFileOffset(*It)) {
631 // The event is for loading a separate executable segment.
632 uint64_t RuntimeSegmentAddress = Event.Address + (*It - Event.Offset);
633 if (!IsContiguousMMapForSegment(It, *It, RuntimeSegmentAddress))
634 exitWithError(Message: "Executable segments not loaded consecutively");
635 } else {
636 if (It == Offsets.begin())
637 exitWithError(Message: "File offset not found");
638 else {
639 // Find the segment the event falls in. A large segment could be loaded
640 // via multiple mmap calls with consecutive memory addresses.
641 --It;
642 assert(*It < Event.Offset);
643 if (!IsContiguousMMapForSegment(It, Event.Offset, Event.Address))
644 exitWithError(Message: "Segment not loaded by consecutive mmaps");
645 }
646 }
647 }
648}
649
650static std::string getContextKeyStr(ContextKey *K,
651 const ProfiledBinary *Binary) {
652 if (const auto *CtxKey = dyn_cast<StringBasedCtxKey>(Val: K)) {
653 return SampleContext::getContextString(Context: CtxKey->Context);
654 } else if (const auto *CtxKey = dyn_cast<AddrBasedCtxKey>(Val: K)) {
655 std::ostringstream OContextStr;
656 for (uint32_t I = 0; I < CtxKey->Context.size(); I++) {
657 if (OContextStr.str().size())
658 OContextStr << " @ ";
659 uint64_t Address = CtxKey->Context[I];
660 if (UseOffset) {
661 if (UseLoadableSegmentAsBase)
662 Address -= Binary->getFirstLoadableAddress();
663 else
664 Address -= Binary->getPreferredBaseAddress();
665 }
666 OContextStr << "0x"
667 << utohexstr(X: Address,
668 /*LowerCase=*/true);
669 }
670 return OContextStr.str();
671 } else {
672 llvm_unreachable("unexpected key type");
673 }
674}
675
676void HybridPerfReader::unwindSamples() {
677 NamedRegionTimer T("unwind", "Unwind samples", TimerGroupName, TimerGroupDesc,
678 TimeProfGen);
679 VirtualUnwinder Unwinder(&SampleCounters, Binary);
680 for (const auto &Item : AggregatedSamples) {
681 const PerfSample *Sample = Item.first.getPtr();
682 Unwinder.unwind(Sample, Repeat: Item.second);
683 }
684
685 // Warn about untracked frames due to missing probes.
686 if (ShowDetailedWarning) {
687 for (auto Address : Unwinder.getUntrackedCallsites())
688 WithColor::warning() << "Profile context truncated due to missing probe "
689 << "for call instruction at "
690 << format(Fmt: "0x%" PRIx64, Vals: Address) << "\n";
691 }
692
693 emitWarningSummary(Num: Unwinder.getUntrackedCallsites().size(),
694 Total: SampleCounters.size(),
695 Msg: "of profiled contexts are truncated due to missing probe "
696 "for call instruction.");
697
698 emitWarningSummary(
699 Num: Unwinder.NumMismatchedExtCallBranch, Total: Unwinder.NumTotalBranches,
700 Msg: "of branches'source is a call instruction but doesn't match call frame "
701 "stack, likely due to unwinding error of external frame.");
702
703 emitWarningSummary(Num: Unwinder.NumPairedExtAddr * 2, Total: Unwinder.NumTotalBranches,
704 Msg: "of branches containing paired external address.");
705
706 emitWarningSummary(Num: Unwinder.NumUnpairedExtAddr, Total: Unwinder.NumTotalBranches,
707 Msg: "of branches containing external address but doesn't have "
708 "another external address to pair, likely due to "
709 "interrupt jmp or broken perf script.");
710
711 emitWarningSummary(
712 Num: Unwinder.NumMismatchedProEpiBranch, Total: Unwinder.NumTotalBranches,
713 Msg: "of branches'source is a call instruction but doesn't match call frame "
714 "stack, likely due to frame in prolog/epilog.");
715
716 emitWarningSummary(Num: Unwinder.NumMissingExternalFrame,
717 Total: Unwinder.NumExtCallBranch,
718 Msg: "of artificial call branches but doesn't have an external "
719 "frame to match.");
720
721 emitWarningSummary(Num: NumBogusTrace, Total: NumTotalHybridSample,
722 Msg: "of hybrid samples had a callchain leaf that disagreed "
723 "with the newest LBR target (bogus trace).");
724 if (NumBogusTrace * 100 > NumTotalHybridSample)
725 WithColor::warning() << "Bogus trace rate exceeds 1%: the profile has high "
726 "sample skid and may not be suitable for "
727 "optimization.\n";
728}
729
730/// Parse a hex address from \p Str.
731static bool parseAddress(StringRef Str, uint64_t &Addr, bool HasPrefix) {
732 Str = Str.take_while(F: [](char C) { return !isspace(C); });
733 if (Str.consume_front(Prefix: "0x") != HasPrefix)
734 return true;
735 return Str.getAsInteger(Radix: 16, Result&: Addr);
736}
737
738bool PerfScriptReader::extractLBRStack(TraceStream &TraceIt,
739 SmallVectorImpl<LBREntry> &LBRStack) {
740 // The raw format of LBR stack is like:
741 // 0x4005c8/0x4005dc/P/-/-/0 0x40062f/0x4005b0/P/-/-/0 ...
742 // ... 0x4005c8/0x4005dc/P/-/-/0
743 // It's in FIFO order and separated by whitespace.
744 SmallVector<StringRef, 32> Records;
745 TraceIt.getCurrentLine().rtrim().split(A&: Records, Separator: " ", MaxSplit: -1, KeepEmpty: false);
746 auto WarnInvalidLBR = [](TraceStream &TraceIt) {
747 WithColor::warning() << "Invalid address in LBR record at line "
748 << TraceIt.getLineNumber() << ": "
749 << TraceIt.getCurrentLine() << "\n";
750 };
751
752 // Skip the leading instruction pointer.
753 size_t Index = 0;
754 uint64_t LeadingAddr;
755 if (!Records.empty() && !Records[0].contains(C: '/')) {
756 if (parseAddress(Str: Records[0], Addr&: LeadingAddr, HasPrefix: false)) {
757 WarnInvalidLBR(TraceIt);
758 TraceIt.advance();
759 return false;
760 }
761 Index = 1;
762 }
763
764 // Now extract LBR samples - note that we do not reverse the
765 // LBR entry order so we can unwind the sample stack as we walk
766 // through LBR entries.
767 while (Index < Records.size()) {
768 auto &Token = Records[Index++];
769 if (Token.size() == 0)
770 continue;
771
772 SmallVector<StringRef, 8> Addresses;
773 Token.split(A&: Addresses, Separator: "/");
774 uint64_t Src;
775 uint64_t Dst;
776
777 // Stop at broken LBR records.
778 if (Addresses.size() < 2 || parseAddress(Str: Addresses[0], Addr&: Src, HasPrefix: true) ||
779 parseAddress(Str: Addresses[1], Addr&: Dst, HasPrefix: true)) {
780 WarnInvalidLBR(TraceIt);
781 break;
782 }
783
784 // Canonicalize to use preferred load address as base address.
785 Src = Binary->canonicalizeVirtualAddress(Address: Src);
786 Dst = Binary->canonicalizeVirtualAddress(Address: Dst);
787 bool SrcIsInternal = Binary->addressIsCode(Address: Src);
788 bool DstIsInternal = Binary->addressIsCode(Address: Dst);
789 if (!SrcIsInternal)
790 Src = ExternalAddr;
791 if (!DstIsInternal)
792 Dst = ExternalAddr;
793 // Filter external-to-external case to reduce LBR trace size.
794 if (!SrcIsInternal && !DstIsInternal)
795 continue;
796
797 LBRStack.emplace_back(Args: LBREntry(Src, Dst));
798 }
799 TraceIt.advance();
800 return !LBRStack.empty();
801}
802
803bool PerfScriptReader::extractCallstack(TraceStream &TraceIt,
804 SmallVectorImpl<uint64_t> &CallStack) {
805 // The raw format of call stack is like:
806 // 4005dc # leaf frame
807 // 400634
808 // 400684 # root frame
809 // It's in bottom-up order with each frame in one line.
810
811 // Extract stack frames from sample
812 while (!TraceIt.isAtEoF() && !isLBRSample(Line: TraceIt.getCurrentLine(), CheckLineStart: true)) {
813 StringRef FrameStr = TraceIt.getCurrentLine().ltrim();
814 uint64_t FrameAddr = 0;
815 if (parseAddress(Str: FrameStr, Addr&: FrameAddr, HasPrefix: false)) {
816 // We might parse a non-perf sample line like empty line and comments,
817 // skip it
818 TraceIt.advance();
819 return false;
820 }
821 TraceIt.advance();
822
823 FrameAddr = Binary->canonicalizeVirtualAddress(Address: FrameAddr);
824 // Currently intermixed frame from different binaries is not supported.
825 if (!Binary->addressIsCode(Address: FrameAddr)) {
826 if (CallStack.empty())
827 NumLeafExternalFrame++;
828 // Push a special value(ExternalAddr) for the external frames so that
829 // unwinder can still work on this with artificial Call/Return branch.
830 // After unwinding, the context will be truncated for external frame.
831 // Also deduplicate the consecutive external addresses.
832 if (CallStack.empty() || CallStack.back() != ExternalAddr)
833 CallStack.emplace_back(Args: ExternalAddr);
834 continue;
835 }
836
837 // We need to translate return address to call address for non-leaf frames.
838 if (!CallStack.empty()) {
839 auto CallAddr = Binary->getCallAddrFromFrameAddr(FrameAddr);
840 if (!CallAddr) {
841 // Stop at an invalid return address caused by bad unwinding. This could
842 // happen to frame-pointer-based unwinding and the callee functions that
843 // do not have the frame pointer chain set up.
844 InvalidReturnAddresses.insert(x: FrameAddr);
845 break;
846 }
847 FrameAddr = CallAddr;
848 }
849
850 CallStack.emplace_back(Args&: FrameAddr);
851 }
852
853 // Strip out the bottom external addr.
854 if (CallStack.size() > 1 && CallStack.back() == ExternalAddr)
855 CallStack.pop_back();
856
857 // Skip other unrelated line, find the next valid LBR line
858 // Note that even for empty call stack, we should skip the address at the
859 // bottom, otherwise the following pass may generate a truncated callstack
860 while (!TraceIt.isAtEoF() && !isLBRSample(Line: TraceIt.getCurrentLine(), CheckLineStart: true)) {
861 TraceIt.advance();
862 }
863 // Filter out broken stack sample. We may not have complete frame info
864 // if sample end up in prolog/epilog, the result is dangling context not
865 // connected to entry point. This should be relatively rare thus not much
866 // impact on overall profile quality. However we do want to filter them
867 // out to reduce the number of different calling contexts. One instance
868 // of such case - when sample landed in prolog/epilog, somehow stack
869 // walking will be broken in an unexpected way that higher frames will be
870 // missing.
871 return !CallStack.empty() &&
872 !Binary->addressInPrologEpilog(Address: CallStack.front());
873}
874
875void PerfScriptReader::warnIfMissingMMap() {
876 if (!Binary->getMissingMMapWarned() && !Binary->getIsLoadedByMMap()) {
877 WithColor::warning() << "No relevant mmap event is matched for "
878 << Binary->getName()
879 << ", will use preferred address ("
880 << format(Fmt: "0x%" PRIx64,
881 Vals: Binary->getPreferredBaseAddress())
882 << ") as the base loading address!\n";
883 // Avoid redundant warning, only warn at the first unmatched sample.
884 Binary->setMissingMMapWarned(true);
885 }
886}
887
888// The unwinder requires that LBR tip belong to the leaf frame.
889// External addresses are not checked.
890static bool isValidTrace(ProfiledBinary *Binary, uint64_t StackLeaf,
891 uint64_t LBRLeaf) {
892 if (StackLeaf == ExternalAddr || LBRLeaf == ExternalAddr)
893 return true;
894 return Binary->findFuncRange(Address: LBRLeaf) == Binary->findFuncRange(Address: StackLeaf);
895}
896
897void HybridPerfReader::parseSample(TraceStream &TraceIt, uint64_t Count) {
898 // The raw hybird sample started with call stack in FILO order and followed
899 // intermediately by LBR sample
900 // e.g.
901 // 4005dc # call stack leaf
902 // 400634
903 // 400684 # call stack root
904 // 0x4005c8/0x4005dc/P/-/-/0 0x40062f/0x4005b0/P/-/-/0 ...
905 // ... 0x4005c8/0x4005dc/P/-/-/0 # LBR Entries
906 //
907 std::shared_ptr<PerfSample> Sample = std::make_shared<PerfSample>();
908#ifndef NDEBUG
909 Sample->Linenum = TraceIt.getLineNumber();
910#endif
911 // Parsing call stack and populate into PerfSample.CallStack
912 if (!extractCallstack(TraceIt, CallStack&: Sample->CallStack)) {
913 // Skip the next LBR line matched current call stack
914 if (!TraceIt.isAtEoF() && isLBRSample(Line: TraceIt.getCurrentLine(), CheckLineStart: true))
915 TraceIt.advance();
916 return;
917 }
918
919 warnIfMissingMMap();
920
921 if (!TraceIt.isAtEoF() && isLBRSample(Line: TraceIt.getCurrentLine(), CheckLineStart: true)) {
922 // Parsing LBR stack and populate into PerfSample.LBRStack
923 if (extractLBRStack(TraceIt, LBRStack&: Sample->LBRStack)) {
924 if (IgnoreStackSamples) {
925 Sample->CallStack.clear();
926 } else {
927 NumTotalHybridSample++;
928 // Drop samples whose callchain and LBR disagree before the
929 // canonicalization below hides the disagreement.
930 uint64_t StackLeaf = Sample->CallStack.front();
931 uint64_t LBRLeaf = Sample->LBRStack[0].Target;
932 if (!isValidTrace(Binary, StackLeaf, LBRLeaf)) {
933 NumBogusTrace++;
934 if (ShowDetailedWarning)
935 WithColor::warning()
936 << "Bogus trace: stack tip = " << format_hex(N: StackLeaf, Width: 10)
937 << ", LBR tip = " << format_hex(N: LBRLeaf, Width: 10) << "\n";
938 return;
939 }
940 // Canonicalize stack leaf to avoid 'random' IP from leaf frame skew LBR
941 // ranges
942 Sample->CallStack.front() = Sample->LBRStack[0].Target;
943 }
944 // Record samples by aggregation
945 AggregatedSamples[Hashable<PerfSample>(Sample)] += Count;
946 }
947 } else {
948 // LBR sample is encoded in single line after stack sample
949 exitWithError(Message: "'Hybrid perf sample is corrupted, No LBR sample line");
950 }
951}
952
953void PerfScriptReader::writeUnsymbolizedProfile(StringRef Filename) {
954 std::error_code EC;
955 raw_fd_ostream OS(Filename, EC, llvm::sys::fs::OF_TextWithCRLF);
956 if (EC)
957 exitWithError(EC, Whence: Filename);
958 writeUnsymbolizedProfile(OS);
959}
960
961// Use ordered map to make the output deterministic
962using OrderedCounterForPrint = std::map<std::string, SampleCounter *>;
963
964void PerfScriptReader::writeUnsymbolizedProfile(raw_fd_ostream &OS) {
965 OrderedCounterForPrint OrderedCounters;
966 for (auto &CI : SampleCounters) {
967 OrderedCounters[getContextKeyStr(K: CI.first.getPtr(), Binary)] = &CI.second;
968 }
969
970 auto SCounterPrinter = [&](RangeSample &Counter, StringRef Separator,
971 uint32_t Indent) {
972 OS.indent(NumSpaces: Indent);
973 OS << Counter.size() << "\n";
974 for (auto &I : Counter) {
975 uint64_t Start = I.first.first;
976 uint64_t End = I.first.second;
977
978 if (UseOffset) {
979 if (UseLoadableSegmentAsBase) {
980 Start -= Binary->getFirstLoadableAddress();
981 End -= Binary->getFirstLoadableAddress();
982 } else {
983 Start -= Binary->getPreferredBaseAddress();
984 End -= Binary->getPreferredBaseAddress();
985 }
986 }
987
988 OS.indent(NumSpaces: Indent);
989 OS << Twine::utohexstr(Val: Start) << Separator << Twine::utohexstr(Val: End) << ":"
990 << I.second << "\n";
991 }
992 };
993
994 for (auto &CI : OrderedCounters) {
995 uint32_t Indent = 0;
996 if (ProfileIsCS) {
997 // Context string key
998 OS << "[" << CI.first << "]\n";
999 Indent = 2;
1000 }
1001
1002 SampleCounter &Counter = *CI.second;
1003 SCounterPrinter(Counter.RangeCounter, "-", Indent);
1004 SCounterPrinter(Counter.BranchCounter, "->", Indent);
1005 }
1006}
1007
1008// Format of input:
1009// number of entries in RangeCounter
1010// from_1-to_1:count_1
1011// from_2-to_2:count_2
1012// ......
1013// from_n-to_n:count_n
1014// number of entries in BranchCounter
1015// src_1->dst_1:count_1
1016// src_2->dst_2:count_2
1017// ......
1018// src_n->dst_n:count_n
1019void UnsymbolizedProfileReader::readSampleCounters(TraceStream &TraceIt,
1020 SampleCounter &SCounters) {
1021 auto exitWithErrorForTraceLine = [](TraceStream &TraceIt) {
1022 std::string Msg = TraceIt.isAtEoF()
1023 ? "Invalid raw profile!"
1024 : "Invalid raw profile at line " +
1025 Twine(TraceIt.getLineNumber()).str() + ": " +
1026 TraceIt.getCurrentLine().str();
1027 exitWithError(Message: Msg);
1028 };
1029 auto ReadNumber = [&](uint64_t &Num) {
1030 if (TraceIt.isAtEoF())
1031 exitWithErrorForTraceLine(TraceIt);
1032 if (TraceIt.getCurrentLine().ltrim().getAsInteger(Radix: 10, Result&: Num))
1033 exitWithErrorForTraceLine(TraceIt);
1034 TraceIt.advance();
1035 };
1036
1037 auto ReadCounter = [&](RangeSample &Counter, StringRef Separator) {
1038 uint64_t Num = 0;
1039 ReadNumber(Num);
1040 while (Num--) {
1041 if (TraceIt.isAtEoF())
1042 exitWithErrorForTraceLine(TraceIt);
1043 StringRef Line = TraceIt.getCurrentLine().ltrim();
1044
1045 uint64_t Count = 0;
1046 auto LineSplit = Line.split(Separator: ":");
1047 if (LineSplit.second.empty() || LineSplit.second.getAsInteger(Radix: 10, Result&: Count))
1048 exitWithErrorForTraceLine(TraceIt);
1049
1050 uint64_t Source = 0;
1051 uint64_t Target = 0;
1052 auto Range = LineSplit.first.split(Separator);
1053 if (Range.second.empty() || Range.first.getAsInteger(Radix: 16, Result&: Source) ||
1054 Range.second.getAsInteger(Radix: 16, Result&: Target))
1055 exitWithErrorForTraceLine(TraceIt);
1056
1057 if (UseOffset) {
1058 if (UseLoadableSegmentAsBase) {
1059 Source += Binary->getFirstLoadableAddress();
1060 Target += Binary->getFirstLoadableAddress();
1061 } else {
1062 Source += Binary->getPreferredBaseAddress();
1063 Target += Binary->getPreferredBaseAddress();
1064 }
1065 }
1066
1067 Counter[{Source, Target}] += Count;
1068 TraceIt.advance();
1069 }
1070 };
1071
1072 ReadCounter(SCounters.RangeCounter, "-");
1073 ReadCounter(SCounters.BranchCounter, "->");
1074}
1075
1076void UnsymbolizedProfileReader::readUnsymbolizedProfile(StringRef FileName) {
1077 TraceStream TraceIt(FileName);
1078 while (!TraceIt.isAtEoF()) {
1079 std::shared_ptr<StringBasedCtxKey> Key =
1080 std::make_shared<StringBasedCtxKey>();
1081 StringRef Line = TraceIt.getCurrentLine();
1082 // Read context stack for CS profile.
1083 if (Line.starts_with(Prefix: "[")) {
1084 ProfileIsCS = true;
1085 auto I = ContextStrSet.insert(key: Line);
1086 SampleContext::createCtxVectorFromStr(ContextStr: I.first->getKey(), Context&: Key->Context);
1087 TraceIt.advance();
1088 }
1089 auto Ret = SampleCounters.try_emplace(Key: Hashable<ContextKey>(Key));
1090 readSampleCounters(TraceIt, SCounters&: Ret.first->second);
1091 }
1092}
1093
1094void UnsymbolizedProfileReader::parsePerfTraces() {
1095 readUnsymbolizedProfile(FileName: PerfTraceFile);
1096}
1097
1098void PerfScriptReader::computeCounterFromLBR(const PerfSample *Sample,
1099 uint64_t Repeat) {
1100 SampleCounter &Counter = SampleCounters.begin()->second;
1101 uint64_t EndAddress = 0;
1102 for (const LBREntry &LBR : Sample->LBRStack) {
1103 uint64_t SourceAddress = LBR.Source;
1104 uint64_t TargetAddress = LBR.Target;
1105
1106 // Record the branch if its SourceAddress is external. It can be the case an
1107 // external source call an internal function, later this branch will be used
1108 // to generate the function's head sample.
1109 if (Binary->addressIsCode(Address: TargetAddress)) {
1110 Counter.recordBranchCount(Source: SourceAddress, Target: TargetAddress, Repeat);
1111 }
1112
1113 // If this not the first LBR, update the range count between TO of current
1114 // LBR and FROM of next LBR.
1115 uint64_t StartAddress = TargetAddress;
1116 if (Binary->addressIsCode(Address: StartAddress) &&
1117 Binary->addressIsCode(Address: EndAddress) &&
1118 isValidFallThroughRange(Start: StartAddress, End: EndAddress, Binary))
1119 Counter.recordRangeCount(Start: StartAddress, End: EndAddress, Repeat);
1120 EndAddress = SourceAddress;
1121 }
1122}
1123
1124void LBRPerfReader::parseSample(TraceStream &TraceIt, uint64_t Count) {
1125 std::shared_ptr<PerfSample> Sample = std::make_shared<PerfSample>();
1126 // Parsing LBR stack and populate into PerfSample.LBRStack
1127 if (extractLBRStack(TraceIt, LBRStack&: Sample->LBRStack)) {
1128 warnIfMissingMMap();
1129 // Record LBR only samples by aggregation
1130 AggregatedSamples[Hashable<PerfSample>(Sample)] += Count;
1131 }
1132}
1133
1134void PerfScriptReader::generateUnsymbolizedProfile() {
1135 // There is no context for LBR only sample, so initialize one entry with
1136 // fake "empty" context key.
1137 assert(SampleCounters.empty() &&
1138 "Sample counter map should be empty before raw profile generation");
1139 std::shared_ptr<StringBasedCtxKey> Key =
1140 std::make_shared<StringBasedCtxKey>();
1141 SampleCounters.try_emplace(Key: Hashable<ContextKey>(Key));
1142 for (const auto &Item : AggregatedSamples) {
1143 const PerfSample *Sample = Item.first.getPtr();
1144 computeCounterFromLBR(Sample, Repeat: Item.second);
1145 }
1146}
1147
1148uint64_t PerfScriptReader::parseAggregatedCount(TraceStream &TraceIt) {
1149 // The aggregated count is optional, so do not skip the line and return 1 if
1150 // it's unmatched
1151 uint64_t Count = 1;
1152 if (!TraceIt.getCurrentLine().getAsInteger(Radix: 10, Result&: Count))
1153 TraceIt.advance();
1154 return Count;
1155}
1156
1157void PerfScriptReader::parseSample(TraceStream &TraceIt) {
1158 NumTotalSample++;
1159 uint64_t Count = parseAggregatedCount(TraceIt);
1160 assert(Count >= 1 && "Aggregated count should be >= 1!");
1161 parseSample(TraceIt, Count);
1162}
1163
1164bool PerfScriptReader::extractMMapEventForBinary(ProfiledBinary *Binary,
1165 StringRef Line,
1166 MMapEvent &MMap) {
1167 if (!Binary->isKernel() && !Line.contains(Other: Binary->getName()) &&
1168 !ShowMmapEvents)
1169 return false;
1170 // Parse a MMap2 line like:
1171 // PERF_RECORD_MMAP2 2113428/2113428: [0x7fd4efb57000(0x204000) @ 0
1172 // 08:04 19532229 3585508847]: r-xp /usr/lib64/libdl-2.17.so
1173 constexpr static const char *const MMap2Pattern =
1174 "PERF_RECORD_MMAP2 (-?[0-9]+)/[0-9]+: "
1175 "\\[(0x[a-f0-9]+)\\((0x[a-f0-9]+)\\) @ "
1176 "(0x[a-f0-9]+|0) .*\\]: ([-a-z]+) (.*)";
1177 // Parse a MMap line like
1178 // PERF_RECORD_MMAP -1/0: [0xffffffff81e00000(0x3e8fa000) @ \
1179 // 0xffffffff81e00000]: x [kernel.kallsyms]_text
1180 constexpr static const char *const MMapPattern =
1181 "PERF_RECORD_MMAP (-?[0-9]+)/[0-9]+: "
1182 "\\[(0x[a-f0-9]+)\\((0x[a-f0-9]+)\\) @ "
1183 "(0x[a-f0-9]+|0)\\]: ([-a-z]+) (.*)";
1184 // Field 0 - whole line
1185 // Field 1 - PID
1186 // Field 2 - base address
1187 // Field 3 - mmapped size
1188 // Field 4 - page offset
1189 // Field 5 - binary path
1190 enum EventIndex {
1191 WHOLE_LINE = 0,
1192 PID = 1,
1193 MMAPPED_ADDRESS = 2,
1194 MMAPPED_SIZE = 3,
1195 PAGE_OFFSET = 4,
1196 MEM_PROTECTION_FLAG = 5,
1197 BINARY_PATH = 6,
1198 };
1199
1200 bool R = false;
1201 SmallVector<StringRef, 7> Fields;
1202 if (Line.contains(Other: "PERF_RECORD_MMAP2 ")) {
1203 Regex RegMmap2(MMap2Pattern);
1204 R = RegMmap2.match(String: Line, Matches: &Fields);
1205 } else if (Line.contains(Other: "PERF_RECORD_MMAP ")) {
1206 Regex RegMmap(MMapPattern);
1207 R = RegMmap.match(String: Line, Matches: &Fields);
1208 } else
1209 llvm_unreachable("unexpected MMAP event entry");
1210
1211 if (!R) {
1212 std::string WarningMsg = "Cannot parse mmap event: " + Line.str() + " \n";
1213 WithColor::warning() << WarningMsg;
1214 return false;
1215 }
1216 long long MMapPID = 0;
1217 getAsSignedInteger(Str: Fields[PID], Radix: 10, Result&: MMapPID);
1218 MMap.PID = MMapPID;
1219 Fields[MMAPPED_ADDRESS].getAsInteger(Radix: 0, Result&: MMap.Address);
1220 Fields[MMAPPED_SIZE].getAsInteger(Radix: 0, Result&: MMap.Size);
1221 Fields[PAGE_OFFSET].getAsInteger(Radix: 0, Result&: MMap.Offset);
1222 MMap.MemProtectionFlag = Fields[MEM_PROTECTION_FLAG];
1223 MMap.BinaryPath = Fields[BINARY_PATH];
1224 if (ShowMmapEvents) {
1225 outs() << "Mmap: Binary " << MMap.BinaryPath << " loaded at "
1226 << format(Fmt: "0x%" PRIx64 ":", Vals: MMap.Address) << " \n";
1227 }
1228
1229 StringRef BinaryName = filename(Path: MMap.BinaryPath, UseBackSlash: Binary->isCOFF());
1230 if (Binary->isKernel()) {
1231 return Binary->isKernelImageName(BinaryName);
1232 }
1233 return Binary->getName() == BinaryName;
1234}
1235
1236void PerfScriptReader::parseMMapEvent(TraceStream &TraceIt) {
1237 MMapEvent MMap;
1238 if (extractMMapEventForBinary(Binary, Line: TraceIt.getCurrentLine(), MMap))
1239 updateBinaryAddress(Event: MMap);
1240 TraceIt.advance();
1241}
1242
1243void PerfScriptReader::parseEventOrSample(TraceStream &TraceIt) {
1244 if (isMMapEvent(Line: TraceIt.getCurrentLine()))
1245 parseMMapEvent(TraceIt);
1246 else
1247 parseSample(TraceIt);
1248}
1249
1250void PerfScriptReader::parseAndAggregateTrace() {
1251 NamedRegionTimer T("parseTrace", "Parse and aggregate trace", TimerGroupName,
1252 TimerGroupDesc, TimeProfGen);
1253 // Trace line iterator
1254 TraceStream TraceIt(PerfTraceFile);
1255 while (!TraceIt.isAtEoF())
1256 parseEventOrSample(TraceIt);
1257}
1258
1259// A LBR sample is like:
1260// 40062f 0x5c6313f/0x5c63170/P/-/-/0 0x5c630e7/0x5c63130/P/-/-/0 ...
1261// A heuristic for fast detection by checking whether a
1262// leading " 0x" and the '/' exist.
1263bool PerfScriptReader::isLBRSample(StringRef Line, bool CheckLineStart) {
1264 // Skip the leading instruction pointer
1265 SmallVector<StringRef, 32> Records;
1266 if (!CheckLineStart)
1267 Line = Line.trim();
1268 // Line might start with IP or only contain brstack. Check first two records
1269 // and fail if no record exists.
1270 Line.split(A&: Records, Separator: " ", MaxSplit: 2, KeepEmpty: CheckLineStart);
1271 for (StringRef Record : Records)
1272 if (Record.starts_with(Prefix: "0x") && Record.contains(C: '/'))
1273 return true;
1274 return false;
1275}
1276
1277bool PerfScriptReader::isMMapEvent(StringRef Line) {
1278 // Short cut to avoid string find is possible.
1279 if (Line.empty() || Line.size() < 50)
1280 return false;
1281
1282 if (std::isdigit(Line[0]))
1283 return false;
1284
1285 // PERF_RECORD_MMAP2 or PERF_RECORD_MMAP does not appear at the beginning of
1286 // the line for ` perf script --show-mmap-events -i ...`
1287 return Line.contains(Other: "PERF_RECORD_MMAP");
1288}
1289
1290// The raw hybird sample is like
1291// e.g.
1292// 4005dc # call stack leaf
1293// 400634
1294// 400684 # call stack root
1295// 0x4005c8/0x4005dc/P/-/-/0 0x40062f/0x4005b0/P/-/-/0 ...
1296// ... 0x4005c8/0x4005dc/P/-/-/0 # LBR Entries
1297// Determine the perfscript contains hybrid samples(call stack + LBRs) by
1298// checking whether there is a non-empty call stack immediately followed by
1299// a LBR sample
1300PerfContent PerfScriptReader::checkPerfScriptType(StringRef FileName) {
1301 TraceStream TraceIt(FileName);
1302 uint64_t FrameAddr = 0;
1303 while (!TraceIt.isAtEoF()) {
1304 // Skip the aggregated count
1305 if (!TraceIt.getCurrentLine().getAsInteger(Radix: 10, Result&: FrameAddr))
1306 TraceIt.advance();
1307
1308 // Detect sample with call stack
1309 int32_t Count = 0;
1310 while (!TraceIt.isAtEoF() &&
1311 !isLBRSample(Line: TraceIt.getCurrentLine(), CheckLineStart: false) &&
1312 !parseAddress(Str: TraceIt.getCurrentLine().ltrim(), Addr&: FrameAddr, HasPrefix: false)) {
1313 Count++;
1314 TraceIt.advance();
1315 }
1316 if (!TraceIt.isAtEoF()) {
1317 if (isLBRSample(Line: TraceIt.getCurrentLine(), CheckLineStart: false)) {
1318 if (Count > 0)
1319 return PerfContent::LBRStack;
1320 else
1321 return PerfContent::LBR;
1322 }
1323 TraceIt.advance();
1324 }
1325 }
1326
1327 exitWithError(Message: "Invalid perf script input!");
1328 return PerfContent::UnknownContent;
1329}
1330
1331void HybridPerfReader::generateUnsymbolizedProfile() {
1332 ProfileIsCS = !IgnoreStackSamples;
1333 if (ProfileIsCS)
1334 unwindSamples();
1335 else
1336 PerfScriptReader::generateUnsymbolizedProfile();
1337}
1338
1339void PerfScriptReader::warnTruncatedStack() {
1340 if (ShowDetailedWarning) {
1341 for (auto Address : InvalidReturnAddresses) {
1342 WithColor::warning()
1343 << "Truncated stack sample due to invalid return address at "
1344 << format(Fmt: "0x%" PRIx64, Vals: Address)
1345 << ", likely caused by frame pointer omission\n";
1346 }
1347 }
1348 emitWarningSummary(
1349 Num: InvalidReturnAddresses.size(), Total: AggregatedSamples.size(),
1350 Msg: "of truncated stack samples due to invalid return address, "
1351 "likely caused by frame pointer omission.");
1352}
1353
1354void PerfScriptReader::warnInvalidRange() {
1355 DenseMap<std::pair<uint64_t, uint64_t>, uint64_t> Ranges;
1356
1357 for (const auto &Item : AggregatedSamples) {
1358 const PerfSample *Sample = Item.first.getPtr();
1359 uint64_t Count = Item.second;
1360 uint64_t EndAddress = 0;
1361 for (const LBREntry &LBR : Sample->LBRStack) {
1362 uint64_t SourceAddress = LBR.Source;
1363 uint64_t StartAddress = LBR.Target;
1364 if (EndAddress != 0)
1365 Ranges[{StartAddress, EndAddress}] += Count;
1366 EndAddress = SourceAddress;
1367 }
1368 }
1369
1370 if (Ranges.empty()) {
1371 WithColor::warning() << "No samples in perf script!\n";
1372 return;
1373 }
1374
1375 auto WarnInvalidRange = [&](uint64_t StartAddress, uint64_t EndAddress,
1376 StringRef Msg) {
1377 if (!ShowDetailedWarning)
1378 return;
1379 WithColor::warning() << "[" << format(Fmt: "%8" PRIx64, Vals: StartAddress) << ","
1380 << format(Fmt: "%8" PRIx64, Vals: EndAddress) << "]: " << Msg
1381 << "\n";
1382 };
1383
1384 const char *EndNotBoundaryMsg = "Range is not on instruction boundary, "
1385 "likely due to profile and binary mismatch.";
1386 const char *DanglingRangeMsg = "Range does not belong to any functions, "
1387 "likely from PLT, .init or .fini section.";
1388 const char *RangeCrossFuncMsg =
1389 "Fall through range should not cross function boundaries, likely due to "
1390 "profile and binary mismatch.";
1391 const char *BogusRangeMsg = "Range start is after or too far from range end.";
1392
1393 uint64_t TotalRangeNum = 0;
1394 uint64_t InstNotBoundary = 0;
1395 uint64_t UnmatchedRange = 0;
1396 uint64_t RecoveredRange = 0;
1397 uint64_t RangeCrossFunc = 0;
1398 uint64_t BogusRange = 0;
1399
1400 for (auto &I : Ranges) {
1401 uint64_t StartAddress = I.first.first;
1402 uint64_t EndAddress = I.first.second;
1403 TotalRangeNum += I.second;
1404
1405 if (!Binary->addressIsCode(Address: StartAddress) &&
1406 !Binary->addressIsCode(Address: EndAddress))
1407 continue;
1408
1409 if (!Binary->addressIsCode(Address: StartAddress) ||
1410 !Binary->addressIsTransfer(Address: EndAddress)) {
1411 InstNotBoundary += I.second;
1412 WarnInvalidRange(StartAddress, EndAddress, EndNotBoundaryMsg);
1413 }
1414
1415 auto *FRange = Binary->findFuncRange(Address: StartAddress);
1416 if (!FRange) {
1417 UnmatchedRange += I.second;
1418 WarnInvalidRange(StartAddress, EndAddress, DanglingRangeMsg);
1419 continue;
1420 }
1421
1422 if (FRange->Func->NameStatus != DwarfNameStatus::Matched)
1423 RecoveredRange += I.second;
1424
1425 if (EndAddress >= FRange->EndAddress) {
1426 RangeCrossFunc += I.second;
1427 WarnInvalidRange(StartAddress, EndAddress, RangeCrossFuncMsg);
1428 }
1429
1430 if (Binary->addressIsCode(Address: StartAddress) &&
1431 Binary->addressIsCode(Address: EndAddress) &&
1432 !isValidFallThroughRange(Start: StartAddress, End: EndAddress, Binary)) {
1433 BogusRange += I.second;
1434 WarnInvalidRange(StartAddress, EndAddress, BogusRangeMsg);
1435 }
1436 }
1437
1438 emitWarningSummary(
1439 Num: InstNotBoundary, Total: TotalRangeNum,
1440 Msg: "of samples are from ranges that are not on instruction boundary.");
1441 emitWarningSummary(
1442 Num: UnmatchedRange, Total: TotalRangeNum,
1443 Msg: "of samples are from ranges that do not belong to any functions.");
1444 emitWarningSummary(Num: RecoveredRange, Total: TotalRangeNum,
1445 Msg: "of samples are from ranges that belong to functions "
1446 "recovered from symbol table.");
1447 emitWarningSummary(
1448 Num: RangeCrossFunc, Total: TotalRangeNum,
1449 Msg: "of samples are from ranges that do cross function boundaries.");
1450 emitWarningSummary(
1451 Num: BogusRange, Total: TotalRangeNum,
1452 Msg: "of samples are from ranges that have range start after or too far from "
1453 "range end acrossing the unconditinal jmp.");
1454}
1455
1456void PerfScriptReader::warnIfBranchTargetMismatch() {
1457 // Collect unique branch source and target addresses from LBR samples,
1458 // then check what percentage don't match known instructions in the binary.
1459
1460 uint64_t MismatchedBranches = 0;
1461 uint64_t MismatchedIndirectTargets = 0;
1462 uint64_t MismatchedTargets = 0;
1463 uint64_t TotalSamples = 0;
1464
1465 for (const auto &Item : AggregatedSamples) {
1466 const PerfSample *Sample = Item.first.getPtr();
1467 for (const LBREntry &LBR : Sample->LBRStack) {
1468 uint64_t Source = LBR.Source;
1469 uint64_t Target = LBR.Target;
1470 if (Source == ExternalAddr || Target == ExternalAddr)
1471 continue;
1472 TotalSamples++;
1473
1474 // Validate Branch sources are Call/Branch/Indirect Branch
1475 if (!Binary->addressIsTransfer(Address: Source))
1476 MismatchedBranches++;
1477
1478 // Validate Indirect Branch targets landed in code. This may over estimate
1479 // the vaid targets only because there's no good way to determine jump
1480 // table targets
1481 if (Binary->addressIsIndirectBranch(Address: Source)) {
1482 if (!Binary->addressIsCode(Address: Target))
1483 MismatchedIndirectTargets++;
1484 } else if (!Binary->addressIsBranchTarget(Address: Target) &&
1485 !Binary->findFuncRangeForStartAddr(Address: Target))
1486 MismatchedTargets++;
1487 }
1488 }
1489
1490 emitWarningSummary(Num: MismatchedBranches, Total: TotalSamples,
1491 Msg: "of branch samples do not match the binary.");
1492 emitWarningSummary(Num: MismatchedTargets, Total: TotalSamples,
1493 Msg: "of branch targets do not match the binary.");
1494 emitWarningSummary(Num: MismatchedIndirectTargets, Total: TotalSamples,
1495 Msg: "of indirect branch targets do not match the binary.");
1496}
1497
1498void PerfScriptReader::parsePerfTraces() {
1499 // Parse perf traces and do aggregation.
1500 parseAndAggregateTrace();
1501 if (Binary->isKernel() && !Binary->getIsLoadedByMMap()) {
1502 exitWithError(
1503 Message: "Kernel is requested, but no kernel is found in mmap events.");
1504 }
1505
1506 emitWarningSummary(Num: NumLeafExternalFrame, Total: NumTotalSample,
1507 Msg: "of samples have leaf external frame in call stack.");
1508 emitWarningSummary(Num: NumLeadingOutgoingLBR, Total: NumTotalSample,
1509 Msg: "of samples have leading external LBR.");
1510
1511 // Generate unsymbolized profile.
1512 warnTruncatedStack();
1513 warnInvalidRange();
1514 warnIfBranchTargetMismatch();
1515 generateUnsymbolizedProfile();
1516 AggregatedSamples.clear();
1517
1518 if (SkipSymbolization)
1519 writeUnsymbolizedProfile(Filename: OutputFilename);
1520}
1521
1522SmallVector<CleanupInstaller, 2> PerfScriptReader::TempFileCleanups;
1523
1524void ETMReader::recordProcessedRange(uint64_t Start, uint64_t End,
1525 uint64_t Count) {
1526 assert(!Counters.empty() && "Counters should not be empty!");
1527 auto &Counter = Counters.begin()->second;
1528 Counter.recordRangeCount(Start, End, Repeat: Count);
1529}
1530
1531class ETMCallback : public ETMDecoder::Callback {
1532 ETMReader *Reader;
1533
1534public:
1535 ETMCallback(ETMReader *R) : Reader(R) {}
1536 void processInstructionRange(uint64_t Start, uint64_t End) override {
1537 Reader->recordProcessedRange(Start, End, Count: 1);
1538 }
1539};
1540
1541void ETMReader::parseETMTraces() {
1542 auto BufferOrErr = MemoryBuffer::getFile(Filename: TraceFile);
1543 if (std::error_code EC = BufferOrErr.getError())
1544 exitWithError(Message: "Could not open ETM trace file: " + EC.message());
1545
1546 ArrayRef<uint8_t> Data(
1547 reinterpret_cast<const uint8_t *>((*BufferOrErr)->getBufferStart()),
1548 (*BufferOrErr)->getBufferSize());
1549
1550 // There is no context for ETM instruction traces.
1551 // Initialize the SampleCounters map with a single empty context key
1552 // to aggregate all instruction hits into a global bucket.
1553 auto Key = std::make_shared<StringBasedCtxKey>();
1554 Counters.try_emplace(Key: Hashable<ContextKey>(Key));
1555
1556 // The protocol utilizes a 0x80 byte as an initial synchronization header.
1557 // Perform a manual search for this sync point to discard any leading
1558 // padding or truncated packets before decoding begins.
1559 size_t StartIdx = 0;
1560 while (StartIdx < Data.size() && Data[StartIdx] != 0x80)
1561 StartIdx++;
1562 if (StartIdx >= Data.size())
1563 exitWithError(Message: "No synchronization header (0x80) found in the bitstream.");
1564 ArrayRef<uint8_t> TraceSlice = Data.slice(N: StartIdx);
1565
1566 auto DecoderOrErr = ETMDecoder::create(
1567 Binary: Binary->getBinary(), TargetTriple: Binary->getTriple(), TraceID: static_cast<uint8_t>(TraceID));
1568
1569 if (!DecoderOrErr)
1570 exitWithError(Message: toString(E: DecoderOrErr.takeError()));
1571 auto Decoder = std::move(*DecoderOrErr);
1572
1573 ETMCallback CB(this);
1574 if (Error E = Decoder->processTrace(TraceData: TraceSlice, TraceCallback&: CB))
1575 exitWithError(Message: toString(E: std::move(E)));
1576}
1577
1578} // end namespace sampleprof
1579} // end namespace llvm
1580