1//===- llvm-profdata.cpp - LLVM profile data tool -------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// llvm-profdata merges .profdata files.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/ADT/DenseMap.h"
14#include "llvm/ADT/ScopeExit.h"
15#include "llvm/ADT/SmallSet.h"
16#include "llvm/ADT/SmallVector.h"
17#include "llvm/ADT/StringRef.h"
18#include "llvm/HTTP/HTTPClient.h"
19#include "llvm/IR/LLVMContext.h"
20#include "llvm/Object/Binary.h"
21#include "llvm/ProfileData/DataAccessProf.h"
22#include "llvm/ProfileData/InstrProfCorrelator.h"
23#include "llvm/ProfileData/InstrProfReader.h"
24#include "llvm/ProfileData/InstrProfWriter.h"
25#include "llvm/ProfileData/MemProf.h"
26#include "llvm/ProfileData/MemProfReader.h"
27#include "llvm/ProfileData/MemProfSummaryBuilder.h"
28#include "llvm/ProfileData/MemProfYAML.h"
29#include "llvm/ProfileData/ProfileCommon.h"
30#include "llvm/ProfileData/SampleProfReader.h"
31#include "llvm/ProfileData/SampleProfWriter.h"
32#include "llvm/Support/BalancedPartitioning.h"
33#include "llvm/Support/CommandLine.h"
34#include "llvm/Support/Discriminator.h"
35#include "llvm/Support/Errc.h"
36#include "llvm/Support/FileSystem.h"
37#include "llvm/Support/Format.h"
38#include "llvm/Support/FormattedStream.h"
39#include "llvm/Support/InitLLVM.h"
40#include "llvm/Support/MD5.h"
41#include "llvm/Support/MemoryBuffer.h"
42#include "llvm/Support/Path.h"
43#include "llvm/Support/Regex.h"
44#include "llvm/Support/ThreadPool.h"
45#include "llvm/Support/Threading.h"
46#include "llvm/Support/VirtualFileSystem.h"
47#include "llvm/Support/WithColor.h"
48#include "llvm/Support/raw_ostream.h"
49#include <algorithm>
50#include <cmath>
51#include <optional>
52
53using namespace llvm;
54
55// https://llvm.org/docs/CommandGuide/llvm-profdata.html has documentations
56// on each subcommand.
57cl::SubCommand ShowSubcommand(
58 "show",
59 "Takes a profile data file and displays the profiles. See detailed "
60 "documentation in "
61 "https://llvm.org/docs/CommandGuide/llvm-profdata.html#profdata-show");
62cl::SubCommand OrderSubcommand(
63 "order",
64 "Reads temporal profiling traces from a profile and outputs a function "
65 "order that reduces the number of page faults for those traces. See "
66 "detailed documentation in "
67 "https://llvm.org/docs/CommandGuide/llvm-profdata.html#profdata-order");
68cl::SubCommand OverlapSubcommand(
69 "overlap",
70 "Computes and displays the overlap between two profiles. See detailed "
71 "documentation in "
72 "https://llvm.org/docs/CommandGuide/llvm-profdata.html#profdata-overlap");
73cl::SubCommand MergeSubcommand(
74 "merge",
75 "Takes several profiles and merge them together. See detailed "
76 "documentation in "
77 "https://llvm.org/docs/CommandGuide/llvm-profdata.html#profdata-merge");
78
79namespace {
80enum ProfileKinds { instr, sample, memory };
81enum FailureMode { warnOnly, failIfAnyAreInvalid, failIfAllAreInvalid };
82
83enum ProfileFormat {
84 PF_None = 0,
85 PF_Text,
86 PF_Compact_Binary, // Deprecated
87 PF_Ext_Binary,
88 PF_GCC,
89 PF_Binary
90};
91
92enum class ShowFormat { Text, Json, Yaml };
93} // namespace
94
95// Common options.
96cl::opt<std::string> OutputFilename("output", cl::value_desc("output"),
97 cl::init(Val: "-"), cl::desc("Output file"),
98 cl::sub(ShowSubcommand),
99 cl::sub(OrderSubcommand),
100 cl::sub(OverlapSubcommand),
101 cl::sub(MergeSubcommand));
102// NOTE: cl::alias must not have cl::sub(), since aliased option's cl::sub()
103// will be used. llvm::cl::alias::done() method asserts this condition.
104static cl::alias OutputFilenameA("o", cl::desc("Alias for --output"),
105 cl::aliasopt(OutputFilename));
106
107// Options common to at least two commands.
108static cl::opt<ProfileKinds> ProfileKind(
109 cl::desc("Profile kind:"), cl::sub(MergeSubcommand),
110 cl::sub(OverlapSubcommand), cl::init(Val: instr),
111 cl::values(clEnumVal(instr, "Instrumentation profile (default)"),
112 clEnumVal(sample, "Sample profile")));
113static cl::opt<std::string> Filename(cl::Positional,
114 cl::desc("<profdata-file>"),
115 cl::sub(ShowSubcommand),
116 cl::sub(OrderSubcommand));
117static cl::opt<unsigned> MaxDbgCorrelationWarnings(
118 "max-debug-info-correlation-warnings",
119 cl::desc("The maximum number of warnings to emit when correlating "
120 "profile from debug info (0 = no limit)"),
121 cl::sub(MergeSubcommand), cl::sub(ShowSubcommand), cl::init(Val: 5));
122static cl::opt<std::string> ProfiledBinary(
123 "profiled-binary", cl::init(Val: ""),
124 cl::desc("Path to binary from which the profile was collected."),
125 cl::sub(ShowSubcommand), cl::sub(MergeSubcommand));
126static cl::opt<std::string> DebugInfoFilename(
127 "debug-info", cl::init(Val: ""),
128 cl::desc(
129 "For show, read and extract profile metadata from debug info and show "
130 "the functions it found. For merge, use the provided debug info to "
131 "correlate the raw profile."),
132 cl::sub(ShowSubcommand), cl::sub(MergeSubcommand));
133static cl::opt<std::string>
134 BinaryFilename("binary-file", cl::init(Val: ""),
135 cl::desc("For merge, use the provided unstripped binary to "
136 "correlate the raw profile."),
137 cl::sub(MergeSubcommand));
138static cl::list<std::string> DebugFileDirectory(
139 "debug-file-directory",
140 cl::desc("Directories to search for object files by build ID"));
141static cl::opt<bool> DebugInfod("debuginfod", cl::init(Val: false), cl::Hidden,
142 cl::sub(MergeSubcommand),
143 cl::desc("Enable debuginfod"));
144static cl::opt<ProfCorrelatorKind> BIDFetcherProfileCorrelate(
145 "correlate",
146 cl::desc("Use debug-info or binary correlation to correlate profiles with "
147 "build id fetcher"),
148 cl::values(clEnumValN(ProfCorrelatorKind::DebugInfo, "debug-info",
149 "Use debug info to correlate"),
150 clEnumValN(ProfCorrelatorKind::Binary, "binary",
151 "Use binary to correlate")));
152static cl::opt<std::string> FuncNameFilter(
153 "function",
154 cl::desc("Only functions matching the filter are shown in the output. For "
155 "overlapping CSSPGO, this takes a function name with calling "
156 "context."),
157 cl::sub(ShowSubcommand), cl::sub(OverlapSubcommand),
158 cl::sub(MergeSubcommand));
159
160// TODO: Consider creating a template class (e.g., MergeOption, ShowOption) to
161// factor out the common cl::sub in cl::opt constructor for subcommand-specific
162// options.
163
164// Options specific to merge subcommand.
165static cl::list<std::string> InputFilenames(cl::Positional,
166 cl::sub(MergeSubcommand),
167 cl::desc("<filename...>"));
168static cl::list<std::string>
169 WeightedInputFilenames("weighted-input", cl::sub(MergeSubcommand),
170 cl::desc("<weight>,<filename>"));
171static cl::opt<ProfileFormat> OutputFormat(
172 cl::desc("Format of output profile"), cl::sub(MergeSubcommand),
173 cl::init(Val: PF_Ext_Binary),
174 cl::values(clEnumValN(PF_Binary, "binary", "Binary encoding"),
175 clEnumValN(PF_Ext_Binary, "extbinary",
176 "Extensible binary encoding "
177 "(default)"),
178 clEnumValN(PF_Text, "text", "Text encoding"),
179 clEnumValN(PF_GCC, "gcc",
180 "GCC encoding (only meaningful for -sample)")));
181static cl::opt<std::string>
182 InputFilenamesFile("input-files", cl::init(Val: ""), cl::sub(MergeSubcommand),
183 cl::desc("Path to file containing newline-separated "
184 "[<weight>,]<filename> entries"));
185static cl::alias InputFilenamesFileA("f", cl::desc("Alias for --input-files"),
186 cl::aliasopt(InputFilenamesFile));
187static cl::opt<bool> DumpInputFileList(
188 "dump-input-file-list", cl::init(Val: false), cl::Hidden,
189 cl::sub(MergeSubcommand),
190 cl::desc("Dump the list of input files and their weights, then exit"));
191static cl::opt<std::string> RemappingFile("remapping-file",
192 cl::value_desc("file"),
193 cl::sub(MergeSubcommand),
194 cl::desc("Symbol remapping file"));
195static cl::alias RemappingFileA("r", cl::desc("Alias for --remapping-file"),
196 cl::aliasopt(RemappingFile));
197static cl::opt<bool>
198 UseMD5("use-md5", cl::init(Val: false), cl::Hidden,
199 cl::desc("Choose to use MD5 to represent string in name table (only "
200 "meaningful for -extbinary)"),
201 cl::sub(MergeSubcommand));
202static cl::opt<bool> CompressAllSections(
203 "compress-all-sections", cl::init(Val: false), cl::Hidden,
204 cl::sub(MergeSubcommand),
205 cl::desc("Compress all sections when writing the profile (only "
206 "meaningful for -extbinary)"));
207static cl::opt<bool> SampleMergeColdContext(
208 "sample-merge-cold-context", cl::init(Val: false), cl::Hidden,
209 cl::sub(MergeSubcommand),
210 cl::desc(
211 "Merge context sample profiles whose count is below cold threshold"));
212static cl::opt<bool> SampleTrimColdContext(
213 "sample-trim-cold-context", cl::init(Val: false), cl::Hidden,
214 cl::sub(MergeSubcommand),
215 cl::desc(
216 "Trim context sample profiles whose count is below cold threshold"));
217static cl::opt<uint32_t> SampleColdContextFrameDepth(
218 "sample-frame-depth-for-cold-context", cl::init(Val: 1),
219 cl::sub(MergeSubcommand),
220 cl::desc("Keep the last K frames while merging cold profile. 1 means the "
221 "context-less base profile"));
222static cl::opt<size_t> OutputSizeLimit(
223 "output-size-limit", cl::init(Val: 0), cl::Hidden, cl::sub(MergeSubcommand),
224 cl::desc("Trim cold functions until profile size is below specified "
225 "limit in bytes. This uses a heursitic and functions may be "
226 "excessively trimmed"));
227static cl::opt<bool> GenPartialProfile(
228 "gen-partial-profile", cl::init(Val: false), cl::Hidden,
229 cl::sub(MergeSubcommand),
230 cl::desc("Generate a partial profile (only meaningful for -extbinary)"));
231static cl::opt<bool> SplitLayout(
232 "split-layout", cl::init(Val: false), cl::Hidden, cl::sub(MergeSubcommand),
233 cl::desc("Split the profile to two sections with one containing sample "
234 "profiles with inlined functions and the other without (only "
235 "meaningful for -extbinary)"));
236static cl::opt<bool>
237 WriteMD5ProfSymList("md5-prof-sym-list", cl::init(Val: false), cl::Hidden,
238 cl::sub(MergeSubcommand),
239 cl::desc("Write ProfileSymbolList (Cold Symbols) as "
240 "64-bit MD5 hashes in Eytzinger layout"));
241static cl::opt<bool> WriteMD5IndexedTables(
242 "md5-indexed-tables", cl::init(Val: false), cl::Hidden, cl::sub(MergeSubcommand),
243 cl::desc("Write MD5-based indexed NameTable and parallel "
244 "FuncOffsetTable in Eytzinger layout (only meaningful for "
245 "-extbinary)"));
246static cl::opt<std::string> SupplInstrWithSample(
247 "supplement-instr-with-sample", cl::init(Val: ""), cl::Hidden,
248 cl::sub(MergeSubcommand),
249 cl::desc("Supplement an instr profile with sample profile, to correct "
250 "the profile unrepresentativeness issue. The sample "
251 "profile is the input of the flag. Output will be in instr "
252 "format (The flag only works with -instr)"));
253static cl::opt<float> ZeroCounterThreshold(
254 "zero-counter-threshold", cl::init(Val: 0.7), cl::Hidden,
255 cl::sub(MergeSubcommand),
256 cl::desc("For the function which is cold in instr profile but hot in "
257 "sample profile, if the ratio of the number of zero counters "
258 "divided by the total number of counters is above the "
259 "threshold, the profile of the function will be regarded as "
260 "being harmful for performance and will be dropped."));
261static cl::opt<unsigned> SupplMinSizeThreshold(
262 "suppl-min-size-threshold", cl::init(Val: 10), cl::Hidden,
263 cl::sub(MergeSubcommand),
264 cl::desc("If the size of a function is smaller than the threshold, "
265 "assume it can be inlined by PGO early inliner and it won't "
266 "be adjusted based on sample profile."));
267static cl::opt<unsigned> InstrProfColdThreshold(
268 "instr-prof-cold-threshold", cl::init(Val: 0), cl::Hidden,
269 cl::sub(MergeSubcommand),
270 cl::desc("User specified cold threshold for instr profile which will "
271 "override the cold threshold got from profile summary. "));
272// WARNING: This reservoir size value is propagated to any input indexed
273// profiles for simplicity. Changing this value between invocations could
274// result in sample bias.
275static cl::opt<uint64_t> TemporalProfTraceReservoirSize(
276 "temporal-profile-trace-reservoir-size", cl::init(Val: 100),
277 cl::sub(MergeSubcommand),
278 cl::desc("The maximum number of stored temporal profile traces (default: "
279 "100)"));
280static cl::opt<uint64_t> TemporalProfMaxTraceLength(
281 "temporal-profile-max-trace-length", cl::init(Val: 10000),
282 cl::sub(MergeSubcommand),
283 cl::desc("The maximum length of a single temporal profile trace "
284 "(default: 10000)"));
285static cl::opt<std::string> FuncNameNegativeFilter(
286 "no-function", cl::init(Val: ""), cl::sub(MergeSubcommand),
287 cl::desc("Exclude functions matching the filter from the output."));
288
289static cl::opt<FailureMode>
290 FailMode("failure-mode", cl::init(Val: failIfAnyAreInvalid),
291 cl::desc("Failure mode:"), cl::sub(MergeSubcommand),
292 cl::values(clEnumValN(warnOnly, "warn",
293 "Do not fail and just print warnings."),
294 clEnumValN(failIfAnyAreInvalid, "any",
295 "Fail if any profile is invalid."),
296 clEnumValN(failIfAllAreInvalid, "all",
297 "Fail only if all profiles are invalid.")));
298
299static cl::opt<bool> OutputSparse(
300 "sparse", cl::init(Val: false), cl::sub(MergeSubcommand),
301 cl::desc("Generate a sparse profile (only meaningful for -instr)"));
302static cl::opt<unsigned> NumThreads(
303 "num-threads", cl::init(Val: 0), cl::sub(MergeSubcommand),
304 cl::desc("Number of merge threads to use (default: autodetect)"));
305static cl::alias NumThreadsA("j", cl::desc("Alias for --num-threads"),
306 cl::aliasopt(NumThreads));
307
308static cl::opt<std::string> ProfileSymbolListFile(
309 "prof-sym-list", cl::init(Val: ""), cl::sub(MergeSubcommand),
310 cl::desc("Path to file containing the list of function symbols "
311 "used to populate profile symbol list"));
312
313static cl::opt<SampleProfileLayout> ProfileLayout(
314 "convert-sample-profile-layout",
315 cl::desc("Convert the generated profile to a profile with a new layout"),
316 cl::sub(MergeSubcommand), cl::init(Val: SPL_None),
317 cl::values(
318 clEnumValN(SPL_Nest, "nest",
319 "Nested profile, the input should be CS flat profile"),
320 clEnumValN(SPL_Flat, "flat",
321 "Profile with nested inlinee flatten out")));
322
323static cl::opt<bool> DropProfileSymbolList(
324 "drop-profile-symbol-list", cl::init(Val: false), cl::Hidden,
325 cl::sub(MergeSubcommand),
326 cl::desc("Drop the profile symbol list when merging AutoFDO profiles "
327 "(only meaningful for -sample)"));
328
329static cl::opt<bool> KeepVTableSymbols(
330 "keep-vtable-symbols", cl::init(Val: false), cl::Hidden,
331 cl::sub(MergeSubcommand),
332 cl::desc("If true, keep the vtable symbols in indexed profiles"));
333
334// Temporary support for writing the previous version of the format, to enable
335// some forward compatibility.
336// TODO: Consider enabling this with future version changes as well, to ease
337// deployment of newer versions of llvm-profdata.
338static cl::opt<bool> DoWritePrevVersion(
339 "write-prev-version", cl::init(Val: false), cl::Hidden,
340 cl::desc("Write the previous version of indexed format, to enable "
341 "some forward compatibility."));
342
343static cl::opt<memprof::IndexedVersion> MemProfVersionRequested(
344 "memprof-version", cl::Hidden, cl::sub(MergeSubcommand),
345 cl::desc("Specify the version of the memprof format to use"),
346 cl::init(Val: memprof::Version3),
347 cl::values(clEnumValN(memprof::Version3, "3", "version 3"),
348 clEnumValN(memprof::Version4, "4", "version 4")));
349
350static cl::opt<bool> MemProfFullSchema(
351 "memprof-full-schema", cl::Hidden, cl::sub(MergeSubcommand),
352 cl::desc("Use the full schema for serialization"), cl::init(Val: false));
353
354static cl::opt<bool> MemprofGenerateRandomHotness(
355 "memprof-random-hotness", cl::init(Val: false), cl::Hidden,
356 cl::sub(MergeSubcommand),
357 cl::desc("Generate random hotness values. Use -random-seed to set the seed "
358 "value, otherwise the constant default seed is used"));
359static cl::opt<unsigned>
360 RandomSeed("random-seed", cl::init(Val: 0), cl::Hidden, cl::sub(MergeSubcommand),
361 cl::desc("Seed for the random number generator used by "
362 "-memprof-random-hotness and temporal profile "
363 "reservoir sampling"));
364static cl::alias MemprofGenerateRandomHotnessSeed(
365 "memprof-random-hotness-seed", cl::Hidden,
366 cl::desc("Alias for -random-seed. Deprecated, please use -random-seed"),
367 cl::aliasopt(RandomSeed));
368
369// Options specific to overlap subcommand.
370static cl::opt<std::string> BaseFilename(cl::Positional, cl::Required,
371 cl::desc("<base profile file>"),
372 cl::sub(OverlapSubcommand));
373static cl::opt<std::string> TestFilename(cl::Positional, cl::Required,
374 cl::desc("<test profile file>"),
375 cl::sub(OverlapSubcommand));
376
377static cl::opt<unsigned long long> SimilarityCutoff(
378 "similarity-cutoff", cl::init(Val: 0),
379 cl::desc("For sample profiles, list function names (with calling context "
380 "for csspgo) for overlapped functions "
381 "with similarities below the cutoff (percentage times 10000)."),
382 cl::sub(OverlapSubcommand));
383
384static cl::opt<bool> IsCS(
385 "cs", cl::init(Val: false),
386 cl::desc("For context sensitive PGO counts. Does not work with CSSPGO."),
387 cl::sub(OverlapSubcommand));
388
389static cl::opt<unsigned long long> OverlapValueCutoff(
390 "value-cutoff", cl::init(Val: -1),
391 cl::desc(
392 "Function level overlap information for every function (with calling "
393 "context for csspgo) in test "
394 "profile with max count value greater than the parameter value"),
395 cl::sub(OverlapSubcommand));
396
397// Options specific to show subcommand.
398static cl::opt<bool>
399 ShowCounts("counts", cl::init(Val: false),
400 cl::desc("Show counter values for shown functions"),
401 cl::sub(ShowSubcommand));
402static cl::opt<ShowFormat>
403 SFormat("show-format", cl::init(Val: ShowFormat::Text),
404 cl::desc("Emit output in the selected format if supported"),
405 cl::sub(ShowSubcommand),
406 cl::values(clEnumValN(ShowFormat::Text, "text",
407 "emit normal text output (default)"),
408 clEnumValN(ShowFormat::Json, "json", "emit JSON"),
409 clEnumValN(ShowFormat::Yaml, "yaml", "emit YAML")));
410// TODO: Consider replacing this with `--show-format=text-encoding`.
411static cl::opt<bool>
412 TextFormat("text", cl::init(Val: false),
413 cl::desc("Show instr profile data in text dump format"),
414 cl::sub(ShowSubcommand));
415static cl::opt<bool>
416 JsonFormat("json",
417 cl::desc("Show sample profile data in the JSON format "
418 "(deprecated, please use --show-format=json)"),
419 cl::sub(ShowSubcommand));
420static cl::opt<bool> ShowIndirectCallTargets(
421 "ic-targets", cl::init(Val: false),
422 cl::desc("Show indirect call site target values for shown functions"),
423 cl::sub(ShowSubcommand));
424static cl::opt<bool>
425 ShowVTables("show-vtables", cl::init(Val: false),
426 cl::desc("Show vtable names for shown functions"),
427 cl::sub(ShowSubcommand));
428static cl::opt<bool> ShowMemOPSizes(
429 "memop-sizes", cl::init(Val: false),
430 cl::desc("Show the profiled sizes of the memory intrinsic calls "
431 "for shown functions"),
432 cl::sub(ShowSubcommand));
433static cl::opt<bool>
434 ShowDetailedSummary("detailed-summary", cl::init(Val: false),
435 cl::desc("Show detailed profile summary"),
436 cl::sub(ShowSubcommand));
437static cl::list<uint32_t> DetailedSummaryCutoffs(
438 cl::CommaSeparated, "detailed-summary-cutoffs",
439 cl::desc(
440 "Cutoff percentages (times 10000) for generating detailed summary"),
441 cl::value_desc("800000,901000,999999"), cl::sub(ShowSubcommand));
442static cl::opt<bool>
443 ShowHotFuncList("hot-func-list", cl::init(Val: false),
444 cl::desc("Show profile summary of a list of hot functions"),
445 cl::sub(ShowSubcommand));
446static cl::opt<bool>
447 ShowAllFunctions("all-functions", cl::init(Val: false),
448 cl::desc("Details for each and every function"),
449 cl::sub(ShowSubcommand));
450static cl::opt<bool> ShowCS("showcs", cl::init(Val: false),
451 cl::desc("Show context sensitive counts"),
452 cl::sub(ShowSubcommand));
453static cl::opt<ProfileKinds> ShowProfileKind(
454 cl::desc("Profile kind supported by show:"), cl::sub(ShowSubcommand),
455 cl::init(Val: instr),
456 cl::values(clEnumVal(instr, "Instrumentation profile (default)"),
457 clEnumVal(sample, "Sample profile"),
458 clEnumVal(memory, "MemProf memory access profile")));
459static cl::opt<uint32_t> TopNFunctions(
460 "topn", cl::init(Val: 0),
461 cl::desc("Show the list of functions with the largest internal counts"),
462 cl::sub(ShowSubcommand));
463static cl::opt<uint32_t> ShowValueCutoff(
464 "value-cutoff", cl::init(Val: 0),
465 cl::desc("Set the count value cutoff. Functions with the maximum count "
466 "less than this value will not be printed out. (Default is 0)"),
467 cl::sub(ShowSubcommand));
468static cl::opt<bool> OnlyListBelow(
469 "list-below-cutoff", cl::init(Val: false),
470 cl::desc("Only output names of functions whose max count values are "
471 "below the cutoff value"),
472 cl::sub(ShowSubcommand));
473static cl::opt<bool> ShowProfileSymbolList(
474 "show-prof-sym-list", cl::init(Val: false),
475 cl::desc("Show profile symbol list if it exists in the profile. "),
476 cl::sub(ShowSubcommand));
477static cl::opt<bool> ShowSectionInfoOnly(
478 "show-sec-info-only", cl::init(Val: false),
479 cl::desc("Show the information of each section in the sample profile. "
480 "The flag is only usable when the sample profile is in "
481 "extbinary format"),
482 cl::sub(ShowSubcommand));
483static cl::opt<bool> ShowCompositeInfoOnly(
484 "show-composite-info-only", cl::init(Val: false),
485 cl::desc("Show type IDs and payload sizes in a composite sample profile"),
486 cl::sub(ShowSubcommand));
487static cl::opt<bool> ShowBinaryIds("binary-ids", cl::init(Val: false),
488 cl::desc("Show binary ids in the profile. "),
489 cl::sub(ShowSubcommand));
490static cl::opt<bool> ShowTemporalProfTraces(
491 "temporal-profile-traces",
492 cl::desc("Show temporal profile traces in the profile."),
493 cl::sub(ShowSubcommand));
494
495static cl::opt<bool>
496 ShowCovered("covered", cl::init(Val: false),
497 cl::desc("Show only the functions that have been executed."),
498 cl::sub(ShowSubcommand));
499
500static cl::opt<bool> ShowProfileVersion("profile-version", cl::init(Val: false),
501 cl::desc("Show profile version. "),
502 cl::sub(ShowSubcommand));
503
504// Options specific to order subcommand.
505static cl::opt<unsigned>
506 NumTestTraces("num-test-traces", cl::init(Val: 0),
507 cl::desc("Keep aside the last <num-test-traces> traces in "
508 "the profile when computing the function order and "
509 "instead use them to evaluate that order"),
510 cl::sub(OrderSubcommand));
511
512// We use this string to indicate that there are
513// multiple static functions map to the same name.
514const std::string DuplicateNameStr = "----";
515
516static void warn(Twine Message, StringRef Whence = "", StringRef Hint = "") {
517 WithColor::warning();
518 if (!Whence.empty())
519 errs() << Whence << ": ";
520 errs() << Message << "\n";
521 if (!Hint.empty())
522 WithColor::note() << Hint << "\n";
523}
524
525static void warn(Error E, StringRef Whence = "") {
526 if (E.isA<InstrProfError>()) {
527 handleAllErrors(E: std::move(E), Handlers: [&](const InstrProfError &IPE) {
528 warn(Message: IPE.message(), Whence);
529 });
530 }
531}
532
533namespace {
534class ProfdataError : public ErrorInfo<ProfdataError> {
535public:
536 static char ID;
537
538 ProfdataError(Twine Message, Twine Whence = "", Twine Hint = "")
539 : Message(Message.str()), Whence(Whence.str()), Hint(Hint.str()) {}
540
541 void log(raw_ostream &OS) const override {
542 if (!Whence.empty())
543 OS << Whence << ": ";
544 OS << Message;
545 }
546
547 void print() const {
548 WithColor::error();
549 log(OS&: errs());
550 errs() << "\n";
551 if (!Hint.empty())
552 WithColor::note() << Hint << "\n";
553 }
554
555 std::error_code convertToErrorCode() const override {
556 return inconvertibleErrorCode();
557 }
558
559private:
560 std::string Message;
561 std::string Whence;
562 std::string Hint;
563};
564
565char ProfdataError::ID = 0;
566} // namespace
567
568static Error makeError(Twine Message, StringRef Whence = "",
569 StringRef Hint = "") {
570 return make_error<ProfdataError>(Args&: Message, Args&: Whence, Args&: Hint);
571}
572
573static Error makeError(Error E, StringRef Whence = "") {
574 if (E.isA<InstrProfError>()) {
575 std::string Msg;
576 std::string Hint;
577 handleAllErrors(E: std::move(E), Handlers: [&](const InstrProfError &IPE) {
578 instrprof_error instrError = IPE.get();
579 if (instrError == instrprof_error::unrecognized_format) {
580 // Hint in case user missed specifying the profile type.
581 Hint = "Perhaps you forgot to use the --sample or --memory option?";
582 }
583 Msg = IPE.message();
584 });
585 return makeError(Message: Msg, Whence, Hint);
586 }
587
588 return makeError(Message: toString(E: std::move(E)), Whence);
589}
590
591static Error makeError(std::error_code EC, StringRef Whence = "") {
592 return makeError(Message: EC.message(), Whence);
593}
594
595static int reportError(Error E) {
596 if (!E)
597 return 0;
598 handleAllErrors(
599 E: std::move(E), Handlers: [](const ProfdataError &PE) { PE.print(); },
600 Handlers: [](const ErrorInfoBase &EIB) {
601 WithColor::error() << EIB.message() << "\n";
602 });
603 return 1;
604}
605
606static Error warnOrErrorGivenError(FailureMode FailMode, std::error_code EC,
607 StringRef Whence = "") {
608 if (FailMode == failIfAnyAreInvalid)
609 return makeError(EC, Whence);
610 warn(Message: EC.message(), Whence);
611 return Error::success();
612}
613
614static void handleMergeWriterError(Error E, StringRef WhenceFile = "",
615 StringRef WhenceFunction = "",
616 bool ShowHint = true) {
617 if (!WhenceFile.empty())
618 errs() << WhenceFile << ": ";
619 if (!WhenceFunction.empty())
620 errs() << WhenceFunction << ": ";
621
622 auto IPE = instrprof_error::success;
623 E = handleErrors(E: std::move(E),
624 Hs: [&IPE](std::unique_ptr<InstrProfError> E) -> Error {
625 IPE = E->get();
626 return Error(std::move(E));
627 });
628 errs() << toString(E: std::move(E)) << "\n";
629
630 if (ShowHint) {
631 StringRef Hint = "";
632 if (IPE != instrprof_error::success) {
633 switch (IPE) {
634 case instrprof_error::hash_mismatch:
635 case instrprof_error::count_mismatch:
636 case instrprof_error::value_site_count_mismatch:
637 Hint = "Make sure that all profile data to be merged is generated "
638 "from the same binary.";
639 break;
640 default:
641 break;
642 }
643 }
644
645 if (!Hint.empty())
646 errs() << Hint << "\n";
647 }
648}
649
650namespace {
651/// A remapper from original symbol names to new symbol names based on a file
652/// containing a list of mappings from old name to new name.
653class SymbolRemapper {
654 std::unique_ptr<MemoryBuffer> File;
655 DenseMap<StringRef, StringRef> RemappingTable;
656
657public:
658 /// Build a SymbolRemapper from a file containing a list of old/new symbols.
659 static Expected<std::unique_ptr<SymbolRemapper>> create(StringRef InputFile) {
660 auto BufOrError = MemoryBuffer::getFileOrSTDIN(Filename: InputFile);
661 if (!BufOrError)
662 return makeError(EC: BufOrError.getError(), Whence: InputFile);
663
664 auto Remapper = std::make_unique<SymbolRemapper>();
665 Remapper->File = std::move(BufOrError.get());
666
667 for (line_iterator LineIt(*Remapper->File, /*SkipBlanks=*/true, '#');
668 !LineIt.is_at_eof(); ++LineIt) {
669 std::pair<StringRef, StringRef> Parts = LineIt->split(Separator: ' ');
670 if (Parts.first.empty() || Parts.second.empty() ||
671 Parts.second.count(C: ' ')) {
672 return makeError(Message: "unexpected line in remapping file",
673 Whence: (InputFile + ":" + Twine(LineIt.line_number())).str(),
674 Hint: "expected 'old_symbol new_symbol'");
675 }
676 Remapper->RemappingTable.insert(KV: Parts);
677 }
678 return std::move(Remapper);
679 }
680
681 /// Attempt to map the given old symbol into a new symbol.
682 ///
683 /// \return The new symbol, or \p Name if no such symbol was found.
684 StringRef operator()(StringRef Name) {
685 StringRef New = RemappingTable.lookup(Val: Name);
686 return New.empty() ? Name : New;
687 }
688
689 FunctionId operator()(FunctionId Name) {
690 // MD5 name cannot be remapped.
691 if (!Name.isStringRef())
692 return Name;
693 StringRef New = RemappingTable.lookup(Val: Name.stringRef());
694 return New.empty() ? Name : FunctionId(New);
695 }
696};
697}
698
699struct WeightedFile {
700 std::string Filename;
701 uint64_t Weight;
702};
703typedef SmallVector<WeightedFile, 5> WeightedFileVector;
704
705/// Keep track of merged data and reported errors.
706struct WriterContext {
707 std::mutex Lock;
708 InstrProfWriter Writer;
709 std::vector<std::pair<Error, std::string>> Errors;
710 std::mutex &ErrLock;
711 SmallSet<instrprof_error, 4> &WriterErrorCodes;
712
713 WriterContext(bool IsSparse, std::mutex &ErrLock,
714 SmallSet<instrprof_error, 4> &WriterErrorCodes,
715 uint64_t ReservoirSize = 0, uint64_t MaxTraceLength = 0)
716 : Writer(IsSparse, ReservoirSize, MaxTraceLength, DoWritePrevVersion,
717 MemProfVersionRequested, MemProfFullSchema,
718 MemprofGenerateRandomHotness, RandomSeed),
719 ErrLock(ErrLock), WriterErrorCodes(WriterErrorCodes) {}
720
721 ~WriterContext() {
722 for (auto &ErrorPair : Errors)
723 consumeError(Err: std::move(ErrorPair.first));
724 }
725};
726
727/// Computer the overlap b/w profile BaseFilename and TestFileName,
728/// and store the program level result to Overlap.
729static void overlapInput(const std::string &BaseFilename,
730 const std::string &TestFilename, WriterContext *WC,
731 OverlapStats &Overlap,
732 const OverlapFuncFilters &FuncFilter,
733 raw_fd_ostream &OS, bool IsCS) {
734 auto FS = vfs::getRealFileSystem();
735 auto ReaderOrErr = InstrProfReader::create(Path: TestFilename, FS&: *FS);
736 if (Error E = ReaderOrErr.takeError()) {
737 // Skip the empty profiles by returning sliently.
738 auto [ErrorCode, Msg] = InstrProfError::take(E: std::move(E));
739 if (ErrorCode != instrprof_error::empty_raw_profile)
740 WC->Errors.emplace_back(args: make_error<InstrProfError>(Args&: ErrorCode, Args&: Msg),
741 args: TestFilename);
742 return;
743 }
744
745 auto Reader = std::move(ReaderOrErr.get());
746 for (auto &I : *Reader) {
747 OverlapStats FuncOverlap(OverlapStats::FunctionLevel);
748 FuncOverlap.setFuncInfo(Name: I.Name, Hash: I.Hash);
749
750 WC->Writer.overlapRecord(Other: std::move(I), Overlap, FuncLevelOverlap&: FuncOverlap, FuncFilter);
751 FuncOverlap.dump(OS);
752 }
753}
754
755/// Load an input into a writer context.
756static Error loadInput(
757 const WeightedFile &Input, SymbolRemapper *Remapper,
758 const InstrProfCorrelator *Correlator, const StringRef ProfiledBinary,
759 WriterContext *WC, const object::BuildIDFetcher *BIDFetcher = nullptr,
760 std::optional<ProfCorrelatorKind> BIDFetcherCorrelatorKind = std::nullopt) {
761 std::unique_lock<std::mutex> CtxGuard{WC->Lock};
762
763 // Copy the filename, because llvm::ThreadPool copied the input "const
764 // WeightedFile &" by value, making a reference to the filename within it
765 // invalid outside of this packaged task.
766 std::string Filename = Input.Filename;
767
768 using ::llvm::memprof::RawMemProfReader;
769 if (RawMemProfReader::hasFormat(Path: Input.Filename)) {
770 auto ReaderOrErr = RawMemProfReader::create(Path: Input.Filename, ProfiledBinary);
771 if (!ReaderOrErr)
772 return makeError(E: ReaderOrErr.takeError(), Whence: Input.Filename);
773 std::unique_ptr<RawMemProfReader> Reader = std::move(ReaderOrErr.get());
774 // Check if the profile types can be merged, e.g. clang frontend profiles
775 // should not be merged with memprof profiles.
776 if (Error E = WC->Writer.mergeProfileKind(Other: Reader->getProfileKind())) {
777 consumeError(Err: std::move(E));
778 WC->Errors.emplace_back(
779 args: make_error<StringError>(
780 Args: "Cannot merge MemProf profile with Clang generated profile.",
781 Args: std::error_code()),
782 args&: Filename);
783 return Error::success();
784 }
785
786 auto MemProfError = [&](Error E) {
787 auto [ErrorCode, Msg] = InstrProfError::take(E: std::move(E));
788 WC->Errors.emplace_back(args: make_error<InstrProfError>(Args&: ErrorCode, Args&: Msg),
789 args&: Filename);
790 };
791
792 WC->Writer.addMemProfData(Incoming: Reader->takeMemProfData(), Warn: MemProfError);
793 return Error::success();
794 }
795
796 using ::llvm::memprof::YAMLMemProfReader;
797 if (YAMLMemProfReader::hasFormat(Path: Input.Filename)) {
798 auto ReaderOrErr = YAMLMemProfReader::create(Path: Input.Filename);
799 if (!ReaderOrErr)
800 return makeError(E: ReaderOrErr.takeError(), Whence: Input.Filename);
801 std::unique_ptr<YAMLMemProfReader> Reader = std::move(ReaderOrErr.get());
802 // Check if the profile types can be merged, e.g. clang frontend profiles
803 // should not be merged with memprof profiles.
804 if (Error E = WC->Writer.mergeProfileKind(Other: Reader->getProfileKind())) {
805 consumeError(Err: std::move(E));
806 WC->Errors.emplace_back(
807 args: make_error<StringError>(
808 Args: "Cannot merge MemProf profile with incompatible profile.",
809 Args: std::error_code()),
810 args&: Filename);
811 return Error::success();
812 }
813
814 auto MemProfError = [&](Error E) {
815 auto [ErrorCode, Msg] = InstrProfError::take(E: std::move(E));
816 WC->Errors.emplace_back(args: make_error<InstrProfError>(Args&: ErrorCode, Args&: Msg),
817 args&: Filename);
818 };
819
820 auto MemProfData = Reader->takeMemProfData();
821
822 auto DataAccessProfData = Reader->takeDataAccessProfData();
823
824 // Check for the empty input in case the YAML file is invalid.
825 if (MemProfData.Records.empty() &&
826 (!DataAccessProfData || DataAccessProfData->empty())) {
827 WC->Errors.emplace_back(
828 args: make_error<StringError>(Args: "The profile is empty.", Args: std::error_code()),
829 args&: Filename);
830 }
831
832 WC->Writer.addMemProfData(Incoming: std::move(MemProfData), Warn: MemProfError);
833 WC->Writer.addDataAccessProfData(DataAccessProfile: std::move(DataAccessProfData));
834 return Error::success();
835 }
836
837 auto FS = vfs::getRealFileSystem();
838 // TODO: This only saves the first non-fatal error from InstrProfReader, and
839 // then added to WriterContext::Errors. However, this is not extensible, if
840 // we have more non-fatal errors from InstrProfReader in the future. How
841 // should this interact with different -failure-mode?
842 std::optional<std::pair<Error, std::string>> ReaderWarning;
843 llvm::scope_exit ReaderWarningScope([&] {
844 // If we hit a different error we may still have an error in ReaderWarning.
845 // Consume it now to avoid an assert
846 if (ReaderWarning)
847 consumeError(Err: std::move(ReaderWarning->first));
848 });
849 auto Warn = [&](Error E) {
850 if (ReaderWarning) {
851 consumeError(Err: std::move(E));
852 return;
853 }
854 // Only show the first time an error occurs in this file.
855 auto [ErrCode, Msg] = InstrProfError::take(E: std::move(E));
856 ReaderWarning = {make_error<InstrProfError>(Args&: ErrCode, Args&: Msg), Filename};
857 };
858
859 auto ReaderOrErr =
860 InstrProfReader::create(Path: Input.Filename, FS&: *FS, Correlator, BIDFetcher,
861 BIDFetcherCorrelatorKind, Warn);
862 if (Error E = ReaderOrErr.takeError()) {
863 // Skip the empty profiles by returning silently.
864 auto [ErrCode, Msg] = InstrProfError::take(E: std::move(E));
865 if (ErrCode != instrprof_error::empty_raw_profile)
866 WC->Errors.emplace_back(args: make_error<InstrProfError>(Args&: ErrCode, Args&: Msg),
867 args&: Filename);
868 return Error::success();
869 }
870
871 auto Reader = std::move(ReaderOrErr.get());
872 if (Error E = WC->Writer.mergeProfileKind(Other: Reader->getProfileKind())) {
873 WC->Errors.emplace_back(args: std::move(E), args&: Filename);
874 return Error::success();
875 }
876
877 for (auto &I : *Reader) {
878 if (Remapper)
879 I.Name = (*Remapper)(I.Name);
880 const StringRef FuncName = I.Name;
881 bool Reported = false;
882
883 WC->Writer.addRecord(I: std::move(I), Weight: Input.Weight, Warn: [&](Error E) {
884 if (Reported) {
885 consumeError(Err: std::move(E));
886 return;
887 }
888 Reported = true;
889 // Only show hint the first time an error occurs.
890 auto [ErrCode, Msg] = InstrProfError::take(E: std::move(E));
891 std::unique_lock<std::mutex> ErrGuard{WC->ErrLock};
892 bool firstTime = WC->WriterErrorCodes.insert(V: ErrCode).second;
893 handleMergeWriterError(E: make_error<InstrProfError>(Args&: ErrCode, Args&: Msg),
894 WhenceFile: Input.Filename, WhenceFunction: FuncName, ShowHint: firstTime);
895 });
896 }
897
898 if (KeepVTableSymbols) {
899 const InstrProfSymtab &symtab = Reader->getSymtab();
900 const auto &VTableNames = symtab.getVTableNames();
901
902 for (const auto &kv : VTableNames)
903 WC->Writer.addVTableName(VTableName: kv.getKey());
904 }
905
906 if (Reader->hasTemporalProfile()) {
907 auto &Traces = Reader->getTemporalProfTraces(Weight: Input.Weight);
908 if (!Traces.empty())
909 WC->Writer.addTemporalProfileTraces(
910 SrcTraces&: Traces, SrcStreamSize: Reader->getTemporalProfTraceStreamSize());
911 }
912 if (Reader->hasError()) {
913 if (Error E = Reader->getError()) {
914 WC->Errors.emplace_back(args: std::move(E), args&: Filename);
915 return Error::success();
916 }
917 }
918
919 std::vector<llvm::object::BuildID> BinaryIds;
920 if (Error E = Reader->readBinaryIds(BinaryIds)) {
921 WC->Errors.emplace_back(args: std::move(E), args&: Filename);
922 return Error::success();
923 }
924 WC->Writer.addBinaryIds(BIs: BinaryIds);
925
926 if (ReaderWarning) {
927 WC->Errors.emplace_back(args: std::move(ReaderWarning->first),
928 args&: ReaderWarning->second);
929 }
930 return Error::success();
931}
932
933/// Merge the \p Src writer context into \p Dst.
934static Error mergeWriterContexts(WriterContext *Dst, WriterContext *Src) {
935 for (auto &ErrorPair : Src->Errors)
936 Dst->Errors.push_back(x: std::move(ErrorPair));
937 Src->Errors.clear();
938
939 if (Error E = Dst->Writer.mergeProfileKind(Other: Src->Writer.getProfileKind()))
940 return makeError(E: std::move(E));
941
942 Dst->Writer.mergeRecordsFromWriter(IPW: std::move(Src->Writer), Warn: [&](Error E) {
943 auto [ErrorCode, Msg] = InstrProfError::take(E: std::move(E));
944 std::unique_lock<std::mutex> ErrGuard{Dst->ErrLock};
945 bool firstTime = Dst->WriterErrorCodes.insert(V: ErrorCode).second;
946 if (firstTime)
947 warn(Message: toString(E: make_error<InstrProfError>(Args&: ErrorCode, Args&: Msg)));
948 });
949 return Error::success();
950}
951
952static StringRef
953getFuncName(const StringMap<InstrProfWriter::ProfilingData>::value_type &Val) {
954 return Val.first();
955}
956
957static std::string
958getFuncName(const SampleProfileMap::value_type &Val) {
959 return Val.second.getContext().toString();
960}
961
962template <typename T> static Error filterFunctions(T &ProfileMap) {
963 bool hasFilter = !FuncNameFilter.empty();
964 bool hasNegativeFilter = !FuncNameNegativeFilter.empty();
965 if (!hasFilter && !hasNegativeFilter)
966 return Error::success();
967
968 // If filter starts with '?' it is MSVC mangled name, not a regex.
969 llvm::Regex ProbablyMSVCMangledName("[?@$_0-9A-Za-z]+");
970 if (hasFilter && FuncNameFilter[0] == '?' &&
971 ProbablyMSVCMangledName.match(String: FuncNameFilter))
972 FuncNameFilter = llvm::Regex::escape(String: FuncNameFilter);
973 if (hasNegativeFilter && FuncNameNegativeFilter[0] == '?' &&
974 ProbablyMSVCMangledName.match(String: FuncNameNegativeFilter))
975 FuncNameNegativeFilter = llvm::Regex::escape(String: FuncNameNegativeFilter);
976
977 size_t Count = ProfileMap.size();
978 llvm::Regex Pattern(FuncNameFilter);
979 llvm::Regex NegativePattern(FuncNameNegativeFilter);
980 std::string RegexError;
981 if (hasFilter && !Pattern.isValid(Error&: RegexError))
982 return makeError(Message: RegexError);
983 if (hasNegativeFilter && !NegativePattern.isValid(Error&: RegexError))
984 return makeError(Message: RegexError);
985
986 // Handle MD5 profile, so it is still able to match using the original name.
987 std::string MD5Name = std::to_string(val: llvm::MD5Hash(Str: FuncNameFilter));
988 std::string NegativeMD5Name =
989 std::to_string(val: llvm::MD5Hash(Str: FuncNameNegativeFilter));
990
991 ProfileMap.remove_if([&](const auto &Entry) {
992 const auto &FuncName = getFuncName(Entry);
993 // Negative filter has higher precedence than positive filter.
994 return (hasNegativeFilter &&
995 (NegativePattern.match(String: FuncName) ||
996 (FunctionSamples::UseMD5 && NegativeMD5Name == FuncName))) ||
997 (hasFilter && !(Pattern.match(String: FuncName) ||
998 (FunctionSamples::UseMD5 && MD5Name == FuncName)));
999 });
1000
1001 llvm::dbgs() << Count - ProfileMap.size() << " of " << Count << " functions "
1002 << "in the original profile are filtered.\n";
1003 return Error::success();
1004}
1005
1006static Error writeInstrProfile(StringRef OutputFilename,
1007 ProfileFormat OutputFormat,
1008 InstrProfWriter &Writer) {
1009 std::error_code EC;
1010 raw_fd_ostream Output(OutputFilename.data(), EC,
1011 OutputFormat == PF_Text ? sys::fs::OF_TextWithCRLF
1012 : sys::fs::OF_None);
1013 if (EC)
1014 return makeError(EC, Whence: OutputFilename);
1015
1016 if (OutputFormat == PF_Text) {
1017 if (Error E = Writer.writeText(OS&: Output))
1018 warn(E: std::move(E));
1019 } else {
1020 if (Output.is_displayed())
1021 return makeError(
1022 Message: "cannot write a non-text format profile to the terminal");
1023 if (Error E = Writer.write(OS&: Output))
1024 warn(E: std::move(E));
1025 }
1026 return Error::success();
1027}
1028
1029static Error mergeInstrProfile(const WeightedFileVector &Inputs,
1030 SymbolRemapper *Remapper,
1031 int MaxDbgCorrelationWarnings,
1032 const StringRef ProfiledBinary) {
1033 const uint64_t TraceReservoirSize = TemporalProfTraceReservoirSize.getValue();
1034 const uint64_t MaxTraceLength = TemporalProfMaxTraceLength.getValue();
1035 if (OutputFormat == PF_Compact_Binary)
1036 return makeError(Message: "Compact Binary is deprecated");
1037 if (OutputFormat != PF_Binary && OutputFormat != PF_Ext_Binary &&
1038 OutputFormat != PF_Text)
1039 return makeError(Message: "unknown format is specified");
1040
1041 // TODO: Maybe we should support correlation with mixture of different
1042 // correlation modes(w/wo debug-info/object correlation).
1043 if (DebugInfoFilename.empty()) {
1044 if (!BinaryFilename.empty() && (DebugInfod || !DebugFileDirectory.empty()))
1045 return makeError(Message: "Expected only one of -binary-file, -debuginfod or "
1046 "-debug-file-directory");
1047 } else if (!BinaryFilename.empty() || DebugInfod ||
1048 !DebugFileDirectory.empty()) {
1049 return makeError(
1050 Message: "Expected only one of -debug-info, -binary-file, -debuginfod "
1051 "or -debug-file-directory");
1052 }
1053 std::string CorrelateFilename;
1054 std::optional<ProfCorrelatorKind> CorrelateKind;
1055 if (!DebugInfoFilename.empty()) {
1056 CorrelateFilename = DebugInfoFilename;
1057 CorrelateKind = ProfCorrelatorKind::DebugInfo;
1058 } else if (!BinaryFilename.empty()) {
1059 CorrelateFilename = BinaryFilename;
1060 CorrelateKind = ProfCorrelatorKind::Binary;
1061 }
1062
1063 std::unique_ptr<InstrProfCorrelator> Correlator;
1064 if (CorrelateKind) {
1065 if (auto Err = InstrProfCorrelator::get(Filename: CorrelateFilename, FileKind: *CorrelateKind)
1066 .moveInto(Value&: Correlator))
1067 return makeError(E: std::move(Err), Whence: CorrelateFilename);
1068 if (auto Err = Correlator->correlateProfileData(MaxWarnings: MaxDbgCorrelationWarnings))
1069 return makeError(E: std::move(Err), Whence: CorrelateFilename);
1070 }
1071
1072 std::optional<ProfCorrelatorKind> BIDFetcherCorrelateKind;
1073 std::unique_ptr<object::BuildIDFetcher> BIDFetcher;
1074 if (DebugInfod) {
1075 llvm::HTTPClient::initialize();
1076 BIDFetcher = std::make_unique<DebuginfodFetcher>(args&: DebugFileDirectory);
1077 if (!BIDFetcherProfileCorrelate.getNumOccurrences())
1078 return makeError(Message: "Expected --correlate when --debuginfod is provided");
1079 BIDFetcherCorrelateKind = BIDFetcherProfileCorrelate;
1080 } else if (!DebugFileDirectory.empty()) {
1081 BIDFetcher = std::make_unique<object::BuildIDFetcher>(args&: DebugFileDirectory);
1082 if (!BIDFetcherProfileCorrelate.getNumOccurrences())
1083 return makeError(Message: "Expected --correlate when --debug-file-directory "
1084 "is provided");
1085 BIDFetcherCorrelateKind = BIDFetcherProfileCorrelate;
1086 } else if (BIDFetcherProfileCorrelate.getNumOccurrences()) {
1087 return makeError(Message: "Expected --debuginfod or --debug-file-directory when "
1088 "--correlate is provided");
1089 }
1090
1091 std::mutex ErrorLock;
1092 SmallSet<instrprof_error, 4> WriterErrorCodes;
1093
1094 // If NumThreads is not specified, auto-detect a good default.
1095 if (NumThreads == 0)
1096 NumThreads = std::min(a: hardware_concurrency().compute_thread_count(),
1097 b: unsigned((Inputs.size() + 1) / 2));
1098
1099 // Initialize the writer contexts.
1100 SmallVector<std::unique_ptr<WriterContext>, 4> Contexts;
1101 for (unsigned I = 0; I < NumThreads; ++I)
1102 Contexts.emplace_back(Args: std::make_unique<WriterContext>(
1103 args&: OutputSparse, args&: ErrorLock, args&: WriterErrorCodes, args: TraceReservoirSize,
1104 args: MaxTraceLength));
1105
1106 if (NumThreads == 1) {
1107 for (const auto &Input : Inputs)
1108 if (Error E = loadInput(Input, Remapper, Correlator: Correlator.get(), ProfiledBinary,
1109 WC: Contexts[0].get(), BIDFetcher: BIDFetcher.get(),
1110 BIDFetcherCorrelatorKind: BIDFetcherCorrelateKind))
1111 return E;
1112 } else {
1113 Error FatalError = Error::success();
1114 auto hasFatalError = [&] {
1115 std::unique_lock<std::mutex> ErrGuard{ErrorLock};
1116 return static_cast<bool>(FatalError);
1117 };
1118
1119 DefaultThreadPool Pool(hardware_concurrency(ThreadCount: NumThreads));
1120 auto Async = [&](auto F, auto &&...Args) {
1121 Pool.async(
1122 [&, F](auto &&...InnerArgs) {
1123 if (hasFatalError())
1124 return;
1125 if (Error E = F(std::forward<decltype(InnerArgs)>(InnerArgs)...)) {
1126 std::unique_lock<std::mutex> ErrGuard{ErrorLock};
1127 if (FatalError)
1128 consumeError(Err: std::move(E));
1129 else
1130 FatalError = std::move(E);
1131 }
1132 },
1133 std::forward<decltype(Args)>(Args)...);
1134 };
1135
1136 // Load the inputs in parallel (N/NumThreads serial steps).
1137 unsigned Ctx = 0;
1138 for (const auto &Input : Inputs) {
1139 if (hasFatalError())
1140 break;
1141 Async(loadInput, Input, Remapper, Correlator.get(), ProfiledBinary,
1142 Contexts[Ctx].get(), BIDFetcher.get(), BIDFetcherCorrelateKind);
1143 Ctx = (Ctx + 1) % NumThreads;
1144 }
1145 Pool.wait();
1146 if (FatalError)
1147 return FatalError;
1148
1149 // Merge the writer contexts together (~ lg(NumThreads) serial steps).
1150 unsigned Mid = Contexts.size() / 2;
1151 unsigned End = Contexts.size();
1152 assert(Mid > 0 && "Expected more than one context");
1153 do {
1154 for (unsigned I = 0; I < Mid; ++I)
1155 Async(mergeWriterContexts, Contexts[I].get(), Contexts[I + Mid].get());
1156 Pool.wait();
1157 if (FatalError)
1158 return FatalError;
1159 if (End & 1) {
1160 Async(mergeWriterContexts, Contexts[0].get(), Contexts[End - 1].get());
1161 Pool.wait();
1162 if (FatalError)
1163 return FatalError;
1164 }
1165 End = Mid;
1166 Mid /= 2;
1167 } while (Mid > 0);
1168 }
1169
1170 // Handle deferred errors encountered during merging. If the number of errors
1171 // is equal to the number of inputs the merge failed.
1172 unsigned NumErrors = 0;
1173 for (std::unique_ptr<WriterContext> &WC : Contexts) {
1174 for (auto &ErrorPair : WC->Errors) {
1175 ++NumErrors;
1176 warn(Message: toString(E: std::move(ErrorPair.first)), Whence: ErrorPair.second);
1177 }
1178 }
1179 if ((NumErrors == Inputs.size() && FailMode == failIfAllAreInvalid) ||
1180 (NumErrors > 0 && FailMode == failIfAnyAreInvalid))
1181 return makeError(Message: "no profile can be merged");
1182
1183 if (Error E = filterFunctions(ProfileMap&: Contexts[0]->Writer.getProfileData()))
1184 return E;
1185
1186 return writeInstrProfile(OutputFilename, OutputFormat, Writer&: Contexts[0]->Writer);
1187}
1188
1189/// The profile entry for a function in instrumentation profile.
1190struct InstrProfileEntry {
1191 uint64_t MaxCount = 0;
1192 uint64_t NumEdgeCounters = 0;
1193 float ZeroCounterRatio = 0.0;
1194 InstrProfRecord *ProfRecord;
1195 InstrProfileEntry(InstrProfRecord *Record);
1196 InstrProfileEntry() = default;
1197};
1198
1199InstrProfileEntry::InstrProfileEntry(InstrProfRecord *Record) {
1200 ProfRecord = Record;
1201 uint64_t CntNum = Record->Counts.size();
1202 uint64_t ZeroCntNum = 0;
1203 for (size_t I = 0; I < CntNum; ++I) {
1204 MaxCount = std::max(a: MaxCount, b: Record->Counts[I]);
1205 ZeroCntNum += !Record->Counts[I];
1206 }
1207 ZeroCounterRatio = (float)ZeroCntNum / CntNum;
1208 NumEdgeCounters = CntNum;
1209}
1210
1211/// Either set all the counters in the instr profile entry \p IFE to
1212/// -1 / -2 /in order to drop the profile or scale up the
1213/// counters in \p IFP to be above hot / cold threshold. We use
1214/// the ratio of zero counters in the profile of a function to
1215/// decide the profile is helpful or harmful for performance,
1216/// and to choose whether to scale up or drop it.
1217static void updateInstrProfileEntry(InstrProfileEntry &IFE, bool SetToHot,
1218 uint64_t HotInstrThreshold,
1219 uint64_t ColdInstrThreshold,
1220 float ZeroCounterThreshold) {
1221 InstrProfRecord *ProfRecord = IFE.ProfRecord;
1222 if (!IFE.MaxCount || IFE.ZeroCounterRatio > ZeroCounterThreshold) {
1223 // If all or most of the counters of the function are zero, the
1224 // profile is unaccountable and should be dropped. Reset all the
1225 // counters to be -1 / -2 and PGO profile-use will drop the profile.
1226 // All counters being -1 also implies that the function is hot so
1227 // PGO profile-use will also set the entry count metadata to be
1228 // above hot threshold.
1229 // All counters being -2 implies that the function is warm so
1230 // PGO profile-use will also set the entry count metadata to be
1231 // above cold threshold.
1232 auto Kind =
1233 (SetToHot ? InstrProfRecord::PseudoHot : InstrProfRecord::PseudoWarm);
1234 ProfRecord->setPseudoCount(Kind);
1235 return;
1236 }
1237
1238 // Scale up the MaxCount to be multiple times above hot / cold threshold.
1239 const unsigned MultiplyFactor = 3;
1240 uint64_t Threshold = (SetToHot ? HotInstrThreshold : ColdInstrThreshold);
1241 uint64_t Numerator = Threshold * MultiplyFactor;
1242
1243 // Make sure Threshold for warm counters is below the HotInstrThreshold.
1244 if (!SetToHot && Threshold >= HotInstrThreshold) {
1245 Threshold = (HotInstrThreshold + ColdInstrThreshold) / 2;
1246 }
1247
1248 uint64_t Denominator = IFE.MaxCount;
1249 if (Numerator <= Denominator)
1250 return;
1251 ProfRecord->scale(N: Numerator, D: Denominator, Warn: [&](instrprof_error E) {
1252 warn(Message: toString(E: make_error<InstrProfError>(Args&: E)));
1253 });
1254}
1255
1256const uint64_t ColdPercentileIdx = 15;
1257const uint64_t HotPercentileIdx = 11;
1258
1259using sampleprof::FSDiscriminatorPass;
1260
1261// Internal options to set FSDiscriminatorPass. Used in merge and show
1262// commands.
1263static cl::opt<FSDiscriminatorPass> FSDiscriminatorPassOption(
1264 "fs-discriminator-pass", cl::init(Val: PassLast), cl::Hidden,
1265 cl::desc("Zero out the discriminator bits for the FS discrimiantor "
1266 "pass beyond this value. The enum values are defined in "
1267 "Support/Discriminator.h"),
1268 cl::values(clEnumVal(Base, "Use base discriminators only"),
1269 clEnumVal(Pass1, "Use base and pass 1 discriminators"),
1270 clEnumVal(Pass2, "Use base and pass 1-2 discriminators"),
1271 clEnumVal(Pass3, "Use base and pass 1-3 discriminators"),
1272 clEnumVal(PassLast, "Use all discriminator bits (default)")));
1273
1274static unsigned getDiscriminatorMask() {
1275 return getN1Bits(N: getFSPassBitEnd(P: FSDiscriminatorPassOption.getValue()));
1276}
1277
1278/// Adjust the instr profile in \p WC based on the sample profile in
1279/// \p Reader.
1280static void
1281adjustInstrProfile(std::unique_ptr<WriterContext> &WC,
1282 std::unique_ptr<sampleprof::SampleProfileReader> &Reader,
1283 unsigned SupplMinSizeThreshold, float ZeroCounterThreshold,
1284 unsigned InstrProfColdThreshold) {
1285 // Function to its entry in instr profile.
1286 StringMap<InstrProfileEntry> InstrProfileMap;
1287 StringMap<StringRef> StaticFuncMap;
1288 InstrProfSummaryBuilder IPBuilder(ProfileSummaryBuilder::DefaultCutoffs);
1289
1290 auto checkSampleProfileHasFUnique = [&Reader]() {
1291 for (const auto &PD : Reader->getProfiles()) {
1292 auto &FContext = PD.second.getContext();
1293 if (FContext.toString().find(s: FunctionSamples::UniqSuffix) !=
1294 std::string::npos) {
1295 return true;
1296 }
1297 }
1298 return false;
1299 };
1300
1301 bool SampleProfileHasFUnique = checkSampleProfileHasFUnique();
1302
1303 auto buildStaticFuncMap = [&StaticFuncMap,
1304 SampleProfileHasFUnique](const StringRef Name) {
1305 std::string FilePrefixes[] = {".cpp", "cc", ".c", ".hpp", ".h"};
1306 size_t PrefixPos = StringRef::npos;
1307 for (auto &FilePrefix : FilePrefixes) {
1308 std::string NamePrefix = FilePrefix + GlobalIdentifierDelimiter;
1309 PrefixPos = Name.find_insensitive(Str: NamePrefix);
1310 if (PrefixPos == StringRef::npos)
1311 continue;
1312 PrefixPos += NamePrefix.size();
1313 break;
1314 }
1315
1316 if (PrefixPos == StringRef::npos) {
1317 return;
1318 }
1319
1320 StringRef NewName = Name.drop_front(N: PrefixPos);
1321 StringRef FName = Name.substr(Start: 0, N: PrefixPos - 1);
1322 if (NewName.size() == 0) {
1323 return;
1324 }
1325
1326 // This name should have a static linkage.
1327 size_t PostfixPos = NewName.find(Str: FunctionSamples::UniqSuffix);
1328 bool ProfileHasFUnique = (PostfixPos != StringRef::npos);
1329
1330 // If sample profile and instrumented profile do not agree on symbol
1331 // uniqification.
1332 if (SampleProfileHasFUnique != ProfileHasFUnique) {
1333 // If instrumented profile uses -funique-internal-linkage-symbols,
1334 // we need to trim the name.
1335 if (ProfileHasFUnique) {
1336 NewName = NewName.substr(Start: 0, N: PostfixPos);
1337 } else {
1338 // If sample profile uses -funique-internal-linkage-symbols,
1339 // we build the map.
1340 std::string NStr =
1341 NewName.str() + getUniqueInternalLinkagePostfix(FName);
1342 NewName = StringRef(NStr);
1343 StaticFuncMap[NewName] = Name;
1344 return;
1345 }
1346 }
1347
1348 auto [It, Inserted] = StaticFuncMap.try_emplace(Key: NewName, Args: Name);
1349 if (!Inserted)
1350 It->second = DuplicateNameStr;
1351 };
1352
1353 // We need to flatten the SampleFDO profile as the InstrFDO
1354 // profile does not have inlined callsite profiles.
1355 // One caveat is the pre-inlined function -- their samples
1356 // should be collapsed into the caller function.
1357 // Here we do a DFS traversal to get the flatten profile
1358 // info: the sum of entrycount and the max of maxcount.
1359 // Here is the algorithm:
1360 // recursive (FS, root_name) {
1361 // name = FS->getName();
1362 // get samples for FS;
1363 // if (InstrProf.find(name) {
1364 // root_name = name;
1365 // } else {
1366 // if (name is in static_func map) {
1367 // root_name = static_name;
1368 // }
1369 // }
1370 // update the Map entry for root_name;
1371 // for (subfs: FS) {
1372 // recursive(subfs, root_name);
1373 // }
1374 // }
1375 //
1376 // Here is an example.
1377 //
1378 // SampleProfile:
1379 // foo:12345:1000
1380 // 1: 1000
1381 // 2.1: 1000
1382 // 15: 5000
1383 // 4: bar:1000
1384 // 1: 1000
1385 // 2: goo:3000
1386 // 1: 3000
1387 // 8: bar:40000
1388 // 1: 10000
1389 // 2: goo:30000
1390 // 1: 30000
1391 //
1392 // InstrProfile has two entries:
1393 // foo
1394 // bar.cc;bar
1395 //
1396 // After BuildMaxSampleMap, we should have the following in FlattenSampleMap:
1397 // {"foo", {1000, 5000}}
1398 // {"bar.cc;bar", {11000, 30000}}
1399 //
1400 // foo's has an entry count of 1000, and max body count of 5000.
1401 // bar.cc;bar has an entry count of 11000 (sum two callsites of 1000 and
1402 // 10000), and max count of 30000 (from the callsite in line 8).
1403 //
1404 // Note that goo's count will remain in bar.cc;bar() as it does not have an
1405 // entry in InstrProfile.
1406 llvm::StringMap<std::pair<uint64_t, uint64_t>> FlattenSampleMap;
1407 auto BuildMaxSampleMap = [&FlattenSampleMap, &StaticFuncMap,
1408 &InstrProfileMap](const FunctionSamples &FS,
1409 const StringRef &RootName) {
1410 auto BuildMaxSampleMapImpl = [&](const FunctionSamples &FS,
1411 const StringRef &RootName,
1412 auto &BuildImpl) -> void {
1413 std::string NameStr = FS.getFunction().str();
1414 const StringRef Name = NameStr;
1415 const StringRef *NewRootName = &RootName;
1416 uint64_t EntrySample = FS.getHeadSamplesEstimate();
1417 uint64_t MaxBodySample = FS.getMaxCountInside(/* SkipCallSite*/ true);
1418
1419 auto It = InstrProfileMap.find(Key: Name);
1420 if (It != InstrProfileMap.end()) {
1421 NewRootName = &Name;
1422 } else {
1423 auto NewName = StaticFuncMap.find(Key: Name);
1424 if (NewName != StaticFuncMap.end()) {
1425 It = InstrProfileMap.find(Key: NewName->second);
1426 if (NewName->second != DuplicateNameStr) {
1427 NewRootName = &NewName->second;
1428 }
1429 } else {
1430 // Here the EntrySample is of an inlined function, so we should not
1431 // update the EntrySample in the map.
1432 EntrySample = 0;
1433 }
1434 }
1435 EntrySample += FlattenSampleMap[*NewRootName].first;
1436 MaxBodySample =
1437 std::max(a: FlattenSampleMap[*NewRootName].second, b: MaxBodySample);
1438 FlattenSampleMap[*NewRootName] =
1439 std::make_pair(x&: EntrySample, y&: MaxBodySample);
1440
1441 for (const auto &C : FS.getCallsiteSamples())
1442 for (const auto &F : C.second)
1443 BuildImpl(F.second, *NewRootName, BuildImpl);
1444 };
1445 BuildMaxSampleMapImpl(FS, RootName, BuildMaxSampleMapImpl);
1446 };
1447
1448 for (auto &PD : WC->Writer.getProfileData()) {
1449 // Populate IPBuilder.
1450 for (const auto &PDV : PD.getValue()) {
1451 InstrProfRecord Record = PDV.second;
1452 IPBuilder.addRecord(Record);
1453 }
1454
1455 // If a function has multiple entries in instr profile, skip it.
1456 if (PD.getValue().size() != 1)
1457 continue;
1458
1459 // Initialize InstrProfileMap.
1460 InstrProfRecord *R = &PD.getValue().begin()->second;
1461 StringRef FullName = PD.getKey();
1462 InstrProfileMap[FullName] = InstrProfileEntry(R);
1463 buildStaticFuncMap(FullName);
1464 }
1465
1466 for (auto &PD : Reader->getProfiles()) {
1467 sampleprof::FunctionSamples &FS = PD.second;
1468 std::string Name = FS.getFunction().str();
1469 BuildMaxSampleMap(FS, Name);
1470 }
1471
1472 ProfileSummary InstrPS = *IPBuilder.getSummary();
1473 ProfileSummary SamplePS = Reader->getSummary();
1474
1475 // Compute cold thresholds for instr profile and sample profile.
1476 uint64_t HotSampleThreshold =
1477 ProfileSummaryBuilder::getEntryForPercentile(
1478 DS: SamplePS.getDetailedSummary(),
1479 Percentile: ProfileSummaryBuilder::DefaultCutoffs[HotPercentileIdx])
1480 .MinCount;
1481 uint64_t ColdSampleThreshold =
1482 ProfileSummaryBuilder::getEntryForPercentile(
1483 DS: SamplePS.getDetailedSummary(),
1484 Percentile: ProfileSummaryBuilder::DefaultCutoffs[ColdPercentileIdx])
1485 .MinCount;
1486 uint64_t HotInstrThreshold =
1487 ProfileSummaryBuilder::getEntryForPercentile(
1488 DS: InstrPS.getDetailedSummary(),
1489 Percentile: ProfileSummaryBuilder::DefaultCutoffs[HotPercentileIdx])
1490 .MinCount;
1491 uint64_t ColdInstrThreshold =
1492 InstrProfColdThreshold
1493 ? InstrProfColdThreshold
1494 : ProfileSummaryBuilder::getEntryForPercentile(
1495 DS: InstrPS.getDetailedSummary(),
1496 Percentile: ProfileSummaryBuilder::DefaultCutoffs[ColdPercentileIdx])
1497 .MinCount;
1498
1499 // Find hot/warm functions in sample profile which is cold in instr profile
1500 // and adjust the profiles of those functions in the instr profile.
1501 for (const auto &E : FlattenSampleMap) {
1502 uint64_t SampleMaxCount = std::max(a: E.second.first, b: E.second.second);
1503 if (SampleMaxCount < ColdSampleThreshold)
1504 continue;
1505 StringRef Name = E.first();
1506 auto It = InstrProfileMap.find(Key: Name);
1507 if (It == InstrProfileMap.end()) {
1508 auto NewName = StaticFuncMap.find(Key: Name);
1509 if (NewName != StaticFuncMap.end()) {
1510 It = InstrProfileMap.find(Key: NewName->second);
1511 if (NewName->second == DuplicateNameStr) {
1512 WithColor::warning()
1513 << "Static function " << Name
1514 << " has multiple promoted names, cannot adjust profile.\n";
1515 }
1516 }
1517 }
1518 if (It == InstrProfileMap.end() ||
1519 It->second.MaxCount > ColdInstrThreshold ||
1520 It->second.NumEdgeCounters < SupplMinSizeThreshold)
1521 continue;
1522 bool SetToHot = SampleMaxCount >= HotSampleThreshold;
1523 updateInstrProfileEntry(IFE&: It->second, SetToHot, HotInstrThreshold,
1524 ColdInstrThreshold, ZeroCounterThreshold);
1525 }
1526}
1527
1528/// The main function to supplement instr profile with sample profile.
1529/// \Inputs contains the instr profile. \p SampleFilename specifies the
1530/// sample profile. \p OutputFilename specifies the output profile name.
1531/// \p OutputFormat specifies the output profile format. \p OutputSparse
1532/// specifies whether to generate sparse profile. \p SupplMinSizeThreshold
1533/// specifies the minimal size for the functions whose profile will be
1534/// adjusted. \p ZeroCounterThreshold is the threshold to check whether
1535/// a function contains too many zero counters and whether its profile
1536/// should be dropped. \p InstrProfColdThreshold is the user specified
1537/// cold threshold which will override the cold threshold got from the
1538/// instr profile summary.
1539static Error supplementInstrProfile(const WeightedFileVector &Inputs,
1540 StringRef SampleFilename, bool OutputSparse,
1541 unsigned SupplMinSizeThreshold,
1542 float ZeroCounterThreshold,
1543 unsigned InstrProfColdThreshold) {
1544 if (OutputFilename == "-")
1545 return makeError(Message: "cannot write indexed profdata format to stdout");
1546 if (Inputs.size() != 1)
1547 return makeError(Message: "expect one input to be an instr profile");
1548 if (Inputs[0].Weight != 1)
1549 return makeError(Message: "expect instr profile doesn't have weight");
1550
1551 StringRef InstrFilename = Inputs[0].Filename;
1552
1553 // Read sample profile.
1554 LLVMContext Context;
1555 auto FS = vfs::getRealFileSystem();
1556 auto ReaderOrErr = sampleprof::SampleProfileReader::create(
1557 Filename: SampleFilename.str(), C&: Context, FS&: *FS, P: FSDiscriminatorPassOption);
1558 if (std::error_code EC = ReaderOrErr.getError())
1559 return makeError(EC, Whence: SampleFilename);
1560 auto Reader = std::move(ReaderOrErr.get());
1561 if (std::error_code EC = Reader->read())
1562 return makeError(EC, Whence: SampleFilename);
1563
1564 // Read instr profile.
1565 std::mutex ErrorLock;
1566 SmallSet<instrprof_error, 4> WriterErrorCodes;
1567 auto WC = std::make_unique<WriterContext>(args&: OutputSparse, args&: ErrorLock,
1568 args&: WriterErrorCodes);
1569 if (Error E = loadInput(Input: Inputs[0], Remapper: nullptr, Correlator: nullptr, /*ProfiledBinary=*/"",
1570 WC: WC.get()))
1571 return E;
1572 if (!WC->Errors.empty())
1573 return makeError(E: std::move(WC->Errors[0].first), Whence: InstrFilename);
1574
1575 adjustInstrProfile(WC, Reader, SupplMinSizeThreshold, ZeroCounterThreshold,
1576 InstrProfColdThreshold);
1577 return writeInstrProfile(OutputFilename, OutputFormat, Writer&: WC->Writer);
1578}
1579
1580/// Make a copy of the given function samples with all symbol names remapped
1581/// by the provided symbol remapper.
1582static sampleprof::FunctionSamples
1583remapSamples(const sampleprof::FunctionSamples &Samples,
1584 SymbolRemapper &Remapper, sampleprof_error &Error) {
1585 sampleprof::FunctionSamples Result;
1586 Result.setFunction(Remapper(Samples.getFunction()));
1587 Result.addTotalSamples(Num: Samples.getTotalSamples());
1588 Result.addHeadSamples(Num: Samples.getHeadSamples());
1589 Result.reserveBodySamples(NumEntries: Samples.getBodySamples().size());
1590 for (const auto &BodySample : Samples.getBodySamples()) {
1591 uint32_t MaskedDiscriminator =
1592 BodySample.first.Discriminator & getDiscriminatorMask();
1593 Result.addBodySamples(LineOffset: BodySample.first.LineOffset, Discriminator: MaskedDiscriminator,
1594 Num: BodySample.second.getSamples());
1595 for (const auto &Target : BodySample.second.getCallTargets()) {
1596 Result.addCalledTargetSamples(LineOffset: BodySample.first.LineOffset,
1597 Discriminator: MaskedDiscriminator,
1598 Func: Remapper(Target.first), Num: Target.second);
1599 }
1600 }
1601 for (const auto &CallsiteSamples : Samples.getCallsiteSamples()) {
1602 sampleprof::FunctionSamplesMap &Target =
1603 Result.functionSamplesAt(Loc: CallsiteSamples.first);
1604 for (const auto &Callsite : CallsiteSamples.second) {
1605 sampleprof::FunctionSamples Remapped =
1606 remapSamples(Samples: Callsite.second, Remapper, Error);
1607 mergeSampleProfErrors(Accumulator&: Error,
1608 Result: Target[Remapped.getFunction()].merge(Other: Remapped));
1609 }
1610 }
1611 return Result;
1612}
1613
1614static sampleprof::SampleProfileFormat FormatMap[] = {
1615 sampleprof::SPF_None,
1616 sampleprof::SPF_Text,
1617 sampleprof::SPF_None,
1618 sampleprof::SPF_Ext_Binary,
1619 sampleprof::SPF_GCC,
1620 sampleprof::SPF_Binary};
1621
1622static Expected<std::unique_ptr<MemoryBuffer>>
1623getInputFileBuf(const StringRef &InputFile) {
1624 if (InputFile == "")
1625 return {nullptr};
1626
1627 auto BufOrError = MemoryBuffer::getFileOrSTDIN(Filename: InputFile);
1628 if (!BufOrError)
1629 return makeError(EC: BufOrError.getError(), Whence: InputFile);
1630
1631 return std::move(*BufOrError);
1632}
1633
1634static void populateProfileSymbolList(MemoryBuffer *Buffer,
1635 sampleprof::ProfileSymbolList &PSL) {
1636 if (!Buffer)
1637 return;
1638
1639 SmallVector<StringRef, 32> SymbolVec;
1640 StringRef Data = Buffer->getBuffer();
1641 Data.split(A&: SymbolVec, Separator: '\n', /*MaxSplit=*/-1, /*KeepEmpty=*/false);
1642
1643 for (StringRef SymbolStr : SymbolVec)
1644 PSL.add(Name: SymbolStr.trim());
1645}
1646
1647static void handleExtBinaryWriter(sampleprof::SampleProfileWriter &Writer,
1648 ProfileFormat OutputFormat,
1649 MemoryBuffer *Buffer,
1650 sampleprof::ProfileSymbolList &WriterList,
1651 bool CompressAllSections, bool UseMD5,
1652 bool GenPartialProfile) {
1653 if (SplitLayout) {
1654 if (OutputFormat == PF_Binary)
1655 warn(Message: "-split-layout is ignored. Specify -extbinary to enable it");
1656 else
1657 Writer.setUseCtxSplitLayout();
1658 }
1659
1660 populateProfileSymbolList(Buffer, PSL&: WriterList);
1661 if (WriterList.size() > 0 && OutputFormat != PF_Ext_Binary)
1662 warn(Message: "Profile Symbol list is not empty but the output format is not "
1663 "ExtBinary format. The list will be lost in the output. ");
1664
1665 Writer.setProfileSymbolList(&WriterList);
1666
1667 if (CompressAllSections) {
1668 if (OutputFormat != PF_Ext_Binary)
1669 warn(Message: "-compress-all-section is ignored. Specify -extbinary to enable it");
1670 else
1671 Writer.setToCompressAllSections();
1672 }
1673 if (UseMD5) {
1674 if (OutputFormat != PF_Ext_Binary)
1675 warn(Message: "-use-md5 is ignored. Specify -extbinary to enable it");
1676 else
1677 Writer.setUseMD5();
1678 }
1679 if (GenPartialProfile) {
1680 if (OutputFormat != PF_Ext_Binary)
1681 warn(Message: "-gen-partial-profile is ignored. Specify -extbinary to enable it");
1682 else
1683 Writer.setPartialProfile();
1684 }
1685 if (WriteMD5ProfSymList) {
1686 if (OutputFormat != PF_Ext_Binary)
1687 warn(Message: "-md5-prof-sym-list is ignored. Specify -extbinary to enable it");
1688 else
1689 Writer.setUseMD5ProfileSymbolList();
1690 }
1691 if (WriteMD5IndexedTables) {
1692 if (OutputFormat != PF_Ext_Binary)
1693 warn(Message: "-md5-indexed-tables is ignored. Specify -extbinary to enable it");
1694 else
1695 Writer.setUseMD5IndexedTables();
1696 }
1697}
1698
1699static Error mergeSampleProfile(const WeightedFileVector &Inputs,
1700 SymbolRemapper *Remapper,
1701 StringRef ProfileSymbolListFile,
1702 size_t OutputSizeLimit) {
1703 using namespace sampleprof;
1704 SampleProfileMap ProfileMap;
1705 SmallVector<std::unique_ptr<sampleprof::SampleProfileReader>, 5> Readers;
1706 LLVMContext Context;
1707 sampleprof::ProfileSymbolList WriterList;
1708 std::optional<bool> ProfileIsProbeBased;
1709 std::optional<bool> ProfileIsCS;
1710 for (const auto &Input : Inputs) {
1711 auto FS = vfs::getRealFileSystem();
1712 auto ReaderOrErr = SampleProfileReader::create(Filename: Input.Filename, C&: Context, FS&: *FS,
1713 P: FSDiscriminatorPassOption);
1714 if (std::error_code EC = ReaderOrErr.getError()) {
1715 if (Error E = warnOrErrorGivenError(FailMode, EC, Whence: Input.Filename))
1716 return E;
1717 continue;
1718 }
1719
1720 // We need to keep the readers around until after all the files are
1721 // read so that we do not lose the function names stored in each
1722 // reader's memory. The function names are needed to write out the
1723 // merged profile map.
1724 Readers.push_back(Elt: std::move(ReaderOrErr.get()));
1725 const auto Reader = Readers.back().get();
1726 if (std::error_code EC = Reader->read()) {
1727 if (Error E = warnOrErrorGivenError(FailMode, EC, Whence: Input.Filename))
1728 return E;
1729 Readers.pop_back();
1730 continue;
1731 }
1732
1733 // Merging cannot preserve payloads that this reader does not understand,
1734 // so make the otherwise intentional forward-compatible skip visible.
1735 if (Reader->hasUnknownProfileTypes())
1736 warn(Message: "unknown composite profile blocks were ignored and will not be "
1737 "preserved",
1738 Whence: Input.Filename);
1739
1740 SampleProfileMap &Profiles = Reader->getProfiles();
1741 if (ProfileIsProbeBased &&
1742 ProfileIsProbeBased != FunctionSamples::ProfileIsProbeBased)
1743 return makeError(
1744 Message: "cannot merge probe-based profile with non-probe-based profile");
1745 ProfileIsProbeBased = FunctionSamples::ProfileIsProbeBased;
1746 if (ProfileIsCS && ProfileIsCS != FunctionSamples::ProfileIsCS)
1747 return makeError(Message: "cannot merge CS profile with non-CS profile");
1748 ProfileIsCS = FunctionSamples::ProfileIsCS;
1749 for (SampleProfileMap::iterator I = Profiles.begin(), E = Profiles.end();
1750 I != E; ++I) {
1751 sampleprof_error Result = sampleprof_error::success;
1752 FunctionSamples Remapped =
1753 Remapper ? remapSamples(Samples: I->second, Remapper&: *Remapper, Error&: Result)
1754 : FunctionSamples();
1755 FunctionSamples &Samples = Remapper ? Remapped : I->second;
1756 SampleContext FContext = Samples.getContext();
1757 mergeSampleProfErrors(Accumulator&: Result,
1758 Result: ProfileMap[FContext].merge(Other: Samples, Weight: Input.Weight));
1759 if (Result != sampleprof_error::success) {
1760 std::error_code EC = make_error_code(E: Result);
1761 handleMergeWriterError(E: errorCodeToError(EC), WhenceFile: Input.Filename,
1762 WhenceFunction: FContext.toString());
1763 }
1764 }
1765
1766 if (!DropProfileSymbolList) {
1767 std::unique_ptr<sampleprof::ProfileSymbolList> ReaderList =
1768 Reader->getProfileSymbolList();
1769 if (ReaderList)
1770 WriterList.merge(List: *ReaderList);
1771 }
1772 }
1773
1774 if (ProfileIsCS && (SampleMergeColdContext || SampleTrimColdContext)) {
1775 // Use threshold calculated from profile summary unless specified.
1776 SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs);
1777 auto Summary = Builder.computeSummaryForProfiles(Profiles: ProfileMap);
1778 uint64_t SampleProfColdThreshold =
1779 ProfileSummaryBuilder::getColdCountThreshold(
1780 DS: (Summary->getDetailedSummary()));
1781
1782 // Trim and merge cold context profile using cold threshold above;
1783 SampleContextTrimmer(ProfileMap)
1784 .trimAndMergeColdContextProfiles(
1785 ColdCountThreshold: SampleProfColdThreshold, TrimColdContext: SampleTrimColdContext,
1786 MergeColdContext: SampleMergeColdContext, ColdContextFrameLength: SampleColdContextFrameDepth, TrimBaseProfileOnly: false);
1787 }
1788
1789 if (ProfileLayout == llvm::sampleprof::SPL_Flat) {
1790 ProfileConverter::flattenProfile(ProfileMap, ProfileIsCS: FunctionSamples::ProfileIsCS);
1791 ProfileIsCS = FunctionSamples::ProfileIsCS = false;
1792 } else if (ProfileIsCS && ProfileLayout == llvm::sampleprof::SPL_Nest) {
1793 ProfileConverter CSConverter(ProfileMap);
1794 CSConverter.convertCSProfiles();
1795 ProfileIsCS = FunctionSamples::ProfileIsCS = false;
1796 }
1797
1798 if (Error E = filterFunctions(ProfileMap))
1799 return E;
1800
1801 auto WriterOrErr =
1802 SampleProfileWriter::create(Filename: OutputFilename, Format: FormatMap[OutputFormat]);
1803 if (std::error_code EC = WriterOrErr.getError())
1804 return makeError(EC, Whence: OutputFilename);
1805
1806 auto Writer = std::move(WriterOrErr.get());
1807 // WriterList will have StringRef refering to string in Buffer.
1808 // Make sure Buffer lives as long as WriterList.
1809 auto BufferOrErr = getInputFileBuf(InputFile: ProfileSymbolListFile);
1810 if (!BufferOrErr)
1811 return BufferOrErr.takeError();
1812 auto Buffer = std::move(*BufferOrErr);
1813 handleExtBinaryWriter(Writer&: *Writer, OutputFormat, Buffer: Buffer.get(), WriterList,
1814 CompressAllSections, UseMD5, GenPartialProfile);
1815
1816 // If OutputSizeLimit is 0 (default), it is the same as write().
1817 if (std::error_code EC =
1818 Writer->writeWithSizeLimit(ProfileMap, OutputSizeLimit))
1819 return makeError(EC);
1820
1821 return Error::success();
1822}
1823
1824static Expected<WeightedFile>
1825parseWeightedFile(const StringRef &WeightedFilename) {
1826 StringRef WeightStr, FileName;
1827 std::tie(args&: WeightStr, args&: FileName) = WeightedFilename.split(Separator: ',');
1828
1829 uint64_t Weight;
1830 if (WeightStr.getAsInteger(Radix: 10, Result&: Weight) || Weight < 1)
1831 return makeError(Message: "input weight must be a positive integer");
1832
1833 llvm::SmallString<128> ResolvedFileName;
1834 llvm::sys::fs::expand_tilde(path: FileName, output&: ResolvedFileName);
1835
1836 return WeightedFile{.Filename: std::string(ResolvedFileName), .Weight: Weight};
1837}
1838
1839static Error addWeightedInput(WeightedFileVector &WNI, const WeightedFile &WF) {
1840 StringRef Filename = WF.Filename;
1841 uint64_t Weight = WF.Weight;
1842
1843 // If it's STDIN just pass it on.
1844 if (Filename == "-") {
1845 WNI.push_back(Elt: {.Filename: std::string(Filename), .Weight: Weight});
1846 return Error::success();
1847 }
1848
1849 llvm::sys::fs::file_status Status;
1850 llvm::sys::fs::status(path: Filename, result&: Status);
1851 if (!llvm::sys::fs::exists(status: Status))
1852 return makeError(EC: make_error_code(E: errc::no_such_file_or_directory),
1853 Whence: Filename);
1854 // If it's a source file, collect it.
1855 if (llvm::sys::fs::is_regular_file(status: Status)) {
1856 WNI.push_back(Elt: {.Filename: std::string(Filename), .Weight: Weight});
1857 return Error::success();
1858 }
1859
1860 if (llvm::sys::fs::is_directory(status: Status)) {
1861 std::error_code EC;
1862 for (llvm::sys::fs::recursive_directory_iterator F(Filename, EC), E;
1863 F != E && !EC; F.increment(ec&: EC)) {
1864 if (llvm::sys::fs::is_regular_file(Path: F->path())) {
1865 if (Error E = addWeightedInput(WNI, WF: {.Filename: F->path(), .Weight: Weight}))
1866 return E;
1867 }
1868 }
1869 if (EC)
1870 return makeError(EC, Whence: Filename);
1871 }
1872 return Error::success();
1873}
1874
1875static Error parseInputFilenamesFile(MemoryBuffer *Buffer,
1876 WeightedFileVector &WFV) {
1877 if (!Buffer)
1878 return Error::success();
1879
1880 SmallVector<StringRef, 8> Entries;
1881 StringRef Data = Buffer->getBuffer();
1882 Data.split(A&: Entries, Separator: '\n', /*MaxSplit=*/-1, /*KeepEmpty=*/false);
1883 for (const StringRef &FileWeightEntry : Entries) {
1884 StringRef SanitizedEntry = FileWeightEntry.trim(Chars: " \t\v\f\r");
1885 // Skip comments.
1886 if (SanitizedEntry.starts_with(Prefix: "#"))
1887 continue;
1888 // If there's no comma, it's an unweighted profile.
1889 else if (!SanitizedEntry.contains(C: ',')) {
1890 if (Error E = addWeightedInput(WNI&: WFV, WF: {.Filename: std::string(SanitizedEntry), .Weight: 1}))
1891 return E;
1892 } else {
1893 auto WFOrErr = parseWeightedFile(WeightedFilename: SanitizedEntry);
1894 if (!WFOrErr)
1895 return WFOrErr.takeError();
1896 if (Error E = addWeightedInput(WNI&: WFV, WF: *WFOrErr))
1897 return E;
1898 }
1899 }
1900 return Error::success();
1901}
1902
1903static Error merge_main(StringRef ProgName) {
1904 WeightedFileVector WeightedInputs;
1905 for (StringRef Filename : InputFilenames)
1906 if (Error E = addWeightedInput(WNI&: WeightedInputs, WF: {.Filename: std::string(Filename), .Weight: 1}))
1907 return E;
1908 for (StringRef WeightedFilename : WeightedInputFilenames) {
1909 auto WFOrErr = parseWeightedFile(WeightedFilename);
1910 if (!WFOrErr)
1911 return WFOrErr.takeError();
1912 if (Error E = addWeightedInput(WNI&: WeightedInputs, WF: *WFOrErr))
1913 return E;
1914 }
1915
1916 // Make sure that the file buffer stays alive for the duration of the
1917 // weighted input vector's lifetime.
1918 auto BufferOrErr = getInputFileBuf(InputFile: InputFilenamesFile);
1919 if (!BufferOrErr)
1920 return BufferOrErr.takeError();
1921 auto Buffer = std::move(*BufferOrErr);
1922 if (Error E = parseInputFilenamesFile(Buffer: Buffer.get(), WFV&: WeightedInputs))
1923 return E;
1924
1925 if (WeightedInputs.empty())
1926 return makeError(Message: "no input files specified. See " + ProgName +
1927 " merge -help");
1928
1929 if (DumpInputFileList) {
1930 for (auto &WF : WeightedInputs)
1931 outs() << WF.Weight << "," << WF.Filename << "\n";
1932 return Error::success();
1933 }
1934
1935 std::unique_ptr<SymbolRemapper> Remapper;
1936 if (!RemappingFile.empty()) {
1937 auto RemapperOrErr = SymbolRemapper::create(InputFile: RemappingFile);
1938 if (!RemapperOrErr)
1939 return RemapperOrErr.takeError();
1940 Remapper = std::move(*RemapperOrErr);
1941 }
1942
1943 if (!SupplInstrWithSample.empty()) {
1944 if (ProfileKind != instr)
1945 return makeError(
1946 Message: "-supplement-instr-with-sample can only work with -instr. ");
1947
1948 return supplementInstrProfile(Inputs: WeightedInputs, SampleFilename: SupplInstrWithSample,
1949 OutputSparse, SupplMinSizeThreshold,
1950 ZeroCounterThreshold, InstrProfColdThreshold);
1951 }
1952
1953 if (ProfileKind == instr)
1954 return mergeInstrProfile(Inputs: WeightedInputs, Remapper: Remapper.get(),
1955 MaxDbgCorrelationWarnings, ProfiledBinary);
1956
1957 return mergeSampleProfile(Inputs: WeightedInputs, Remapper: Remapper.get(),
1958 ProfileSymbolListFile, OutputSizeLimit);
1959}
1960
1961/// Computer the overlap b/w profile BaseFilename and profile TestFilename.
1962static Error overlapInstrProfile(const std::string &BaseFilename,
1963 const std::string &TestFilename,
1964 const OverlapFuncFilters &FuncFilter,
1965 raw_fd_ostream &OS, bool IsCS) {
1966 std::mutex ErrorLock;
1967 SmallSet<instrprof_error, 4> WriterErrorCodes;
1968 WriterContext Context(false, ErrorLock, WriterErrorCodes);
1969 WeightedFile WeightedInput{.Filename: BaseFilename, .Weight: 1};
1970 OverlapStats Overlap;
1971 Error E = Overlap.accumulateCounts(BaseFilename, TestFilename, IsCS);
1972 if (E)
1973 return makeError(E: std::move(E), Whence: "error in getting profile count sums");
1974 if (Overlap.Base.CountSum < 1.0f) {
1975 OS << "Sum of edge counts for profile " << BaseFilename << " is 0.\n";
1976 return Error::success();
1977 }
1978 if (Overlap.Test.CountSum < 1.0f) {
1979 OS << "Sum of edge counts for profile " << TestFilename << " is 0.\n";
1980 return Error::success();
1981 }
1982 if (Error E = loadInput(Input: WeightedInput, Remapper: nullptr, Correlator: nullptr,
1983 /*ProfiledBinary=*/"", WC: &Context))
1984 return E;
1985 overlapInput(BaseFilename, TestFilename, WC: &Context, Overlap, FuncFilter, OS,
1986 IsCS);
1987 Overlap.dump(OS);
1988 return Error::success();
1989}
1990
1991namespace {
1992struct SampleOverlapStats {
1993 SampleContext BaseName;
1994 SampleContext TestName;
1995 // Number of overlap units
1996 uint64_t OverlapCount = 0;
1997 // Total samples of overlap units
1998 uint64_t OverlapSample = 0;
1999 // Number of and total samples of units that only present in base or test
2000 // profile
2001 uint64_t BaseUniqueCount = 0;
2002 uint64_t BaseUniqueSample = 0;
2003 uint64_t TestUniqueCount = 0;
2004 uint64_t TestUniqueSample = 0;
2005 // Number of units and total samples in base or test profile
2006 uint64_t BaseCount = 0;
2007 uint64_t BaseSample = 0;
2008 uint64_t TestCount = 0;
2009 uint64_t TestSample = 0;
2010 // Number of and total samples of units that present in at least one profile
2011 uint64_t UnionCount = 0;
2012 uint64_t UnionSample = 0;
2013 // Weighted similarity
2014 double Similarity = 0.0;
2015 // For SampleOverlapStats instances representing functions, weights of the
2016 // function in base and test profiles
2017 double BaseWeight = 0.0;
2018 double TestWeight = 0.0;
2019
2020 SampleOverlapStats() = default;
2021};
2022} // end anonymous namespace
2023
2024namespace {
2025struct FuncSampleStats {
2026 uint64_t SampleSum = 0;
2027 uint64_t MaxSample = 0;
2028 uint64_t HotBlockCount = 0;
2029 FuncSampleStats() = default;
2030 FuncSampleStats(uint64_t SampleSum, uint64_t MaxSample,
2031 uint64_t HotBlockCount)
2032 : SampleSum(SampleSum), MaxSample(MaxSample),
2033 HotBlockCount(HotBlockCount) {}
2034};
2035} // end anonymous namespace
2036
2037namespace {
2038enum MatchStatus { MS_Match, MS_FirstUnique, MS_SecondUnique, MS_None };
2039
2040// Class for updating merging steps for two sorted maps. The class should be
2041// instantiated with a map iterator type.
2042template <class T> class MatchStep {
2043public:
2044 MatchStep() = delete;
2045
2046 MatchStep(T FirstIter, T FirstEnd, T SecondIter, T SecondEnd)
2047 : FirstIter(FirstIter), FirstEnd(FirstEnd), SecondIter(SecondIter),
2048 SecondEnd(SecondEnd), Status(MS_None) {}
2049
2050 bool areBothFinished() const {
2051 return (FirstIter == FirstEnd && SecondIter == SecondEnd);
2052 }
2053
2054 bool isFirstFinished() const { return FirstIter == FirstEnd; }
2055
2056 bool isSecondFinished() const { return SecondIter == SecondEnd; }
2057
2058 /// Advance one step based on the previous match status unless the previous
2059 /// status is MS_None. Then update Status based on the comparison between two
2060 /// container iterators at the current step. If the previous status is
2061 /// MS_None, it means two iterators are at the beginning and no comparison has
2062 /// been made, so we simply update Status without advancing the iterators.
2063 void updateOneStep();
2064
2065 T getFirstIter() const { return FirstIter; }
2066
2067 T getSecondIter() const { return SecondIter; }
2068
2069 MatchStatus getMatchStatus() const { return Status; }
2070
2071private:
2072 // Current iterator and end iterator of the first container.
2073 T FirstIter;
2074 T FirstEnd;
2075 // Current iterator and end iterator of the second container.
2076 T SecondIter;
2077 T SecondEnd;
2078 // Match status of the current step.
2079 MatchStatus Status;
2080};
2081} // end anonymous namespace
2082
2083template <class T> void MatchStep<T>::updateOneStep() {
2084 switch (Status) {
2085 case MS_Match:
2086 ++FirstIter;
2087 ++SecondIter;
2088 break;
2089 case MS_FirstUnique:
2090 ++FirstIter;
2091 break;
2092 case MS_SecondUnique:
2093 ++SecondIter;
2094 break;
2095 case MS_None:
2096 break;
2097 }
2098
2099 // Update Status according to iterators at the current step.
2100 if (areBothFinished())
2101 return;
2102 if (FirstIter != FirstEnd &&
2103 (SecondIter == SecondEnd || FirstIter->first < SecondIter->first))
2104 Status = MS_FirstUnique;
2105 else if (SecondIter != SecondEnd &&
2106 (FirstIter == FirstEnd || SecondIter->first < FirstIter->first))
2107 Status = MS_SecondUnique;
2108 else
2109 Status = MS_Match;
2110}
2111
2112// Return the sum of line/block samples, the max line/block sample, and the
2113// number of line/block samples above the given threshold in a function
2114// including its inlinees.
2115static void getFuncSampleStats(const sampleprof::FunctionSamples &Func,
2116 FuncSampleStats &FuncStats,
2117 uint64_t HotThreshold) {
2118 for (const auto &L : Func.getBodySamples()) {
2119 uint64_t Sample = L.second.getSamples();
2120 FuncStats.SampleSum += Sample;
2121 FuncStats.MaxSample = std::max(a: FuncStats.MaxSample, b: Sample);
2122 if (Sample >= HotThreshold)
2123 ++FuncStats.HotBlockCount;
2124 }
2125
2126 for (const auto &C : Func.getCallsiteSamples()) {
2127 for (const auto &F : C.second)
2128 getFuncSampleStats(Func: F.second, FuncStats, HotThreshold);
2129 }
2130}
2131
2132/// Predicate that determines if a function is hot with a given threshold. We
2133/// keep it separate from its callsites for possible extension in the future.
2134static bool isFunctionHot(const FuncSampleStats &FuncStats,
2135 uint64_t HotThreshold) {
2136 // We intentionally compare the maximum sample count in a function with the
2137 // HotThreshold to get an approximate determination on hot functions.
2138 return (FuncStats.MaxSample >= HotThreshold);
2139}
2140
2141namespace {
2142class SampleOverlapAggregator {
2143public:
2144 SampleOverlapAggregator(const std::string &BaseFilename,
2145 const std::string &TestFilename,
2146 double LowSimilarityThreshold, double Epsilon,
2147 const OverlapFuncFilters &FuncFilter)
2148 : BaseFilename(BaseFilename), TestFilename(TestFilename),
2149 LowSimilarityThreshold(LowSimilarityThreshold), Epsilon(Epsilon),
2150 FuncFilter(FuncFilter) {}
2151
2152 /// Detect 0-sample input profile and report to output stream. This interface
2153 /// should be called after loadProfiles().
2154 bool detectZeroSampleProfile(raw_fd_ostream &OS) const;
2155
2156 /// Write out function-level similarity statistics for functions specified by
2157 /// options --function, --value-cutoff, and --similarity-cutoff.
2158 void dumpFuncSimilarity(raw_fd_ostream &OS) const;
2159
2160 /// Write out program-level similarity and overlap statistics.
2161 void dumpProgramSummary(raw_fd_ostream &OS) const;
2162
2163 /// Write out hot-function and hot-block statistics for base_profile,
2164 /// test_profile, and their overlap. For both cases, the overlap HO is
2165 /// calculated as follows:
2166 /// Given the number of functions (or blocks) that are hot in both profiles
2167 /// HCommon and the number of functions (or blocks) that are hot in at
2168 /// least one profile HUnion, HO = HCommon / HUnion.
2169 void dumpHotFuncAndBlockOverlap(raw_fd_ostream &OS) const;
2170
2171 /// This function tries matching functions in base and test profiles. For each
2172 /// pair of matched functions, it aggregates the function-level
2173 /// similarity into a profile-level similarity. It also dump function-level
2174 /// similarity information of functions specified by --function,
2175 /// --value-cutoff, and --similarity-cutoff options. The program-level
2176 /// similarity PS is computed as follows:
2177 /// Given function-level similarity FS(A) for all function A, the
2178 /// weight of function A in base profile WB(A), and the weight of function
2179 /// A in test profile WT(A), compute PS(base_profile, test_profile) =
2180 /// sum_A(FS(A) * avg(WB(A), WT(A))) ranging in [0.0f to 1.0f] with 0.0
2181 /// meaning no-overlap.
2182 void computeSampleProfileOverlap(raw_fd_ostream &OS);
2183
2184 /// Initialize ProfOverlap with the sum of samples in base and test
2185 /// profiles. This function also computes and keeps the sum of samples and
2186 /// max sample counts of each function in BaseStats and TestStats for later
2187 /// use to avoid re-computations.
2188 void initializeSampleProfileOverlap();
2189
2190 /// Load profiles specified by BaseFilename and TestFilename.
2191 Error loadProfiles();
2192
2193 using FuncSampleStatsMap = DenseMap<SampleContext, FuncSampleStats>;
2194
2195private:
2196 SampleOverlapStats ProfOverlap;
2197 SampleOverlapStats HotFuncOverlap;
2198 SampleOverlapStats HotBlockOverlap;
2199 std::string BaseFilename;
2200 std::string TestFilename;
2201 std::unique_ptr<sampleprof::SampleProfileReader> BaseReader;
2202 std::unique_ptr<sampleprof::SampleProfileReader> TestReader;
2203 // BaseStats and TestStats hold FuncSampleStats for each function, with
2204 // function name as the key.
2205 FuncSampleStatsMap BaseStats;
2206 FuncSampleStatsMap TestStats;
2207 // Low similarity threshold in floating point number
2208 double LowSimilarityThreshold;
2209 // Block samples above BaseHotThreshold or TestHotThreshold are considered hot
2210 // for tracking hot blocks.
2211 uint64_t BaseHotThreshold;
2212 uint64_t TestHotThreshold;
2213 // A small threshold used to round the results of floating point accumulations
2214 // to resolve imprecision.
2215 const double Epsilon;
2216 std::multimap<double, SampleOverlapStats, std::greater<double>>
2217 FuncSimilarityDump;
2218 // FuncFilter carries specifications in options --value-cutoff and
2219 // --function.
2220 OverlapFuncFilters FuncFilter;
2221 // Column offsets for printing the function-level details table.
2222 static const unsigned int TestWeightCol = 15;
2223 static const unsigned int SimilarityCol = 30;
2224 static const unsigned int OverlapCol = 43;
2225 static const unsigned int BaseUniqueCol = 53;
2226 static const unsigned int TestUniqueCol = 67;
2227 static const unsigned int BaseSampleCol = 81;
2228 static const unsigned int TestSampleCol = 96;
2229 static const unsigned int FuncNameCol = 111;
2230
2231 /// Return a similarity of two line/block sample counters in the same
2232 /// function in base and test profiles. The line/block-similarity BS(i) is
2233 /// computed as follows:
2234 /// For an offsets i, given the sample count at i in base profile BB(i),
2235 /// the sample count at i in test profile BT(i), the sum of sample counts
2236 /// in this function in base profile SB, and the sum of sample counts in
2237 /// this function in test profile ST, compute BS(i) = 1.0 - fabs(BB(i)/SB -
2238 /// BT(i)/ST), ranging in [0.0f to 1.0f] with 0.0 meaning no-overlap.
2239 double computeBlockSimilarity(uint64_t BaseSample, uint64_t TestSample,
2240 const SampleOverlapStats &FuncOverlap) const;
2241
2242 void updateHotBlockOverlap(uint64_t BaseSample, uint64_t TestSample,
2243 uint64_t HotBlockCount);
2244
2245 void getHotFunctions(const FuncSampleStatsMap &ProfStats,
2246 FuncSampleStatsMap &HotFunc,
2247 uint64_t HotThreshold) const;
2248
2249 void computeHotFuncOverlap();
2250
2251 /// This function updates statistics in FuncOverlap, HotBlockOverlap, and
2252 /// Difference for two sample units in a matched function according to the
2253 /// given match status.
2254 void updateOverlapStatsForFunction(uint64_t BaseSample, uint64_t TestSample,
2255 uint64_t HotBlockCount,
2256 SampleOverlapStats &FuncOverlap,
2257 double &Difference, MatchStatus Status);
2258
2259 /// This function updates statistics in FuncOverlap, HotBlockOverlap, and
2260 /// Difference for unmatched callees that only present in one profile in a
2261 /// matched caller function.
2262 void updateForUnmatchedCallee(const sampleprof::FunctionSamples &Func,
2263 SampleOverlapStats &FuncOverlap,
2264 double &Difference, MatchStatus Status);
2265
2266 /// This function updates sample overlap statistics of an overlap function in
2267 /// base and test profile. It also calculates a function-internal similarity
2268 /// FIS as follows:
2269 /// For offsets i that have samples in at least one profile in this
2270 /// function A, given BS(i) returned by computeBlockSimilarity(), compute
2271 /// FIS(A) = (2.0 - sum_i(1.0 - BS(i))) / 2, ranging in [0.0f to 1.0f] with
2272 /// 0.0 meaning no overlap.
2273 double computeSampleFunctionInternalOverlap(
2274 const sampleprof::FunctionSamples &BaseFunc,
2275 const sampleprof::FunctionSamples &TestFunc,
2276 SampleOverlapStats &FuncOverlap);
2277
2278 /// Function-level similarity (FS) is a weighted value over function internal
2279 /// similarity (FIS). This function computes a function's FS from its FIS by
2280 /// applying the weight.
2281 double weightForFuncSimilarity(double FuncSimilarity, uint64_t BaseFuncSample,
2282 uint64_t TestFuncSample) const;
2283
2284 /// The function-level similarity FS(A) for a function A is computed as
2285 /// follows:
2286 /// Compute a function-internal similarity FIS(A) by
2287 /// computeSampleFunctionInternalOverlap(). Then, with the weight of
2288 /// function A in base profile WB(A), and the weight of function A in test
2289 /// profile WT(A), compute FS(A) = FIS(A) * (1.0 - fabs(WB(A) - WT(A)))
2290 /// ranging in [0.0f to 1.0f] with 0.0 meaning no overlap.
2291 double
2292 computeSampleFunctionOverlap(const sampleprof::FunctionSamples *BaseFunc,
2293 const sampleprof::FunctionSamples *TestFunc,
2294 SampleOverlapStats *FuncOverlap,
2295 uint64_t BaseFuncSample,
2296 uint64_t TestFuncSample);
2297
2298 /// Profile-level similarity (PS) is a weighted aggregate over function-level
2299 /// similarities (FS). This method weights the FS value by the function
2300 /// weights in the base and test profiles for the aggregation.
2301 double weightByImportance(double FuncSimilarity, uint64_t BaseFuncSample,
2302 uint64_t TestFuncSample) const;
2303};
2304} // end anonymous namespace
2305
2306bool SampleOverlapAggregator::detectZeroSampleProfile(
2307 raw_fd_ostream &OS) const {
2308 bool HaveZeroSample = false;
2309 if (ProfOverlap.BaseSample == 0) {
2310 OS << "Sum of sample counts for profile " << BaseFilename << " is 0.\n";
2311 HaveZeroSample = true;
2312 }
2313 if (ProfOverlap.TestSample == 0) {
2314 OS << "Sum of sample counts for profile " << TestFilename << " is 0.\n";
2315 HaveZeroSample = true;
2316 }
2317 return HaveZeroSample;
2318}
2319
2320double SampleOverlapAggregator::computeBlockSimilarity(
2321 uint64_t BaseSample, uint64_t TestSample,
2322 const SampleOverlapStats &FuncOverlap) const {
2323 double BaseFrac = 0.0;
2324 double TestFrac = 0.0;
2325 if (FuncOverlap.BaseSample > 0)
2326 BaseFrac = static_cast<double>(BaseSample) / FuncOverlap.BaseSample;
2327 if (FuncOverlap.TestSample > 0)
2328 TestFrac = static_cast<double>(TestSample) / FuncOverlap.TestSample;
2329 return 1.0 - std::fabs(x: BaseFrac - TestFrac);
2330}
2331
2332void SampleOverlapAggregator::updateHotBlockOverlap(uint64_t BaseSample,
2333 uint64_t TestSample,
2334 uint64_t HotBlockCount) {
2335 bool IsBaseHot = (BaseSample >= BaseHotThreshold);
2336 bool IsTestHot = (TestSample >= TestHotThreshold);
2337 if (!IsBaseHot && !IsTestHot)
2338 return;
2339
2340 HotBlockOverlap.UnionCount += HotBlockCount;
2341 if (IsBaseHot)
2342 HotBlockOverlap.BaseCount += HotBlockCount;
2343 if (IsTestHot)
2344 HotBlockOverlap.TestCount += HotBlockCount;
2345 if (IsBaseHot && IsTestHot)
2346 HotBlockOverlap.OverlapCount += HotBlockCount;
2347}
2348
2349void SampleOverlapAggregator::getHotFunctions(
2350 const FuncSampleStatsMap &ProfStats, FuncSampleStatsMap &HotFunc,
2351 uint64_t HotThreshold) const {
2352 for (const auto &F : ProfStats) {
2353 if (isFunctionHot(FuncStats: F.second, HotThreshold))
2354 HotFunc.try_emplace(Key: F.first, Args: F.second);
2355 }
2356}
2357
2358void SampleOverlapAggregator::computeHotFuncOverlap() {
2359 FuncSampleStatsMap BaseHotFunc;
2360 getHotFunctions(ProfStats: BaseStats, HotFunc&: BaseHotFunc, HotThreshold: BaseHotThreshold);
2361 HotFuncOverlap.BaseCount = BaseHotFunc.size();
2362
2363 FuncSampleStatsMap TestHotFunc;
2364 getHotFunctions(ProfStats: TestStats, HotFunc&: TestHotFunc, HotThreshold: TestHotThreshold);
2365 HotFuncOverlap.TestCount = TestHotFunc.size();
2366 HotFuncOverlap.UnionCount = HotFuncOverlap.TestCount;
2367
2368 for (const auto &F : BaseHotFunc) {
2369 if (TestHotFunc.count(Val: F.first))
2370 ++HotFuncOverlap.OverlapCount;
2371 else
2372 ++HotFuncOverlap.UnionCount;
2373 }
2374}
2375
2376void SampleOverlapAggregator::updateOverlapStatsForFunction(
2377 uint64_t BaseSample, uint64_t TestSample, uint64_t HotBlockCount,
2378 SampleOverlapStats &FuncOverlap, double &Difference, MatchStatus Status) {
2379 assert(Status != MS_None &&
2380 "Match status should be updated before updating overlap statistics");
2381 if (Status == MS_FirstUnique) {
2382 TestSample = 0;
2383 FuncOverlap.BaseUniqueSample += BaseSample;
2384 } else if (Status == MS_SecondUnique) {
2385 BaseSample = 0;
2386 FuncOverlap.TestUniqueSample += TestSample;
2387 } else {
2388 ++FuncOverlap.OverlapCount;
2389 }
2390
2391 FuncOverlap.UnionSample += std::max(a: BaseSample, b: TestSample);
2392 FuncOverlap.OverlapSample += std::min(a: BaseSample, b: TestSample);
2393 Difference +=
2394 1.0 - computeBlockSimilarity(BaseSample, TestSample, FuncOverlap);
2395 updateHotBlockOverlap(BaseSample, TestSample, HotBlockCount);
2396}
2397
2398void SampleOverlapAggregator::updateForUnmatchedCallee(
2399 const sampleprof::FunctionSamples &Func, SampleOverlapStats &FuncOverlap,
2400 double &Difference, MatchStatus Status) {
2401 assert((Status == MS_FirstUnique || Status == MS_SecondUnique) &&
2402 "Status must be either of the two unmatched cases");
2403 FuncSampleStats FuncStats;
2404 if (Status == MS_FirstUnique) {
2405 getFuncSampleStats(Func, FuncStats, HotThreshold: BaseHotThreshold);
2406 updateOverlapStatsForFunction(BaseSample: FuncStats.SampleSum, TestSample: 0,
2407 HotBlockCount: FuncStats.HotBlockCount, FuncOverlap,
2408 Difference, Status);
2409 } else {
2410 getFuncSampleStats(Func, FuncStats, HotThreshold: TestHotThreshold);
2411 updateOverlapStatsForFunction(BaseSample: 0, TestSample: FuncStats.SampleSum,
2412 HotBlockCount: FuncStats.HotBlockCount, FuncOverlap,
2413 Difference, Status);
2414 }
2415}
2416
2417double SampleOverlapAggregator::computeSampleFunctionInternalOverlap(
2418 const sampleprof::FunctionSamples &BaseFunc,
2419 const sampleprof::FunctionSamples &TestFunc,
2420 SampleOverlapStats &FuncOverlap) {
2421
2422 using namespace sampleprof;
2423
2424 double Difference = 0;
2425
2426 // Accumulate Difference for regular line/block samples in the function.
2427 // We match them through sort-merge join algorithm because
2428 // FunctionSamples::getBodySamples() returns a map of sample counters ordered
2429 // by their offsets.
2430 MatchStep<BodySampleMap::const_iterator> BlockIterStep(
2431 BaseFunc.getBodySamples().cbegin(), BaseFunc.getBodySamples().cend(),
2432 TestFunc.getBodySamples().cbegin(), TestFunc.getBodySamples().cend());
2433 BlockIterStep.updateOneStep();
2434 while (!BlockIterStep.areBothFinished()) {
2435 uint64_t BaseSample =
2436 BlockIterStep.isFirstFinished()
2437 ? 0
2438 : BlockIterStep.getFirstIter()->second.getSamples();
2439 uint64_t TestSample =
2440 BlockIterStep.isSecondFinished()
2441 ? 0
2442 : BlockIterStep.getSecondIter()->second.getSamples();
2443 updateOverlapStatsForFunction(BaseSample, TestSample, HotBlockCount: 1, FuncOverlap,
2444 Difference, Status: BlockIterStep.getMatchStatus());
2445
2446 BlockIterStep.updateOneStep();
2447 }
2448
2449 // Accumulate Difference for callsite lines in the function. We match
2450 // them through sort-merge algorithm because
2451 // FunctionSamples::getCallsiteSamples() returns a map of callsite records
2452 // ordered by their offsets.
2453 MatchStep<CallsiteSampleMap::const_iterator> CallsiteIterStep(
2454 BaseFunc.getCallsiteSamples().cbegin(),
2455 BaseFunc.getCallsiteSamples().cend(),
2456 TestFunc.getCallsiteSamples().cbegin(),
2457 TestFunc.getCallsiteSamples().cend());
2458 CallsiteIterStep.updateOneStep();
2459 while (!CallsiteIterStep.areBothFinished()) {
2460 MatchStatus CallsiteStepStatus = CallsiteIterStep.getMatchStatus();
2461 assert(CallsiteStepStatus != MS_None &&
2462 "Match status should be updated before entering loop body");
2463
2464 if (CallsiteStepStatus != MS_Match) {
2465 auto Callsite = (CallsiteStepStatus == MS_FirstUnique)
2466 ? CallsiteIterStep.getFirstIter()
2467 : CallsiteIterStep.getSecondIter();
2468 for (const auto &F : Callsite->second)
2469 updateForUnmatchedCallee(Func: F.second, FuncOverlap, Difference,
2470 Status: CallsiteStepStatus);
2471 } else {
2472 // There may be multiple inlinees at the same offset, so we need to try
2473 // matching all of them. This match is implemented through sort-merge
2474 // algorithm because callsite records at the same offset are ordered by
2475 // function names.
2476 MatchStep<FunctionSamplesMap::const_iterator> CalleeIterStep(
2477 CallsiteIterStep.getFirstIter()->second.cbegin(),
2478 CallsiteIterStep.getFirstIter()->second.cend(),
2479 CallsiteIterStep.getSecondIter()->second.cbegin(),
2480 CallsiteIterStep.getSecondIter()->second.cend());
2481 CalleeIterStep.updateOneStep();
2482 while (!CalleeIterStep.areBothFinished()) {
2483 MatchStatus CalleeStepStatus = CalleeIterStep.getMatchStatus();
2484 if (CalleeStepStatus != MS_Match) {
2485 auto Callee = (CalleeStepStatus == MS_FirstUnique)
2486 ? CalleeIterStep.getFirstIter()
2487 : CalleeIterStep.getSecondIter();
2488 updateForUnmatchedCallee(Func: Callee->second, FuncOverlap, Difference,
2489 Status: CalleeStepStatus);
2490 } else {
2491 // An inlined function can contain other inlinees inside, so compute
2492 // the Difference recursively.
2493 Difference += 2.0 - 2 * computeSampleFunctionInternalOverlap(
2494 BaseFunc: CalleeIterStep.getFirstIter()->second,
2495 TestFunc: CalleeIterStep.getSecondIter()->second,
2496 FuncOverlap);
2497 }
2498 CalleeIterStep.updateOneStep();
2499 }
2500 }
2501 CallsiteIterStep.updateOneStep();
2502 }
2503
2504 // Difference reflects the total differences of line/block samples in this
2505 // function and ranges in [0.0f to 2.0f]. Take (2.0 - Difference) / 2 to
2506 // reflect the similarity between function profiles in [0.0f to 1.0f].
2507 return (2.0 - Difference) / 2;
2508}
2509
2510double SampleOverlapAggregator::weightForFuncSimilarity(
2511 double FuncInternalSimilarity, uint64_t BaseFuncSample,
2512 uint64_t TestFuncSample) const {
2513 // Compute the weight as the distance between the function weights in two
2514 // profiles.
2515 double BaseFrac = 0.0;
2516 double TestFrac = 0.0;
2517 assert(ProfOverlap.BaseSample > 0 &&
2518 "Total samples in base profile should be greater than 0");
2519 BaseFrac = static_cast<double>(BaseFuncSample) / ProfOverlap.BaseSample;
2520 assert(ProfOverlap.TestSample > 0 &&
2521 "Total samples in test profile should be greater than 0");
2522 TestFrac = static_cast<double>(TestFuncSample) / ProfOverlap.TestSample;
2523 double WeightDistance = std::fabs(x: BaseFrac - TestFrac);
2524
2525 // Take WeightDistance into the similarity.
2526 return FuncInternalSimilarity * (1 - WeightDistance);
2527}
2528
2529double
2530SampleOverlapAggregator::weightByImportance(double FuncSimilarity,
2531 uint64_t BaseFuncSample,
2532 uint64_t TestFuncSample) const {
2533
2534 double BaseFrac = 0.0;
2535 double TestFrac = 0.0;
2536 assert(ProfOverlap.BaseSample > 0 &&
2537 "Total samples in base profile should be greater than 0");
2538 BaseFrac = static_cast<double>(BaseFuncSample) / ProfOverlap.BaseSample / 2.0;
2539 assert(ProfOverlap.TestSample > 0 &&
2540 "Total samples in test profile should be greater than 0");
2541 TestFrac = static_cast<double>(TestFuncSample) / ProfOverlap.TestSample / 2.0;
2542 return FuncSimilarity * (BaseFrac + TestFrac);
2543}
2544
2545double SampleOverlapAggregator::computeSampleFunctionOverlap(
2546 const sampleprof::FunctionSamples *BaseFunc,
2547 const sampleprof::FunctionSamples *TestFunc,
2548 SampleOverlapStats *FuncOverlap, uint64_t BaseFuncSample,
2549 uint64_t TestFuncSample) {
2550 // Default function internal similarity before weighted, meaning two functions
2551 // has no overlap.
2552 const double DefaultFuncInternalSimilarity = 0;
2553 double FuncSimilarity;
2554 double FuncInternalSimilarity;
2555
2556 // If BaseFunc or TestFunc is nullptr, it means the functions do not overlap.
2557 // In this case, we use DefaultFuncInternalSimilarity as the function internal
2558 // similarity.
2559 if (!BaseFunc || !TestFunc) {
2560 FuncInternalSimilarity = DefaultFuncInternalSimilarity;
2561 } else {
2562 assert(FuncOverlap != nullptr &&
2563 "FuncOverlap should be provided in this case");
2564 FuncInternalSimilarity = computeSampleFunctionInternalOverlap(
2565 BaseFunc: *BaseFunc, TestFunc: *TestFunc, FuncOverlap&: *FuncOverlap);
2566 // Now, FuncInternalSimilarity may be a little less than 0 due to
2567 // imprecision of floating point accumulations. Make it zero if the
2568 // difference is below Epsilon.
2569 FuncInternalSimilarity = (std::fabs(x: FuncInternalSimilarity - 0) < Epsilon)
2570 ? 0
2571 : FuncInternalSimilarity;
2572 }
2573 FuncSimilarity = weightForFuncSimilarity(FuncInternalSimilarity,
2574 BaseFuncSample, TestFuncSample);
2575 return FuncSimilarity;
2576}
2577
2578void SampleOverlapAggregator::computeSampleProfileOverlap(raw_fd_ostream &OS) {
2579 using namespace sampleprof;
2580
2581 DenseMap<SampleContext, const FunctionSamples *> BaseFuncProf;
2582 const auto &BaseProfiles = BaseReader->getProfiles();
2583 for (const auto &BaseFunc : BaseProfiles) {
2584 BaseFuncProf.try_emplace(Key: BaseFunc.second.getContext(), Args: &(BaseFunc.second));
2585 }
2586 ProfOverlap.UnionCount = BaseFuncProf.size();
2587
2588 const auto &TestProfiles = TestReader->getProfiles();
2589 for (const auto &TestFunc : TestProfiles) {
2590 SampleOverlapStats FuncOverlap;
2591 FuncOverlap.TestName = TestFunc.second.getContext();
2592 assert(TestStats.count(FuncOverlap.TestName) &&
2593 "TestStats should have records for all functions in test profile "
2594 "except inlinees");
2595 FuncOverlap.TestSample = TestStats[FuncOverlap.TestName].SampleSum;
2596
2597 bool Matched = false;
2598 const auto Match = BaseFuncProf.find(Val: FuncOverlap.TestName);
2599 if (Match == BaseFuncProf.end()) {
2600 const FuncSampleStats &FuncStats = TestStats[FuncOverlap.TestName];
2601 ++ProfOverlap.TestUniqueCount;
2602 ProfOverlap.TestUniqueSample += FuncStats.SampleSum;
2603 FuncOverlap.TestUniqueSample = FuncStats.SampleSum;
2604
2605 updateHotBlockOverlap(BaseSample: 0, TestSample: FuncStats.SampleSum, HotBlockCount: FuncStats.HotBlockCount);
2606
2607 double FuncSimilarity = computeSampleFunctionOverlap(
2608 BaseFunc: nullptr, TestFunc: nullptr, FuncOverlap: nullptr, BaseFuncSample: 0, TestFuncSample: FuncStats.SampleSum);
2609 ProfOverlap.Similarity +=
2610 weightByImportance(FuncSimilarity, BaseFuncSample: 0, TestFuncSample: FuncStats.SampleSum);
2611
2612 ++ProfOverlap.UnionCount;
2613 ProfOverlap.UnionSample += FuncStats.SampleSum;
2614 } else {
2615 ++ProfOverlap.OverlapCount;
2616
2617 // Two functions match with each other. Compute function-level overlap and
2618 // aggregate them into profile-level overlap.
2619 FuncOverlap.BaseName = Match->second->getContext();
2620 assert(BaseStats.count(FuncOverlap.BaseName) &&
2621 "BaseStats should have records for all functions in base profile "
2622 "except inlinees");
2623 FuncOverlap.BaseSample = BaseStats[FuncOverlap.BaseName].SampleSum;
2624
2625 FuncOverlap.Similarity = computeSampleFunctionOverlap(
2626 BaseFunc: Match->second, TestFunc: &TestFunc.second, FuncOverlap: &FuncOverlap, BaseFuncSample: FuncOverlap.BaseSample,
2627 TestFuncSample: FuncOverlap.TestSample);
2628 ProfOverlap.Similarity +=
2629 weightByImportance(FuncSimilarity: FuncOverlap.Similarity, BaseFuncSample: FuncOverlap.BaseSample,
2630 TestFuncSample: FuncOverlap.TestSample);
2631 ProfOverlap.OverlapSample += FuncOverlap.OverlapSample;
2632 ProfOverlap.UnionSample += FuncOverlap.UnionSample;
2633
2634 // Accumulate the percentage of base unique and test unique samples into
2635 // ProfOverlap.
2636 ProfOverlap.BaseUniqueSample += FuncOverlap.BaseUniqueSample;
2637 ProfOverlap.TestUniqueSample += FuncOverlap.TestUniqueSample;
2638
2639 // Remove matched base functions for later reporting functions not found
2640 // in test profile.
2641 BaseFuncProf.erase(I: Match);
2642 Matched = true;
2643 }
2644
2645 // Print function-level similarity information if specified by options.
2646 assert(TestStats.count(FuncOverlap.TestName) &&
2647 "TestStats should have records for all functions in test profile "
2648 "except inlinees");
2649 if (TestStats[FuncOverlap.TestName].MaxSample >= FuncFilter.ValueCutoff ||
2650 (Matched && FuncOverlap.Similarity < LowSimilarityThreshold) ||
2651 (Matched && !FuncFilter.NameFilter.empty() &&
2652 FuncOverlap.BaseName.toString().find(str: FuncFilter.NameFilter) !=
2653 std::string::npos)) {
2654 assert(ProfOverlap.BaseSample > 0 &&
2655 "Total samples in base profile should be greater than 0");
2656 FuncOverlap.BaseWeight =
2657 static_cast<double>(FuncOverlap.BaseSample) / ProfOverlap.BaseSample;
2658 assert(ProfOverlap.TestSample > 0 &&
2659 "Total samples in test profile should be greater than 0");
2660 FuncOverlap.TestWeight =
2661 static_cast<double>(FuncOverlap.TestSample) / ProfOverlap.TestSample;
2662 FuncSimilarityDump.emplace(args&: FuncOverlap.BaseWeight, args&: FuncOverlap);
2663 }
2664 }
2665
2666 // Traverse through functions in base profile but not in test profile.
2667 for (const auto &F : BaseFuncProf) {
2668 assert(BaseStats.count(F.second->getContext()) &&
2669 "BaseStats should have records for all functions in base profile "
2670 "except inlinees");
2671 const FuncSampleStats &FuncStats = BaseStats[F.second->getContext()];
2672 ++ProfOverlap.BaseUniqueCount;
2673 ProfOverlap.BaseUniqueSample += FuncStats.SampleSum;
2674
2675 updateHotBlockOverlap(BaseSample: FuncStats.SampleSum, TestSample: 0, HotBlockCount: FuncStats.HotBlockCount);
2676
2677 double FuncSimilarity = computeSampleFunctionOverlap(
2678 BaseFunc: nullptr, TestFunc: nullptr, FuncOverlap: nullptr, BaseFuncSample: FuncStats.SampleSum, TestFuncSample: 0);
2679 ProfOverlap.Similarity +=
2680 weightByImportance(FuncSimilarity, BaseFuncSample: FuncStats.SampleSum, TestFuncSample: 0);
2681
2682 ProfOverlap.UnionSample += FuncStats.SampleSum;
2683 }
2684
2685 // Now, ProfSimilarity may be a little greater than 1 due to imprecision
2686 // of floating point accumulations. Make it 1.0 if the difference is below
2687 // Epsilon.
2688 ProfOverlap.Similarity = (std::fabs(x: ProfOverlap.Similarity - 1) < Epsilon)
2689 ? 1
2690 : ProfOverlap.Similarity;
2691
2692 computeHotFuncOverlap();
2693}
2694
2695void SampleOverlapAggregator::initializeSampleProfileOverlap() {
2696 const auto &BaseProf = BaseReader->getProfiles();
2697 for (const auto &I : BaseProf) {
2698 ++ProfOverlap.BaseCount;
2699 FuncSampleStats FuncStats;
2700 getFuncSampleStats(Func: I.second, FuncStats, HotThreshold: BaseHotThreshold);
2701 ProfOverlap.BaseSample += FuncStats.SampleSum;
2702 BaseStats.try_emplace(Key: I.second.getContext(), Args&: FuncStats);
2703 }
2704
2705 const auto &TestProf = TestReader->getProfiles();
2706 for (const auto &I : TestProf) {
2707 ++ProfOverlap.TestCount;
2708 FuncSampleStats FuncStats;
2709 getFuncSampleStats(Func: I.second, FuncStats, HotThreshold: TestHotThreshold);
2710 ProfOverlap.TestSample += FuncStats.SampleSum;
2711 TestStats.try_emplace(Key: I.second.getContext(), Args&: FuncStats);
2712 }
2713
2714 ProfOverlap.BaseName = StringRef(BaseFilename);
2715 ProfOverlap.TestName = StringRef(TestFilename);
2716}
2717
2718void SampleOverlapAggregator::dumpFuncSimilarity(raw_fd_ostream &OS) const {
2719 using namespace sampleprof;
2720
2721 if (FuncSimilarityDump.empty())
2722 return;
2723
2724 formatted_raw_ostream FOS(OS);
2725 FOS << "Function-level details:\n";
2726 FOS << "Base weight";
2727 FOS.PadToColumn(NewCol: TestWeightCol);
2728 FOS << "Test weight";
2729 FOS.PadToColumn(NewCol: SimilarityCol);
2730 FOS << "Similarity";
2731 FOS.PadToColumn(NewCol: OverlapCol);
2732 FOS << "Overlap";
2733 FOS.PadToColumn(NewCol: BaseUniqueCol);
2734 FOS << "Base unique";
2735 FOS.PadToColumn(NewCol: TestUniqueCol);
2736 FOS << "Test unique";
2737 FOS.PadToColumn(NewCol: BaseSampleCol);
2738 FOS << "Base samples";
2739 FOS.PadToColumn(NewCol: TestSampleCol);
2740 FOS << "Test samples";
2741 FOS.PadToColumn(NewCol: FuncNameCol);
2742 FOS << "Function name\n";
2743 for (const auto &F : FuncSimilarityDump) {
2744 double OverlapPercent =
2745 F.second.UnionSample > 0
2746 ? static_cast<double>(F.second.OverlapSample) / F.second.UnionSample
2747 : 0;
2748 double BaseUniquePercent =
2749 F.second.BaseSample > 0
2750 ? static_cast<double>(F.second.BaseUniqueSample) /
2751 F.second.BaseSample
2752 : 0;
2753 double TestUniquePercent =
2754 F.second.TestSample > 0
2755 ? static_cast<double>(F.second.TestUniqueSample) /
2756 F.second.TestSample
2757 : 0;
2758
2759 FOS << format(Fmt: "%.2f%%", Vals: F.second.BaseWeight * 100);
2760 FOS.PadToColumn(NewCol: TestWeightCol);
2761 FOS << format(Fmt: "%.2f%%", Vals: F.second.TestWeight * 100);
2762 FOS.PadToColumn(NewCol: SimilarityCol);
2763 FOS << format(Fmt: "%.2f%%", Vals: F.second.Similarity * 100);
2764 FOS.PadToColumn(NewCol: OverlapCol);
2765 FOS << format(Fmt: "%.2f%%", Vals: OverlapPercent * 100);
2766 FOS.PadToColumn(NewCol: BaseUniqueCol);
2767 FOS << format(Fmt: "%.2f%%", Vals: BaseUniquePercent * 100);
2768 FOS.PadToColumn(NewCol: TestUniqueCol);
2769 FOS << format(Fmt: "%.2f%%", Vals: TestUniquePercent * 100);
2770 FOS.PadToColumn(NewCol: BaseSampleCol);
2771 FOS << F.second.BaseSample;
2772 FOS.PadToColumn(NewCol: TestSampleCol);
2773 FOS << F.second.TestSample;
2774 FOS.PadToColumn(NewCol: FuncNameCol);
2775 FOS << F.second.TestName.toString() << "\n";
2776 }
2777}
2778
2779void SampleOverlapAggregator::dumpProgramSummary(raw_fd_ostream &OS) const {
2780 OS << "Profile overlap information for base_profile: "
2781 << ProfOverlap.BaseName.toString()
2782 << " and test_profile: " << ProfOverlap.TestName.toString()
2783 << "\nProgram level:\n";
2784
2785 OS << " Whole program profile similarity: "
2786 << format(Fmt: "%.3f%%", Vals: ProfOverlap.Similarity * 100) << "\n";
2787
2788 assert(ProfOverlap.UnionSample > 0 &&
2789 "Total samples in two profile should be greater than 0");
2790 double OverlapPercent =
2791 static_cast<double>(ProfOverlap.OverlapSample) / ProfOverlap.UnionSample;
2792 assert(ProfOverlap.BaseSample > 0 &&
2793 "Total samples in base profile should be greater than 0");
2794 double BaseUniquePercent = static_cast<double>(ProfOverlap.BaseUniqueSample) /
2795 ProfOverlap.BaseSample;
2796 assert(ProfOverlap.TestSample > 0 &&
2797 "Total samples in test profile should be greater than 0");
2798 double TestUniquePercent = static_cast<double>(ProfOverlap.TestUniqueSample) /
2799 ProfOverlap.TestSample;
2800
2801 OS << " Whole program sample overlap: "
2802 << format(Fmt: "%.3f%%", Vals: OverlapPercent * 100) << "\n";
2803 OS << " percentage of samples unique in base profile: "
2804 << format(Fmt: "%.3f%%", Vals: BaseUniquePercent * 100) << "\n";
2805 OS << " percentage of samples unique in test profile: "
2806 << format(Fmt: "%.3f%%", Vals: TestUniquePercent * 100) << "\n";
2807 OS << " total samples in base profile: " << ProfOverlap.BaseSample << "\n"
2808 << " total samples in test profile: " << ProfOverlap.TestSample << "\n";
2809
2810 assert(ProfOverlap.UnionCount > 0 &&
2811 "There should be at least one function in two input profiles");
2812 double FuncOverlapPercent =
2813 static_cast<double>(ProfOverlap.OverlapCount) / ProfOverlap.UnionCount;
2814 OS << " Function overlap: " << format(Fmt: "%.3f%%", Vals: FuncOverlapPercent * 100)
2815 << "\n";
2816 OS << " overlap functions: " << ProfOverlap.OverlapCount << "\n";
2817 OS << " functions unique in base profile: " << ProfOverlap.BaseUniqueCount
2818 << "\n";
2819 OS << " functions unique in test profile: " << ProfOverlap.TestUniqueCount
2820 << "\n";
2821}
2822
2823void SampleOverlapAggregator::dumpHotFuncAndBlockOverlap(
2824 raw_fd_ostream &OS) const {
2825 assert(HotFuncOverlap.UnionCount > 0 &&
2826 "There should be at least one hot function in two input profiles");
2827 OS << " Hot-function overlap: "
2828 << format(Fmt: "%.3f%%", Vals: static_cast<double>(HotFuncOverlap.OverlapCount) /
2829 HotFuncOverlap.UnionCount * 100)
2830 << "\n";
2831 OS << " overlap hot functions: " << HotFuncOverlap.OverlapCount << "\n";
2832 OS << " hot functions unique in base profile: "
2833 << HotFuncOverlap.BaseCount - HotFuncOverlap.OverlapCount << "\n";
2834 OS << " hot functions unique in test profile: "
2835 << HotFuncOverlap.TestCount - HotFuncOverlap.OverlapCount << "\n";
2836
2837 assert(HotBlockOverlap.UnionCount > 0 &&
2838 "There should be at least one hot block in two input profiles");
2839 OS << " Hot-block overlap: "
2840 << format(Fmt: "%.3f%%", Vals: static_cast<double>(HotBlockOverlap.OverlapCount) /
2841 HotBlockOverlap.UnionCount * 100)
2842 << "\n";
2843 OS << " overlap hot blocks: " << HotBlockOverlap.OverlapCount << "\n";
2844 OS << " hot blocks unique in base profile: "
2845 << HotBlockOverlap.BaseCount - HotBlockOverlap.OverlapCount << "\n";
2846 OS << " hot blocks unique in test profile: "
2847 << HotBlockOverlap.TestCount - HotBlockOverlap.OverlapCount << "\n";
2848}
2849
2850Error SampleOverlapAggregator::loadProfiles() {
2851 using namespace sampleprof;
2852
2853 LLVMContext Context;
2854 auto FS = vfs::getRealFileSystem();
2855 auto BaseReaderOrErr = SampleProfileReader::create(Filename: BaseFilename, C&: Context, FS&: *FS,
2856 P: FSDiscriminatorPassOption);
2857 if (std::error_code EC = BaseReaderOrErr.getError())
2858 return makeError(EC, Whence: BaseFilename);
2859
2860 auto TestReaderOrErr = SampleProfileReader::create(Filename: TestFilename, C&: Context, FS&: *FS,
2861 P: FSDiscriminatorPassOption);
2862 if (std::error_code EC = TestReaderOrErr.getError())
2863 return makeError(EC, Whence: TestFilename);
2864
2865 BaseReader = std::move(BaseReaderOrErr.get());
2866 TestReader = std::move(TestReaderOrErr.get());
2867
2868 if (std::error_code EC = BaseReader->read())
2869 return makeError(EC, Whence: BaseFilename);
2870 if (std::error_code EC = TestReader->read())
2871 return makeError(EC, Whence: TestFilename);
2872 if (BaseReader->profileIsProbeBased() != TestReader->profileIsProbeBased())
2873 return makeError(
2874 Message: "cannot compare probe-based profile with non-probe-based profile");
2875 if (BaseReader->profileIsCS() != TestReader->profileIsCS())
2876 return makeError(Message: "cannot compare CS profile with non-CS profile");
2877
2878 // Load BaseHotThreshold and TestHotThreshold as 99-percentile threshold in
2879 // profile summary.
2880 ProfileSummary &BasePS = BaseReader->getSummary();
2881 ProfileSummary &TestPS = TestReader->getSummary();
2882 BaseHotThreshold =
2883 ProfileSummaryBuilder::getHotCountThreshold(DS: BasePS.getDetailedSummary());
2884 TestHotThreshold =
2885 ProfileSummaryBuilder::getHotCountThreshold(DS: TestPS.getDetailedSummary());
2886
2887 return Error::success();
2888}
2889
2890static Error overlapSampleProfile(const std::string &BaseFilename,
2891 const std::string &TestFilename,
2892 const OverlapFuncFilters &FuncFilter,
2893 uint64_t SimilarityCutoff,
2894 raw_fd_ostream &OS) {
2895 using namespace sampleprof;
2896
2897 // We use 0.000005 to initialize OverlapAggr.Epsilon because the final metrics
2898 // report 2--3 places after decimal point in percentage numbers.
2899 SampleOverlapAggregator OverlapAggr(
2900 BaseFilename, TestFilename,
2901 static_cast<double>(SimilarityCutoff) / 1000000, 0.000005, FuncFilter);
2902 if (Error E = OverlapAggr.loadProfiles())
2903 return E;
2904
2905 OverlapAggr.initializeSampleProfileOverlap();
2906 if (OverlapAggr.detectZeroSampleProfile(OS))
2907 return Error::success();
2908
2909 OverlapAggr.computeSampleProfileOverlap(OS);
2910
2911 OverlapAggr.dumpProgramSummary(OS);
2912 OverlapAggr.dumpHotFuncAndBlockOverlap(OS);
2913 OverlapAggr.dumpFuncSimilarity(OS);
2914 return Error::success();
2915}
2916
2917static Error overlap_main() {
2918 std::error_code EC;
2919 raw_fd_ostream OS(OutputFilename.data(), EC, sys::fs::OF_TextWithCRLF);
2920 if (EC)
2921 return makeError(EC, Whence: OutputFilename);
2922
2923 if (ProfileKind == instr)
2924 return overlapInstrProfile(
2925 BaseFilename, TestFilename,
2926 FuncFilter: OverlapFuncFilters{.ValueCutoff: OverlapValueCutoff, .NameFilter: FuncNameFilter}, OS, IsCS);
2927
2928 return overlapSampleProfile(
2929 BaseFilename, TestFilename,
2930 FuncFilter: OverlapFuncFilters{.ValueCutoff: OverlapValueCutoff, .NameFilter: FuncNameFilter}, SimilarityCutoff,
2931 OS);
2932}
2933
2934namespace {
2935struct ValueSitesStats {
2936 ValueSitesStats() = default;
2937 uint64_t TotalNumValueSites = 0;
2938 uint64_t TotalNumValueSitesWithValueProfile = 0;
2939 uint64_t TotalNumValues = 0;
2940 std::vector<unsigned> ValueSitesHistogram;
2941};
2942} // namespace
2943
2944static void traverseAllValueSites(const InstrProfRecord &Func, uint32_t VK,
2945 ValueSitesStats &Stats, raw_fd_ostream &OS,
2946 InstrProfSymtab *Symtab) {
2947 uint32_t NS = Func.getNumValueSites(ValueKind: VK);
2948 Stats.TotalNumValueSites += NS;
2949 for (size_t I = 0; I < NS; ++I) {
2950 auto VD = Func.getValueArrayForSite(ValueKind: VK, Site: I);
2951 uint32_t NV = VD.size();
2952 if (NV == 0)
2953 continue;
2954 Stats.TotalNumValues += NV;
2955 Stats.TotalNumValueSitesWithValueProfile++;
2956 if (NV > Stats.ValueSitesHistogram.size())
2957 Stats.ValueSitesHistogram.resize(new_size: NV, x: 0);
2958 Stats.ValueSitesHistogram[NV - 1]++;
2959
2960 uint64_t SiteSum = 0;
2961 for (const auto &V : VD)
2962 SiteSum += V.Count;
2963 if (SiteSum == 0)
2964 SiteSum = 1;
2965
2966 for (const auto &V : VD) {
2967 OS << "\t[ " << format(Fmt: "%2u", Vals: I) << ", ";
2968 if (Symtab == nullptr)
2969 OS << format(Fmt: "%4" PRIu64, Vals: V.Value);
2970 else
2971 OS << Symtab->getFuncOrVarName(MD5Hash: V.Value);
2972 OS << ", " << format(Fmt: "%10" PRId64, Vals: V.Count) << " ] ("
2973 << format(Fmt: "%.2f%%", Vals: (V.Count * 100.0 / SiteSum)) << ")\n";
2974 }
2975 }
2976}
2977
2978static void showValueSitesStats(raw_fd_ostream &OS, uint32_t VK,
2979 ValueSitesStats &Stats) {
2980 OS << " Total number of sites: " << Stats.TotalNumValueSites << "\n";
2981 OS << " Total number of sites with values: "
2982 << Stats.TotalNumValueSitesWithValueProfile << "\n";
2983 OS << " Total number of profiled values: " << Stats.TotalNumValues << "\n";
2984
2985 OS << " Value sites histogram:\n\tNumTargets, SiteCount\n";
2986 for (unsigned I = 0; I < Stats.ValueSitesHistogram.size(); I++) {
2987 if (Stats.ValueSitesHistogram[I] > 0)
2988 OS << "\t" << I + 1 << ", " << Stats.ValueSitesHistogram[I] << "\n";
2989 }
2990}
2991
2992static Error showInstrProfile(ShowFormat SFormat, raw_fd_ostream &OS) {
2993 if (SFormat == ShowFormat::Json)
2994 return makeError(Message: "JSON output is not supported for instr profiles");
2995 if (SFormat == ShowFormat::Yaml)
2996 return makeError(Message: "YAML output is not supported for instr profiles");
2997 auto FS = vfs::getRealFileSystem();
2998 auto ReaderOrErr = InstrProfReader::create(Path: Filename, FS&: *FS);
2999 std::vector<uint32_t> Cutoffs = std::move(DetailedSummaryCutoffs);
3000 if (Cutoffs.empty() && (ShowDetailedSummary || ShowHotFuncList))
3001 Cutoffs = ProfileSummaryBuilder::DefaultCutoffs;
3002 InstrProfSummaryBuilder Builder(std::move(Cutoffs));
3003 if (Error E = ReaderOrErr.takeError())
3004 return makeError(E: std::move(E), Whence: Filename);
3005
3006 auto Reader = std::move(ReaderOrErr.get());
3007 bool IsIRInstr = Reader->isIRLevelProfile();
3008 size_t ShownFunctions = 0;
3009 size_t BelowCutoffFunctions = 0;
3010 int NumVPKind = IPVK_Last - IPVK_First + 1;
3011 std::vector<ValueSitesStats> VPStats(NumVPKind);
3012
3013 std::vector<std::pair<StringRef, uint64_t>> NameAndMaxCount;
3014
3015 if (!TextFormat && OnlyListBelow) {
3016 OS << "The list of functions with the maximum counter less than "
3017 << ShowValueCutoff << ":\n";
3018 }
3019
3020 // Add marker so that IR-level instrumentation round-trips properly.
3021 if (TextFormat && IsIRInstr)
3022 OS << ":ir\n";
3023
3024 for (const auto &Func : *Reader) {
3025 if (Reader->isIRLevelProfile()) {
3026 bool FuncIsCS = NamedInstrProfRecord::hasCSFlagInHash(FuncHash: Func.Hash);
3027 if (FuncIsCS != ShowCS)
3028 continue;
3029 }
3030 bool Show = ShowAllFunctions ||
3031 (!FuncNameFilter.empty() && Func.Name.contains(Other: FuncNameFilter));
3032
3033 bool doTextFormatDump = (Show && TextFormat);
3034
3035 if (doTextFormatDump) {
3036 InstrProfSymtab &Symtab = Reader->getSymtab();
3037 InstrProfWriter::writeRecordInText(Name: Func.Name, Hash: Func.Hash, Counters: Func, Symtab,
3038 OS);
3039 continue;
3040 }
3041
3042 assert(Func.Counts.size() > 0 && "function missing entry counter");
3043 Builder.addRecord(Func);
3044
3045 if (ShowCovered) {
3046 if (llvm::any_of(Range: Func.Counts, P: [](uint64_t C) { return C; }))
3047 OS << Func.Name << "\n";
3048 continue;
3049 }
3050
3051 uint64_t FuncMax = 0;
3052 uint64_t FuncSum = 0;
3053
3054 auto PseudoKind = Func.getCountPseudoKind();
3055 if (PseudoKind != InstrProfRecord::NotPseudo) {
3056 if (Show) {
3057 if (!ShownFunctions)
3058 OS << "Counters:\n";
3059 ++ShownFunctions;
3060 OS << " " << Func.Name << ":\n"
3061 << " Hash: " << format(Fmt: "0x%016" PRIx64, Vals: Func.Hash) << "\n"
3062 << " Counters: " << Func.Counts.size();
3063 if (PseudoKind == InstrProfRecord::PseudoHot)
3064 OS << " <PseudoHot>\n";
3065 else if (PseudoKind == InstrProfRecord::PseudoWarm)
3066 OS << " <PseudoWarm>\n";
3067 else
3068 llvm_unreachable("Unknown PseudoKind");
3069 }
3070 continue;
3071 }
3072
3073 for (uint64_t Count : Func.Counts) {
3074 FuncMax = std::max(a: FuncMax, b: Count);
3075 FuncSum += Count;
3076 }
3077
3078 if (FuncMax < ShowValueCutoff) {
3079 ++BelowCutoffFunctions;
3080 if (OnlyListBelow) {
3081 OS << " " << Func.Name << ": (Max = " << FuncMax
3082 << " Sum = " << FuncSum << ")\n";
3083 }
3084 continue;
3085 } else if (OnlyListBelow)
3086 continue;
3087
3088 if (TopNFunctions || ShowHotFuncList)
3089 NameAndMaxCount.emplace_back(args: Func.Name, args&: FuncMax);
3090
3091 if (Show) {
3092 if (!ShownFunctions)
3093 OS << "Counters:\n";
3094
3095 ++ShownFunctions;
3096
3097 OS << " " << Func.Name << ":\n"
3098 << " Hash: " << format(Fmt: "0x%016" PRIx64, Vals: Func.Hash) << "\n"
3099 << " Counters: " << Func.Counts.size() << "\n";
3100 if (!IsIRInstr)
3101 OS << " Function count: " << Func.Counts[0] << "\n";
3102
3103 if (ShowIndirectCallTargets)
3104 OS << " Indirect Call Site Count: "
3105 << Func.getNumValueSites(ValueKind: IPVK_IndirectCallTarget) << "\n";
3106
3107 if (ShowVTables)
3108 OS << " Number of instrumented vtables: "
3109 << Func.getNumValueSites(ValueKind: IPVK_VTableTarget) << "\n";
3110
3111 uint32_t NumMemOPCalls = Func.getNumValueSites(ValueKind: IPVK_MemOPSize);
3112 if (ShowMemOPSizes && NumMemOPCalls > 0)
3113 OS << " Number of Memory Intrinsics Calls: " << NumMemOPCalls
3114 << "\n";
3115
3116 if (ShowCounts) {
3117 OS << " Block counts: [";
3118 size_t Start = (IsIRInstr ? 0 : 1);
3119 for (size_t I = Start, E = Func.Counts.size(); I < E; ++I) {
3120 OS << (I == Start ? "" : ", ") << Func.Counts[I];
3121 }
3122 OS << "]\n";
3123
3124 // Show uniformity bits if present
3125 if (!Func.UniformityBits.empty()) {
3126 OS << " Block uniformity: [";
3127 for (size_t I = Start, E = Func.Counts.size(); I < E; ++I) {
3128 bool IsUniform = Func.isBlockUniform(BlockIdx: I);
3129 OS << (I == Start ? "" : ", ") << (IsUniform ? "U" : "D");
3130 }
3131 OS << "]\n";
3132 }
3133 }
3134
3135 if (ShowIndirectCallTargets) {
3136 OS << " Indirect Target Results:\n";
3137 traverseAllValueSites(Func, VK: IPVK_IndirectCallTarget,
3138 Stats&: VPStats[IPVK_IndirectCallTarget], OS,
3139 Symtab: &(Reader->getSymtab()));
3140 }
3141
3142 if (ShowVTables) {
3143 OS << " VTable Results:\n";
3144 traverseAllValueSites(Func, VK: IPVK_VTableTarget,
3145 Stats&: VPStats[IPVK_VTableTarget], OS,
3146 Symtab: &(Reader->getSymtab()));
3147 }
3148
3149 if (ShowMemOPSizes && NumMemOPCalls > 0) {
3150 OS << " Memory Intrinsic Size Results:\n";
3151 traverseAllValueSites(Func, VK: IPVK_MemOPSize, Stats&: VPStats[IPVK_MemOPSize], OS,
3152 Symtab: nullptr);
3153 }
3154 }
3155 }
3156 if (Reader->hasError())
3157 return makeError(E: Reader->getError(), Whence: Filename);
3158
3159 if (TextFormat || ShowCovered)
3160 return Error::success();
3161 std::unique_ptr<ProfileSummary> PS(Builder.getSummary());
3162 bool IsIR = Reader->isIRLevelProfile();
3163 OS << "Instrumentation level: " << (IsIR ? "IR" : "Front-end");
3164 if (IsIR) {
3165 OS << " entry_first = " << Reader->instrEntryBBEnabled();
3166 OS << " instrument_loop_entries = " << Reader->instrLoopEntriesEnabled();
3167 }
3168 OS << "\n";
3169 if (ShowAllFunctions || !FuncNameFilter.empty())
3170 OS << "Functions shown: " << ShownFunctions << "\n";
3171 PS->printSummary(OS);
3172 if (ShowValueCutoff > 0) {
3173 OS << "Number of functions with maximum count (< " << ShowValueCutoff
3174 << "): " << BelowCutoffFunctions << "\n";
3175 OS << "Number of functions with maximum count (>= " << ShowValueCutoff
3176 << "): " << PS->getNumFunctions() - BelowCutoffFunctions << "\n";
3177 }
3178
3179 // Sort by MaxCount in decreasing order
3180 llvm::stable_sort(Range&: NameAndMaxCount, C: [](const auto &L, const auto &R) {
3181 return L.second > R.second;
3182 });
3183 if (TopNFunctions) {
3184 OS << "Top " << TopNFunctions
3185 << " functions with the largest internal block counts: \n";
3186 auto TopFuncs = ArrayRef(NameAndMaxCount).take_front(N: TopNFunctions);
3187 for (auto [Name, MaxCount] : TopFuncs)
3188 OS << " " << Name << ", max count = " << MaxCount << "\n";
3189 }
3190
3191 if (ShowHotFuncList) {
3192 auto HotCountThreshold =
3193 ProfileSummaryBuilder::getHotCountThreshold(DS: PS->getDetailedSummary());
3194 OS << "# Hot count threshold: " << HotCountThreshold << "\n";
3195 for (auto [Name, MaxCount] : NameAndMaxCount) {
3196 if (MaxCount < HotCountThreshold)
3197 break;
3198 OS << Name << "\n";
3199 }
3200 }
3201
3202 if (ShownFunctions && ShowIndirectCallTargets) {
3203 OS << "Statistics for indirect call sites profile:\n";
3204 showValueSitesStats(OS, VK: IPVK_IndirectCallTarget,
3205 Stats&: VPStats[IPVK_IndirectCallTarget]);
3206 }
3207
3208 if (ShownFunctions && ShowVTables) {
3209 OS << "Statistics for vtable profile:\n";
3210 showValueSitesStats(OS, VK: IPVK_VTableTarget, Stats&: VPStats[IPVK_VTableTarget]);
3211 }
3212
3213 if (ShownFunctions && ShowMemOPSizes) {
3214 OS << "Statistics for memory intrinsic calls sizes profile:\n";
3215 showValueSitesStats(OS, VK: IPVK_MemOPSize, Stats&: VPStats[IPVK_MemOPSize]);
3216 }
3217
3218 if (ShowDetailedSummary)
3219 PS->printDetailedSummary(OS);
3220
3221 if (ShowBinaryIds)
3222 if (Error E = Reader->printBinaryIds(OS))
3223 return makeError(E: std::move(E), Whence: Filename);
3224
3225 if (ShowProfileVersion)
3226 OS << "Profile version: " << Reader->getVersion() << "\n";
3227
3228 if (ShowTemporalProfTraces) {
3229 auto &Traces = Reader->getTemporalProfTraces();
3230 OS << "Temporal Profile Traces (samples=" << Traces.size()
3231 << " seen=" << Reader->getTemporalProfTraceStreamSize() << "):\n";
3232 for (unsigned i = 0; i < Traces.size(); i++) {
3233 OS << " Temporal Profile Trace " << i << " (weight=" << Traces[i].Weight
3234 << " count=" << Traces[i].FunctionNameRefs.size() << "):\n";
3235 for (auto &NameRef : Traces[i].FunctionNameRefs)
3236 OS << " " << Reader->getSymtab().getFuncOrVarName(MD5Hash: NameRef) << "\n";
3237 }
3238 }
3239
3240 return Error::success();
3241}
3242
3243static void showSectionInfo(sampleprof::SampleProfileReader *Reader,
3244 raw_fd_ostream &OS) {
3245 if (!Reader->dumpSectionInfo(OS)) {
3246 WithColor::warning() << "-show-sec-info-only is only supported for "
3247 << "sample profile in extbinary format and is "
3248 << "ignored for other formats.\n";
3249 return;
3250 }
3251}
3252
3253namespace {
3254struct HotFuncInfo {
3255 std::string FuncName;
3256 uint64_t TotalCount = 0;
3257 double TotalCountPercent = 0.0f;
3258 uint64_t MaxCount = 0;
3259 uint64_t EntryCount = 0;
3260
3261 HotFuncInfo() = default;
3262
3263 HotFuncInfo(StringRef FN, uint64_t TS, double TSP, uint64_t MS, uint64_t ES)
3264 : FuncName(FN.begin(), FN.end()), TotalCount(TS), TotalCountPercent(TSP),
3265 MaxCount(MS), EntryCount(ES) {}
3266};
3267} // namespace
3268
3269// Print out detailed information about hot functions in PrintValues vector.
3270// Users specify titles and offset of every columns through ColumnTitle and
3271// ColumnOffset. The size of ColumnTitle and ColumnOffset need to be the same
3272// and at least 4. Besides, users can optionally give a HotFuncMetric string to
3273// print out or let it be an empty string.
3274static void dumpHotFunctionList(const std::vector<std::string> &ColumnTitle,
3275 const std::vector<int> &ColumnOffset,
3276 const std::vector<HotFuncInfo> &PrintValues,
3277 uint64_t HotFuncCount, uint64_t TotalFuncCount,
3278 uint64_t HotProfCount, uint64_t TotalProfCount,
3279 const std::string &HotFuncMetric,
3280 uint32_t TopNFunctions, raw_fd_ostream &OS) {
3281 assert(ColumnOffset.size() == ColumnTitle.size() &&
3282 "ColumnOffset and ColumnTitle should have the same size");
3283 assert(ColumnTitle.size() >= 4 &&
3284 "ColumnTitle should have at least 4 elements");
3285 assert(TotalFuncCount > 0 &&
3286 "There should be at least one function in the profile");
3287 double TotalProfPercent = 0;
3288 if (TotalProfCount > 0)
3289 TotalProfPercent = static_cast<double>(HotProfCount) / TotalProfCount * 100;
3290
3291 formatted_raw_ostream FOS(OS);
3292 FOS << HotFuncCount << " out of " << TotalFuncCount
3293 << " functions with profile ("
3294 << format(Fmt: "%.2f%%",
3295 Vals: (static_cast<double>(HotFuncCount) / TotalFuncCount * 100))
3296 << ") are considered hot functions";
3297 if (!HotFuncMetric.empty())
3298 FOS << " (" << HotFuncMetric << ")";
3299 FOS << ".\n";
3300 FOS << HotProfCount << " out of " << TotalProfCount << " profile counts ("
3301 << format(Fmt: "%.2f%%", Vals: TotalProfPercent) << ") are from hot functions.\n";
3302
3303 for (size_t I = 0; I < ColumnTitle.size(); ++I) {
3304 FOS.PadToColumn(NewCol: ColumnOffset[I]);
3305 FOS << ColumnTitle[I];
3306 }
3307 FOS << "\n";
3308
3309 uint32_t Count = 0;
3310 for (const auto &R : PrintValues) {
3311 if (TopNFunctions && (Count++ == TopNFunctions))
3312 break;
3313 FOS.PadToColumn(NewCol: ColumnOffset[0]);
3314 FOS << R.TotalCount << " (" << format(Fmt: "%.2f%%", Vals: R.TotalCountPercent) << ")";
3315 FOS.PadToColumn(NewCol: ColumnOffset[1]);
3316 FOS << R.MaxCount;
3317 FOS.PadToColumn(NewCol: ColumnOffset[2]);
3318 FOS << R.EntryCount;
3319 FOS.PadToColumn(NewCol: ColumnOffset[3]);
3320 FOS << R.FuncName << "\n";
3321 }
3322}
3323
3324static int showHotFunctionList(const sampleprof::SampleProfileMap &Profiles,
3325 ProfileSummary &PS, uint32_t TopN,
3326 raw_fd_ostream &OS) {
3327 using namespace sampleprof;
3328
3329 const uint32_t HotFuncCutoff = 990000;
3330 auto &SummaryVector = PS.getDetailedSummary();
3331 uint64_t MinCountThreshold = 0;
3332 for (const ProfileSummaryEntry &SummaryEntry : SummaryVector) {
3333 if (SummaryEntry.Cutoff == HotFuncCutoff) {
3334 MinCountThreshold = SummaryEntry.MinCount;
3335 break;
3336 }
3337 }
3338
3339 // Traverse all functions in the profile and keep only hot functions.
3340 // The following loop also calculates the sum of total samples of all
3341 // functions.
3342 std::multimap<uint64_t, std::pair<const FunctionSamples *, const uint64_t>,
3343 std::greater<uint64_t>>
3344 HotFunc;
3345 uint64_t ProfileTotalSample = 0;
3346 uint64_t HotFuncSample = 0;
3347 uint64_t HotFuncCount = 0;
3348
3349 for (const auto &I : Profiles) {
3350 FuncSampleStats FuncStats;
3351 const FunctionSamples &FuncProf = I.second;
3352 ProfileTotalSample += FuncProf.getTotalSamples();
3353 getFuncSampleStats(Func: FuncProf, FuncStats, HotThreshold: MinCountThreshold);
3354
3355 if (isFunctionHot(FuncStats, HotThreshold: MinCountThreshold)) {
3356 HotFunc.emplace(args: FuncProf.getTotalSamples(),
3357 args: std::make_pair(x: &(I.second), y&: FuncStats.MaxSample));
3358 HotFuncSample += FuncProf.getTotalSamples();
3359 ++HotFuncCount;
3360 }
3361 }
3362
3363 std::vector<std::string> ColumnTitle{"Total sample (%)", "Max sample",
3364 "Entry sample", "Function name"};
3365 std::vector<int> ColumnOffset{0, 24, 42, 58};
3366 std::string Metric =
3367 std::string("max sample >= ") + std::to_string(val: MinCountThreshold);
3368 std::vector<HotFuncInfo> PrintValues;
3369 for (const auto &FuncPair : HotFunc) {
3370 const FunctionSamples &Func = *FuncPair.second.first;
3371 double TotalSamplePercent =
3372 (ProfileTotalSample > 0)
3373 ? (Func.getTotalSamples() * 100.0) / ProfileTotalSample
3374 : 0;
3375 PrintValues.emplace_back(
3376 args: HotFuncInfo(Func.getContext().toString(), Func.getTotalSamples(),
3377 TotalSamplePercent, FuncPair.second.second,
3378 Func.getHeadSamplesEstimate()));
3379 }
3380 dumpHotFunctionList(ColumnTitle, ColumnOffset, PrintValues, HotFuncCount,
3381 TotalFuncCount: Profiles.size(), HotProfCount: HotFuncSample, TotalProfCount: ProfileTotalSample,
3382 HotFuncMetric: Metric, TopNFunctions: TopN, OS);
3383
3384 return 0;
3385}
3386
3387static Error showSampleProfile(ShowFormat SFormat, raw_fd_ostream &OS) {
3388 if (SFormat == ShowFormat::Yaml)
3389 return makeError(Message: "YAML output is not supported for sample profiles");
3390 if (ShowSectionInfoOnly && ShowCompositeInfoOnly)
3391 return makeError(Message: "-show-sec-info-only and "
3392 "-show-composite-info-only cannot be used together");
3393
3394 using namespace sampleprof;
3395 LLVMContext Context;
3396 auto FS = vfs::getRealFileSystem();
3397 auto ReaderOrErr = SampleProfileReader::create(Filename, C&: Context, FS&: *FS,
3398 P: FSDiscriminatorPassOption);
3399 if (std::error_code EC = ReaderOrErr.getError())
3400 return makeError(EC, Whence: Filename);
3401
3402 auto Reader = std::move(ReaderOrErr.get());
3403 if (ShowSectionInfoOnly) {
3404 showSectionInfo(Reader: Reader.get(), OS);
3405 return Error::success();
3406 }
3407
3408 if (ShowCompositeInfoOnly) {
3409 if (!Reader->hasCompositeProfileSection()) {
3410 WithColor::warning() << "no composite profile section; nothing to show\n";
3411 return Error::success();
3412 }
3413 if (std::error_code EC = Reader->dumpProfileTypeInfo(OS)) {
3414 OS.flush();
3415 return makeError(EC, Whence: Filename);
3416 }
3417 return Error::success();
3418 }
3419
3420 if (std::error_code EC = Reader->read())
3421 return makeError(EC, Whence: Filename);
3422
3423 if (ShowAllFunctions || FuncNameFilter.empty()) {
3424 if (SFormat == ShowFormat::Json)
3425 Reader->dumpJson(OS);
3426 else
3427 Reader->dump(OS);
3428 } else {
3429 if (SFormat == ShowFormat::Json)
3430 return makeError(
3431 Message: "the JSON format is supported only when all functions are to "
3432 "be printed");
3433
3434 // TODO: parse context string to support filtering by contexts.
3435 FunctionSamples *FS = Reader->getSamplesFor(Fname: StringRef(FuncNameFilter));
3436 Reader->dumpFunctionProfile(FS: FS ? *FS : FunctionSamples(), OS);
3437 }
3438
3439 if (ShowProfileSymbolList) {
3440 std::unique_ptr<sampleprof::ProfileSymbolList> ReaderList =
3441 Reader->getProfileSymbolList();
3442 ReaderList->dump(OS);
3443 }
3444
3445 if (ShowDetailedSummary) {
3446 auto &PS = Reader->getSummary();
3447 PS.printSummary(OS);
3448 PS.printDetailedSummary(OS);
3449 }
3450
3451 if (ShowHotFuncList || TopNFunctions)
3452 showHotFunctionList(Profiles: Reader->getProfiles(), PS&: Reader->getSummary(),
3453 TopN: TopNFunctions, OS);
3454
3455 return Error::success();
3456}
3457
3458static Error showMemProfProfile(ShowFormat SFormat, raw_fd_ostream &OS) {
3459 if (SFormat == ShowFormat::Json)
3460 return makeError(Message: "JSON output is not supported for MemProf");
3461
3462 // Show the raw profile in YAML.
3463 if (memprof::RawMemProfReader::hasFormat(Path: Filename)) {
3464 auto ReaderOr = llvm::memprof::RawMemProfReader::create(
3465 Path: Filename, ProfiledBinary, /*KeepNames=*/KeepName: true);
3466 if (Error E = ReaderOr.takeError()) {
3467 // Since the error can be related to the profile or the binary we do not
3468 // pass whence. Instead additional context is provided where necessary in
3469 // the error message.
3470 return makeError(E: std::move(E), /*Whence*/ "");
3471 }
3472
3473 std::unique_ptr<llvm::memprof::RawMemProfReader> Reader(
3474 ReaderOr.get().release());
3475
3476 Reader->printYAML(OS);
3477 return Error::success();
3478 }
3479
3480 // Show the indexed MemProf profile in YAML.
3481 auto FS = vfs::getRealFileSystem();
3482 auto ReaderOrErr = IndexedInstrProfReader::create(Path: Filename, FS&: *FS);
3483 if (Error E = ReaderOrErr.takeError())
3484 return makeError(E: std::move(E), Whence: Filename);
3485
3486 auto Reader = std::move(ReaderOrErr.get());
3487 memprof::AllMemProfData Data = Reader->getAllMemProfData();
3488
3489 // For v4 and above the summary is serialized in the indexed profile, and can
3490 // be accessed from the reader. Earlier versions build the summary below.
3491 // The summary is emitted as YAML comments at the start of the output.
3492 if (auto *MemProfSum = Reader->getMemProfSummary()) {
3493 MemProfSum->printSummaryYaml(OS);
3494 } else {
3495 memprof::MemProfSummaryBuilder MemProfSumBuilder;
3496 for (auto &Pair : Data.HeapProfileRecords)
3497 MemProfSumBuilder.addRecord(Pair.Record);
3498 MemProfSumBuilder.getSummary()->printSummaryYaml(OS);
3499 }
3500 // Construct yaml::Output with the maximum column width of 80 so that each
3501 // Frame fits in one line.
3502 yaml::Output Yout(OS, nullptr, 80);
3503 Yout << Data;
3504
3505 return Error::success();
3506}
3507
3508static Error showDebugInfoCorrelation(const std::string &Filename,
3509 ShowFormat SFormat, raw_fd_ostream &OS) {
3510 if (SFormat == ShowFormat::Json)
3511 return makeError(Message: "JSON output is not supported for debug info correlation");
3512 std::unique_ptr<InstrProfCorrelator> Correlator;
3513 if (auto Err =
3514 InstrProfCorrelator::get(Filename, FileKind: ProfCorrelatorKind::DebugInfo)
3515 .moveInto(Value&: Correlator))
3516 return makeError(E: std::move(Err), Whence: Filename);
3517 if (SFormat == ShowFormat::Yaml) {
3518 if (auto Err = Correlator->dumpYaml(MaxWarnings: MaxDbgCorrelationWarnings, OS))
3519 return makeError(E: std::move(Err), Whence: Filename);
3520 return Error::success();
3521 }
3522
3523 if (auto Err = Correlator->correlateProfileData(MaxWarnings: MaxDbgCorrelationWarnings))
3524 return makeError(E: std::move(Err), Whence: Filename);
3525
3526 InstrProfSymtab Symtab;
3527 if (auto Err = Symtab.create(
3528 NameStrings: StringRef(Correlator->getNamesPointer(), Correlator->getNamesSize())))
3529 return makeError(E: std::move(Err), Whence: Filename);
3530
3531 if (ShowProfileSymbolList)
3532 Symtab.dumpNames(OS);
3533 // TODO: Read "Profile Data Type" from debug info to compute and show how many
3534 // counters the section holds.
3535 if (ShowDetailedSummary)
3536 OS << "Counters section size: 0x"
3537 << Twine::utohexstr(Val: Correlator->getCountersSectionSize()) << " bytes\n";
3538 OS << "Found " << Correlator->getDataSize() << " functions\n";
3539
3540 return Error::success();
3541}
3542
3543static Error show_main(StringRef ProgName) {
3544 if (Filename.empty() && DebugInfoFilename.empty())
3545 return makeError(
3546 Message: "the positional argument '<profdata-file>' is required unless '--" +
3547 DebugInfoFilename.ArgStr + "' is provided");
3548
3549 if (Filename == OutputFilename)
3550 return makeError(
3551 Message: "Input file name cannot be the same as the output file name!",
3552 Whence: (ProgName + " show").str());
3553 if (JsonFormat)
3554 SFormat = ShowFormat::Json;
3555
3556 std::error_code EC;
3557 raw_fd_ostream OS(OutputFilename.data(), EC, sys::fs::OF_TextWithCRLF);
3558 if (EC)
3559 return makeError(EC, Whence: OutputFilename);
3560
3561 if (ShowAllFunctions && !FuncNameFilter.empty())
3562 WithColor::warning() << "-function argument ignored: showing all functions\n";
3563
3564 if (!DebugInfoFilename.empty())
3565 return showDebugInfoCorrelation(Filename: DebugInfoFilename, SFormat, OS);
3566
3567 if (ShowProfileKind == instr)
3568 return showInstrProfile(SFormat, OS);
3569 if (ShowProfileKind == sample)
3570 return showSampleProfile(SFormat, OS);
3571 return showMemProfProfile(SFormat, OS);
3572}
3573
3574static Error order_main() {
3575 std::error_code EC;
3576 raw_fd_ostream OS(OutputFilename.data(), EC, sys::fs::OF_TextWithCRLF);
3577 if (EC)
3578 return makeError(EC, Whence: OutputFilename);
3579 auto FS = vfs::getRealFileSystem();
3580 auto ReaderOrErr = InstrProfReader::create(Path: Filename, FS&: *FS);
3581 if (Error E = ReaderOrErr.takeError())
3582 return makeError(E: std::move(E), Whence: Filename);
3583
3584 auto Reader = std::move(ReaderOrErr.get());
3585 for (auto &I : *Reader) {
3586 // Read all entries
3587 (void)I;
3588 }
3589 ArrayRef Traces = Reader->getTemporalProfTraces();
3590 if (NumTestTraces && NumTestTraces >= Traces.size())
3591 return makeError(
3592 Message: "--" + NumTestTraces.ArgStr +
3593 " must be smaller than the total number of traces: expected: < " +
3594 Twine(Traces.size()) + ", actual: " + Twine(NumTestTraces));
3595 ArrayRef TestTraces = Traces.take_back(N: NumTestTraces);
3596 Traces = Traces.drop_back(N: NumTestTraces);
3597
3598 std::vector<BPFunctionNode> Nodes;
3599 TemporalProfTraceTy::createBPFunctionNodes(Traces, Nodes);
3600 BalancedPartitioningConfig Config;
3601 BalancedPartitioning BP(Config);
3602 BP.run(Nodes);
3603
3604 OS << "# Ordered " << Nodes.size() << " functions\n";
3605 if (!TestTraces.empty()) {
3606 // Since we don't know the symbol sizes, we assume 32 functions per page.
3607 DenseMap<BPFunctionNode::IDT, unsigned> IdToPageNumber;
3608 for (auto &Node : Nodes)
3609 IdToPageNumber[Node.Id] = IdToPageNumber.size() / 32;
3610
3611 SmallSet<unsigned, 0> TouchedPages;
3612 uint64_t Area = 0;
3613 for (auto &Trace : TestTraces) {
3614 for (auto Id : Trace.FunctionNameRefs) {
3615 auto It = IdToPageNumber.find(Val: Id);
3616 if (It == IdToPageNumber.end())
3617 continue;
3618 TouchedPages.insert(V: It->getSecond());
3619 Area += TouchedPages.size();
3620 }
3621 TouchedPages.clear();
3622 }
3623 OS << "# Average area under the page fault curve: "
3624 << (float)Area / TestTraces.size() << "\n";
3625 }
3626 OS << "# Warning: Mach-O may prefix symbols with \"_\" depending on the "
3627 "linkage and this output does not take that into account. Some "
3628 "post-processing may be required before passing to the linker via "
3629 "-order_file.\n";
3630 for (auto &N : Nodes) {
3631 auto [Filename, ParsedFuncName] =
3632 getParsedIRPGOName(IRPGOName: Reader->getSymtab().getFuncOrVarName(MD5Hash: N.Id));
3633 if (!Filename.empty())
3634 OS << "# " << Filename << "\n";
3635 OS << ParsedFuncName << "\n";
3636 }
3637 return Error::success();
3638}
3639
3640int main(int argc, const char *argv[]) {
3641 InitLLVM X(argc, argv);
3642 StringRef ProgName(sys::path::filename(path: argv[0]));
3643
3644 if (argc < 2) {
3645 errs()
3646 << ProgName
3647 << ": No subcommand specified! Run llvm-profdata --help for usage.\n";
3648 return 1;
3649 }
3650
3651 cl::ParseCommandLineOptions(argc, argv, Overview: "LLVM profile data\n");
3652
3653 if (ShowSubcommand)
3654 return reportError(E: show_main(ProgName));
3655
3656 if (OrderSubcommand)
3657 return reportError(E: order_main());
3658
3659 if (OverlapSubcommand)
3660 return reportError(E: overlap_main());
3661
3662 if (MergeSubcommand)
3663 return reportError(E: merge_main(ProgName));
3664
3665 errs() << ProgName
3666 << ": Unknown command. Run llvm-profdata --help for usage.\n";
3667 return 1;
3668}
3669