1//===- llvm-profdata.cpp - LLVM profile data tool -------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// llvm-profdata merges .profdata files.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/ADT/DenseMap.h"
14#include "llvm/ADT/ScopeExit.h"
15#include "llvm/ADT/SmallSet.h"
16#include "llvm/ADT/SmallVector.h"
17#include "llvm/ADT/StringRef.h"
18#include "llvm/HTTP/HTTPClient.h"
19#include "llvm/IR/LLVMContext.h"
20#include "llvm/Object/Binary.h"
21#include "llvm/ProfileData/DataAccessProf.h"
22#include "llvm/ProfileData/InstrProfCorrelator.h"
23#include "llvm/ProfileData/InstrProfReader.h"
24#include "llvm/ProfileData/InstrProfWriter.h"
25#include "llvm/ProfileData/MemProf.h"
26#include "llvm/ProfileData/MemProfReader.h"
27#include "llvm/ProfileData/MemProfSummaryBuilder.h"
28#include "llvm/ProfileData/MemProfYAML.h"
29#include "llvm/ProfileData/ProfileCommon.h"
30#include "llvm/ProfileData/SampleProfReader.h"
31#include "llvm/ProfileData/SampleProfWriter.h"
32#include "llvm/Support/BalancedPartitioning.h"
33#include "llvm/Support/CommandLine.h"
34#include "llvm/Support/Discriminator.h"
35#include "llvm/Support/Errc.h"
36#include "llvm/Support/FileSystem.h"
37#include "llvm/Support/Format.h"
38#include "llvm/Support/FormattedStream.h"
39#include "llvm/Support/InitLLVM.h"
40#include "llvm/Support/MD5.h"
41#include "llvm/Support/MemoryBuffer.h"
42#include "llvm/Support/Path.h"
43#include "llvm/Support/Regex.h"
44#include "llvm/Support/ThreadPool.h"
45#include "llvm/Support/Threading.h"
46#include "llvm/Support/VirtualFileSystem.h"
47#include "llvm/Support/WithColor.h"
48#include "llvm/Support/raw_ostream.h"
49#include <algorithm>
50#include <cmath>
51#include <optional>
52
53using namespace llvm;
54using ProfCorrelatorKind = InstrProfCorrelator::ProfCorrelatorKind;
55
56// https://llvm.org/docs/CommandGuide/llvm-profdata.html has documentations
57// on each subcommand.
58cl::SubCommand ShowSubcommand(
59 "show",
60 "Takes a profile data file and displays the profiles. See detailed "
61 "documentation in "
62 "https://llvm.org/docs/CommandGuide/llvm-profdata.html#profdata-show");
63cl::SubCommand OrderSubcommand(
64 "order",
65 "Reads temporal profiling traces from a profile and outputs a function "
66 "order that reduces the number of page faults for those traces. See "
67 "detailed documentation in "
68 "https://llvm.org/docs/CommandGuide/llvm-profdata.html#profdata-order");
69cl::SubCommand OverlapSubcommand(
70 "overlap",
71 "Computes and displays the overlap between two profiles. See detailed "
72 "documentation in "
73 "https://llvm.org/docs/CommandGuide/llvm-profdata.html#profdata-overlap");
74cl::SubCommand MergeSubcommand(
75 "merge",
76 "Takes several profiles and merge them together. See detailed "
77 "documentation in "
78 "https://llvm.org/docs/CommandGuide/llvm-profdata.html#profdata-merge");
79
80namespace {
81enum ProfileKinds { instr, sample, memory };
82enum FailureMode { warnOnly, failIfAnyAreInvalid, failIfAllAreInvalid };
83
84enum ProfileFormat {
85 PF_None = 0,
86 PF_Text,
87 PF_Compact_Binary, // Deprecated
88 PF_Ext_Binary,
89 PF_GCC,
90 PF_Binary
91};
92
93enum class ShowFormat { Text, Json, Yaml };
94} // namespace
95
96// Common options.
97cl::opt<std::string> OutputFilename("output", cl::value_desc("output"),
98 cl::init(Val: "-"), cl::desc("Output file"),
99 cl::sub(ShowSubcommand),
100 cl::sub(OrderSubcommand),
101 cl::sub(OverlapSubcommand),
102 cl::sub(MergeSubcommand));
103// NOTE: cl::alias must not have cl::sub(), since aliased option's cl::sub()
104// will be used. llvm::cl::alias::done() method asserts this condition.
105static cl::alias OutputFilenameA("o", cl::desc("Alias for --output"),
106 cl::aliasopt(OutputFilename));
107
108// Options common to at least two commands.
109static cl::opt<ProfileKinds> ProfileKind(
110 cl::desc("Profile kind:"), cl::sub(MergeSubcommand),
111 cl::sub(OverlapSubcommand), cl::init(Val: instr),
112 cl::values(clEnumVal(instr, "Instrumentation profile (default)"),
113 clEnumVal(sample, "Sample profile")));
114static cl::opt<std::string> Filename(cl::Positional,
115 cl::desc("<profdata-file>"),
116 cl::sub(ShowSubcommand),
117 cl::sub(OrderSubcommand));
118static cl::opt<unsigned> MaxDbgCorrelationWarnings(
119 "max-debug-info-correlation-warnings",
120 cl::desc("The maximum number of warnings to emit when correlating "
121 "profile from debug info (0 = no limit)"),
122 cl::sub(MergeSubcommand), cl::sub(ShowSubcommand), cl::init(Val: 5));
123static cl::opt<std::string> ProfiledBinary(
124 "profiled-binary", cl::init(Val: ""),
125 cl::desc("Path to binary from which the profile was collected."),
126 cl::sub(ShowSubcommand), cl::sub(MergeSubcommand));
127static cl::opt<std::string> DebugInfoFilename(
128 "debug-info", cl::init(Val: ""),
129 cl::desc(
130 "For show, read and extract profile metadata from debug info and show "
131 "the functions it found. For merge, use the provided debug info to "
132 "correlate the raw profile."),
133 cl::sub(ShowSubcommand), cl::sub(MergeSubcommand));
134static cl::opt<std::string>
135 BinaryFilename("binary-file", cl::init(Val: ""),
136 cl::desc("For merge, use the provided unstripped binary to "
137 "correlate the raw profile."),
138 cl::sub(MergeSubcommand));
139static cl::list<std::string> DebugFileDirectory(
140 "debug-file-directory",
141 cl::desc("Directories to search for object files by build ID"));
142static cl::opt<bool> DebugInfod("debuginfod", cl::init(Val: false), cl::Hidden,
143 cl::sub(MergeSubcommand),
144 cl::desc("Enable debuginfod"));
145static cl::opt<ProfCorrelatorKind> BIDFetcherProfileCorrelate(
146 "correlate",
147 cl::desc("Use debug-info or binary correlation to correlate profiles with "
148 "build id fetcher"),
149 cl::init(Val: InstrProfCorrelator::NONE),
150 cl::values(clEnumValN(InstrProfCorrelator::NONE, "",
151 "No profile correlation"),
152 clEnumValN(InstrProfCorrelator::DEBUG_INFO, "debug-info",
153 "Use debug info to correlate"),
154 clEnumValN(InstrProfCorrelator::BINARY, "binary",
155 "Use binary to correlate")));
156static cl::opt<std::string> FuncNameFilter(
157 "function",
158 cl::desc("Only functions matching the filter are shown in the output. For "
159 "overlapping CSSPGO, this takes a function name with calling "
160 "context."),
161 cl::sub(ShowSubcommand), cl::sub(OverlapSubcommand),
162 cl::sub(MergeSubcommand));
163
164// TODO: Consider creating a template class (e.g., MergeOption, ShowOption) to
165// factor out the common cl::sub in cl::opt constructor for subcommand-specific
166// options.
167
168// Options specific to merge subcommand.
169static cl::list<std::string> InputFilenames(cl::Positional,
170 cl::sub(MergeSubcommand),
171 cl::desc("<filename...>"));
172static cl::list<std::string>
173 WeightedInputFilenames("weighted-input", cl::sub(MergeSubcommand),
174 cl::desc("<weight>,<filename>"));
175static cl::opt<ProfileFormat> OutputFormat(
176 cl::desc("Format of output profile"), cl::sub(MergeSubcommand),
177 cl::init(Val: PF_Ext_Binary),
178 cl::values(clEnumValN(PF_Binary, "binary", "Binary encoding"),
179 clEnumValN(PF_Ext_Binary, "extbinary",
180 "Extensible binary encoding "
181 "(default)"),
182 clEnumValN(PF_Text, "text", "Text encoding"),
183 clEnumValN(PF_GCC, "gcc",
184 "GCC encoding (only meaningful for -sample)")));
185static cl::opt<std::string>
186 InputFilenamesFile("input-files", cl::init(Val: ""), cl::sub(MergeSubcommand),
187 cl::desc("Path to file containing newline-separated "
188 "[<weight>,]<filename> entries"));
189static cl::alias InputFilenamesFileA("f", cl::desc("Alias for --input-files"),
190 cl::aliasopt(InputFilenamesFile));
191static cl::opt<bool> DumpInputFileList(
192 "dump-input-file-list", cl::init(Val: false), cl::Hidden,
193 cl::sub(MergeSubcommand),
194 cl::desc("Dump the list of input files and their weights, then exit"));
195static cl::opt<std::string> RemappingFile("remapping-file",
196 cl::value_desc("file"),
197 cl::sub(MergeSubcommand),
198 cl::desc("Symbol remapping file"));
199static cl::alias RemappingFileA("r", cl::desc("Alias for --remapping-file"),
200 cl::aliasopt(RemappingFile));
201static cl::opt<bool>
202 UseMD5("use-md5", cl::init(Val: false), cl::Hidden,
203 cl::desc("Choose to use MD5 to represent string in name table (only "
204 "meaningful for -extbinary)"),
205 cl::sub(MergeSubcommand));
206static cl::opt<bool> CompressAllSections(
207 "compress-all-sections", cl::init(Val: false), cl::Hidden,
208 cl::sub(MergeSubcommand),
209 cl::desc("Compress all sections when writing the profile (only "
210 "meaningful for -extbinary)"));
211static cl::opt<bool> SampleMergeColdContext(
212 "sample-merge-cold-context", cl::init(Val: false), cl::Hidden,
213 cl::sub(MergeSubcommand),
214 cl::desc(
215 "Merge context sample profiles whose count is below cold threshold"));
216static cl::opt<bool> SampleTrimColdContext(
217 "sample-trim-cold-context", cl::init(Val: false), cl::Hidden,
218 cl::sub(MergeSubcommand),
219 cl::desc(
220 "Trim context sample profiles whose count is below cold threshold"));
221static cl::opt<uint32_t> SampleColdContextFrameDepth(
222 "sample-frame-depth-for-cold-context", cl::init(Val: 1),
223 cl::sub(MergeSubcommand),
224 cl::desc("Keep the last K frames while merging cold profile. 1 means the "
225 "context-less base profile"));
226static cl::opt<size_t> OutputSizeLimit(
227 "output-size-limit", cl::init(Val: 0), cl::Hidden, cl::sub(MergeSubcommand),
228 cl::desc("Trim cold functions until profile size is below specified "
229 "limit in bytes. This uses a heursitic and functions may be "
230 "excessively trimmed"));
231static cl::opt<bool> GenPartialProfile(
232 "gen-partial-profile", cl::init(Val: false), cl::Hidden,
233 cl::sub(MergeSubcommand),
234 cl::desc("Generate a partial profile (only meaningful for -extbinary)"));
235static cl::opt<bool> SplitLayout(
236 "split-layout", cl::init(Val: false), cl::Hidden, cl::sub(MergeSubcommand),
237 cl::desc("Split the profile to two sections with one containing sample "
238 "profiles with inlined functions and the other without (only "
239 "meaningful for -extbinary)"));
240static cl::opt<bool>
241 WriteMD5ProfSymList("md5-prof-sym-list", cl::init(Val: false), cl::Hidden,
242 cl::sub(MergeSubcommand),
243 cl::desc("Write ProfileSymbolList (Cold Symbols) as "
244 "64-bit MD5 hashes in Eytzinger layout"));
245static cl::opt<bool> WriteMD5IndexedTables(
246 "md5-indexed-tables", cl::init(Val: false), cl::Hidden, cl::sub(MergeSubcommand),
247 cl::desc("Write MD5-based indexed NameTable and parallel "
248 "FuncOffsetTable in Eytzinger layout (only meaningful for "
249 "-extbinary)"));
250static cl::opt<std::string> SupplInstrWithSample(
251 "supplement-instr-with-sample", cl::init(Val: ""), cl::Hidden,
252 cl::sub(MergeSubcommand),
253 cl::desc("Supplement an instr profile with sample profile, to correct "
254 "the profile unrepresentativeness issue. The sample "
255 "profile is the input of the flag. Output will be in instr "
256 "format (The flag only works with -instr)"));
257static cl::opt<float> ZeroCounterThreshold(
258 "zero-counter-threshold", cl::init(Val: 0.7), cl::Hidden,
259 cl::sub(MergeSubcommand),
260 cl::desc("For the function which is cold in instr profile but hot in "
261 "sample profile, if the ratio of the number of zero counters "
262 "divided by the total number of counters is above the "
263 "threshold, the profile of the function will be regarded as "
264 "being harmful for performance and will be dropped."));
265static cl::opt<unsigned> SupplMinSizeThreshold(
266 "suppl-min-size-threshold", cl::init(Val: 10), cl::Hidden,
267 cl::sub(MergeSubcommand),
268 cl::desc("If the size of a function is smaller than the threshold, "
269 "assume it can be inlined by PGO early inliner and it won't "
270 "be adjusted based on sample profile."));
271static cl::opt<unsigned> InstrProfColdThreshold(
272 "instr-prof-cold-threshold", cl::init(Val: 0), cl::Hidden,
273 cl::sub(MergeSubcommand),
274 cl::desc("User specified cold threshold for instr profile which will "
275 "override the cold threshold got from profile summary. "));
276// WARNING: This reservoir size value is propagated to any input indexed
277// profiles for simplicity. Changing this value between invocations could
278// result in sample bias.
279static cl::opt<uint64_t> TemporalProfTraceReservoirSize(
280 "temporal-profile-trace-reservoir-size", cl::init(Val: 100),
281 cl::sub(MergeSubcommand),
282 cl::desc("The maximum number of stored temporal profile traces (default: "
283 "100)"));
284static cl::opt<uint64_t> TemporalProfMaxTraceLength(
285 "temporal-profile-max-trace-length", cl::init(Val: 10000),
286 cl::sub(MergeSubcommand),
287 cl::desc("The maximum length of a single temporal profile trace "
288 "(default: 10000)"));
289static cl::opt<std::string> FuncNameNegativeFilter(
290 "no-function", cl::init(Val: ""), cl::sub(MergeSubcommand),
291 cl::desc("Exclude functions matching the filter from the output."));
292
293static cl::opt<FailureMode>
294 FailMode("failure-mode", cl::init(Val: failIfAnyAreInvalid),
295 cl::desc("Failure mode:"), cl::sub(MergeSubcommand),
296 cl::values(clEnumValN(warnOnly, "warn",
297 "Do not fail and just print warnings."),
298 clEnumValN(failIfAnyAreInvalid, "any",
299 "Fail if any profile is invalid."),
300 clEnumValN(failIfAllAreInvalid, "all",
301 "Fail only if all profiles are invalid.")));
302
303static cl::opt<bool> OutputSparse(
304 "sparse", cl::init(Val: false), cl::sub(MergeSubcommand),
305 cl::desc("Generate a sparse profile (only meaningful for -instr)"));
306static cl::opt<unsigned> NumThreads(
307 "num-threads", cl::init(Val: 0), cl::sub(MergeSubcommand),
308 cl::desc("Number of merge threads to use (default: autodetect)"));
309static cl::alias NumThreadsA("j", cl::desc("Alias for --num-threads"),
310 cl::aliasopt(NumThreads));
311
312static cl::opt<std::string> ProfileSymbolListFile(
313 "prof-sym-list", cl::init(Val: ""), cl::sub(MergeSubcommand),
314 cl::desc("Path to file containing the list of function symbols "
315 "used to populate profile symbol list"));
316
317static cl::opt<SampleProfileLayout> ProfileLayout(
318 "convert-sample-profile-layout",
319 cl::desc("Convert the generated profile to a profile with a new layout"),
320 cl::sub(MergeSubcommand), cl::init(Val: SPL_None),
321 cl::values(
322 clEnumValN(SPL_Nest, "nest",
323 "Nested profile, the input should be CS flat profile"),
324 clEnumValN(SPL_Flat, "flat",
325 "Profile with nested inlinee flatten out")));
326
327static cl::opt<bool> DropProfileSymbolList(
328 "drop-profile-symbol-list", cl::init(Val: false), cl::Hidden,
329 cl::sub(MergeSubcommand),
330 cl::desc("Drop the profile symbol list when merging AutoFDO profiles "
331 "(only meaningful for -sample)"));
332
333static cl::opt<bool> KeepVTableSymbols(
334 "keep-vtable-symbols", cl::init(Val: false), cl::Hidden,
335 cl::sub(MergeSubcommand),
336 cl::desc("If true, keep the vtable symbols in indexed profiles"));
337
338// Temporary support for writing the previous version of the format, to enable
339// some forward compatibility.
340// TODO: Consider enabling this with future version changes as well, to ease
341// deployment of newer versions of llvm-profdata.
342static cl::opt<bool> DoWritePrevVersion(
343 "write-prev-version", cl::init(Val: false), cl::Hidden,
344 cl::desc("Write the previous version of indexed format, to enable "
345 "some forward compatibility."));
346
347static cl::opt<memprof::IndexedVersion> MemProfVersionRequested(
348 "memprof-version", cl::Hidden, cl::sub(MergeSubcommand),
349 cl::desc("Specify the version of the memprof format to use"),
350 cl::init(Val: memprof::Version3),
351 cl::values(clEnumValN(memprof::Version3, "3", "version 3"),
352 clEnumValN(memprof::Version4, "4", "version 4")));
353
354static cl::opt<bool> MemProfFullSchema(
355 "memprof-full-schema", cl::Hidden, cl::sub(MergeSubcommand),
356 cl::desc("Use the full schema for serialization"), cl::init(Val: false));
357
358static cl::opt<bool> MemprofGenerateRandomHotness(
359 "memprof-random-hotness", cl::init(Val: false), cl::Hidden,
360 cl::sub(MergeSubcommand),
361 cl::desc("Generate random hotness values. Use -random-seed to set the seed "
362 "value, otherwise the constant default seed is used"));
363static cl::opt<unsigned>
364 RandomSeed("random-seed", cl::init(Val: 0), cl::Hidden, cl::sub(MergeSubcommand),
365 cl::desc("Seed for the random number generator used by "
366 "-memprof-random-hotness and temporal profile "
367 "reservoir sampling"));
368static cl::alias MemprofGenerateRandomHotnessSeed(
369 "memprof-random-hotness-seed", cl::Hidden,
370 cl::desc("Alias for -random-seed. Deprecated, please use -random-seed"),
371 cl::aliasopt(RandomSeed));
372
373// Options specific to overlap subcommand.
374static cl::opt<std::string> BaseFilename(cl::Positional, cl::Required,
375 cl::desc("<base profile file>"),
376 cl::sub(OverlapSubcommand));
377static cl::opt<std::string> TestFilename(cl::Positional, cl::Required,
378 cl::desc("<test profile file>"),
379 cl::sub(OverlapSubcommand));
380
381static cl::opt<unsigned long long> SimilarityCutoff(
382 "similarity-cutoff", cl::init(Val: 0),
383 cl::desc("For sample profiles, list function names (with calling context "
384 "for csspgo) for overlapped functions "
385 "with similarities below the cutoff (percentage times 10000)."),
386 cl::sub(OverlapSubcommand));
387
388static cl::opt<bool> IsCS(
389 "cs", cl::init(Val: false),
390 cl::desc("For context sensitive PGO counts. Does not work with CSSPGO."),
391 cl::sub(OverlapSubcommand));
392
393static cl::opt<unsigned long long> OverlapValueCutoff(
394 "value-cutoff", cl::init(Val: -1),
395 cl::desc(
396 "Function level overlap information for every function (with calling "
397 "context for csspgo) in test "
398 "profile with max count value greater than the parameter value"),
399 cl::sub(OverlapSubcommand));
400
401// Options specific to show subcommand.
402static cl::opt<bool>
403 ShowCounts("counts", cl::init(Val: false),
404 cl::desc("Show counter values for shown functions"),
405 cl::sub(ShowSubcommand));
406static cl::opt<ShowFormat>
407 SFormat("show-format", cl::init(Val: ShowFormat::Text),
408 cl::desc("Emit output in the selected format if supported"),
409 cl::sub(ShowSubcommand),
410 cl::values(clEnumValN(ShowFormat::Text, "text",
411 "emit normal text output (default)"),
412 clEnumValN(ShowFormat::Json, "json", "emit JSON"),
413 clEnumValN(ShowFormat::Yaml, "yaml", "emit YAML")));
414// TODO: Consider replacing this with `--show-format=text-encoding`.
415static cl::opt<bool>
416 TextFormat("text", cl::init(Val: false),
417 cl::desc("Show instr profile data in text dump format"),
418 cl::sub(ShowSubcommand));
419static cl::opt<bool>
420 JsonFormat("json",
421 cl::desc("Show sample profile data in the JSON format "
422 "(deprecated, please use --show-format=json)"),
423 cl::sub(ShowSubcommand));
424static cl::opt<bool> ShowIndirectCallTargets(
425 "ic-targets", cl::init(Val: false),
426 cl::desc("Show indirect call site target values for shown functions"),
427 cl::sub(ShowSubcommand));
428static cl::opt<bool>
429 ShowVTables("show-vtables", cl::init(Val: false),
430 cl::desc("Show vtable names for shown functions"),
431 cl::sub(ShowSubcommand));
432static cl::opt<bool> ShowMemOPSizes(
433 "memop-sizes", cl::init(Val: false),
434 cl::desc("Show the profiled sizes of the memory intrinsic calls "
435 "for shown functions"),
436 cl::sub(ShowSubcommand));
437static cl::opt<bool>
438 ShowDetailedSummary("detailed-summary", cl::init(Val: false),
439 cl::desc("Show detailed profile summary"),
440 cl::sub(ShowSubcommand));
441static cl::list<uint32_t> DetailedSummaryCutoffs(
442 cl::CommaSeparated, "detailed-summary-cutoffs",
443 cl::desc(
444 "Cutoff percentages (times 10000) for generating detailed summary"),
445 cl::value_desc("800000,901000,999999"), cl::sub(ShowSubcommand));
446static cl::opt<bool>
447 ShowHotFuncList("hot-func-list", cl::init(Val: false),
448 cl::desc("Show profile summary of a list of hot functions"),
449 cl::sub(ShowSubcommand));
450static cl::opt<bool>
451 ShowAllFunctions("all-functions", cl::init(Val: false),
452 cl::desc("Details for each and every function"),
453 cl::sub(ShowSubcommand));
454static cl::opt<bool> ShowCS("showcs", cl::init(Val: false),
455 cl::desc("Show context sensitive counts"),
456 cl::sub(ShowSubcommand));
457static cl::opt<ProfileKinds> ShowProfileKind(
458 cl::desc("Profile kind supported by show:"), cl::sub(ShowSubcommand),
459 cl::init(Val: instr),
460 cl::values(clEnumVal(instr, "Instrumentation profile (default)"),
461 clEnumVal(sample, "Sample profile"),
462 clEnumVal(memory, "MemProf memory access profile")));
463static cl::opt<uint32_t> TopNFunctions(
464 "topn", cl::init(Val: 0),
465 cl::desc("Show the list of functions with the largest internal counts"),
466 cl::sub(ShowSubcommand));
467static cl::opt<uint32_t> ShowValueCutoff(
468 "value-cutoff", cl::init(Val: 0),
469 cl::desc("Set the count value cutoff. Functions with the maximum count "
470 "less than this value will not be printed out. (Default is 0)"),
471 cl::sub(ShowSubcommand));
472static cl::opt<bool> OnlyListBelow(
473 "list-below-cutoff", cl::init(Val: false),
474 cl::desc("Only output names of functions whose max count values are "
475 "below the cutoff value"),
476 cl::sub(ShowSubcommand));
477static cl::opt<bool> ShowProfileSymbolList(
478 "show-prof-sym-list", cl::init(Val: false),
479 cl::desc("Show profile symbol list if it exists in the profile. "),
480 cl::sub(ShowSubcommand));
481static cl::opt<bool> ShowSectionInfoOnly(
482 "show-sec-info-only", cl::init(Val: false),
483 cl::desc("Show the information of each section in the sample profile. "
484 "The flag is only usable when the sample profile is in "
485 "extbinary format"),
486 cl::sub(ShowSubcommand));
487static cl::opt<bool> ShowCompositeInfoOnly(
488 "show-composite-info-only", cl::init(Val: false),
489 cl::desc("Show type IDs and payload sizes in a composite sample profile"),
490 cl::sub(ShowSubcommand));
491static cl::opt<bool> ShowBinaryIds("binary-ids", cl::init(Val: false),
492 cl::desc("Show binary ids in the profile. "),
493 cl::sub(ShowSubcommand));
494static cl::opt<bool> ShowTemporalProfTraces(
495 "temporal-profile-traces",
496 cl::desc("Show temporal profile traces in the profile."),
497 cl::sub(ShowSubcommand));
498
499static cl::opt<bool>
500 ShowCovered("covered", cl::init(Val: false),
501 cl::desc("Show only the functions that have been executed."),
502 cl::sub(ShowSubcommand));
503
504static cl::opt<bool> ShowProfileVersion("profile-version", cl::init(Val: false),
505 cl::desc("Show profile version. "),
506 cl::sub(ShowSubcommand));
507
508// Options specific to order subcommand.
509static cl::opt<unsigned>
510 NumTestTraces("num-test-traces", cl::init(Val: 0),
511 cl::desc("Keep aside the last <num-test-traces> traces in "
512 "the profile when computing the function order and "
513 "instead use them to evaluate that order"),
514 cl::sub(OrderSubcommand));
515
516// We use this string to indicate that there are
517// multiple static functions map to the same name.
518const std::string DuplicateNameStr = "----";
519
520static void warn(Twine Message, StringRef Whence = "", StringRef Hint = "") {
521 WithColor::warning();
522 if (!Whence.empty())
523 errs() << Whence << ": ";
524 errs() << Message << "\n";
525 if (!Hint.empty())
526 WithColor::note() << Hint << "\n";
527}
528
529static void warn(Error E, StringRef Whence = "") {
530 if (E.isA<InstrProfError>()) {
531 handleAllErrors(E: std::move(E), Handlers: [&](const InstrProfError &IPE) {
532 warn(Message: IPE.message(), Whence);
533 });
534 }
535}
536
537namespace {
538class ProfdataError : public ErrorInfo<ProfdataError> {
539public:
540 static char ID;
541
542 ProfdataError(Twine Message, Twine Whence = "", Twine Hint = "")
543 : Message(Message.str()), Whence(Whence.str()), Hint(Hint.str()) {}
544
545 void log(raw_ostream &OS) const override {
546 if (!Whence.empty())
547 OS << Whence << ": ";
548 OS << Message;
549 }
550
551 void print() const {
552 WithColor::error();
553 log(OS&: errs());
554 errs() << "\n";
555 if (!Hint.empty())
556 WithColor::note() << Hint << "\n";
557 }
558
559 std::error_code convertToErrorCode() const override {
560 return inconvertibleErrorCode();
561 }
562
563private:
564 std::string Message;
565 std::string Whence;
566 std::string Hint;
567};
568
569char ProfdataError::ID = 0;
570} // namespace
571
572static Error makeError(Twine Message, StringRef Whence = "",
573 StringRef Hint = "") {
574 return make_error<ProfdataError>(Args&: Message, Args&: Whence, Args&: Hint);
575}
576
577static Error makeError(Error E, StringRef Whence = "") {
578 if (E.isA<InstrProfError>()) {
579 std::string Msg;
580 std::string Hint;
581 handleAllErrors(E: std::move(E), Handlers: [&](const InstrProfError &IPE) {
582 instrprof_error instrError = IPE.get();
583 if (instrError == instrprof_error::unrecognized_format) {
584 // Hint in case user missed specifying the profile type.
585 Hint = "Perhaps you forgot to use the --sample or --memory option?";
586 }
587 Msg = IPE.message();
588 });
589 return makeError(Message: Msg, Whence, Hint);
590 }
591
592 return makeError(Message: toString(E: std::move(E)), Whence);
593}
594
595static Error makeError(std::error_code EC, StringRef Whence = "") {
596 return makeError(Message: EC.message(), Whence);
597}
598
599static int reportError(Error E) {
600 if (!E)
601 return 0;
602 handleAllErrors(
603 E: std::move(E), Handlers: [](const ProfdataError &PE) { PE.print(); },
604 Handlers: [](const ErrorInfoBase &EIB) {
605 WithColor::error() << EIB.message() << "\n";
606 });
607 return 1;
608}
609
610static Error warnOrErrorGivenError(FailureMode FailMode, std::error_code EC,
611 StringRef Whence = "") {
612 if (FailMode == failIfAnyAreInvalid)
613 return makeError(EC, Whence);
614 warn(Message: EC.message(), Whence);
615 return Error::success();
616}
617
618static void handleMergeWriterError(Error E, StringRef WhenceFile = "",
619 StringRef WhenceFunction = "",
620 bool ShowHint = true) {
621 if (!WhenceFile.empty())
622 errs() << WhenceFile << ": ";
623 if (!WhenceFunction.empty())
624 errs() << WhenceFunction << ": ";
625
626 auto IPE = instrprof_error::success;
627 E = handleErrors(E: std::move(E),
628 Hs: [&IPE](std::unique_ptr<InstrProfError> E) -> Error {
629 IPE = E->get();
630 return Error(std::move(E));
631 });
632 errs() << toString(E: std::move(E)) << "\n";
633
634 if (ShowHint) {
635 StringRef Hint = "";
636 if (IPE != instrprof_error::success) {
637 switch (IPE) {
638 case instrprof_error::hash_mismatch:
639 case instrprof_error::count_mismatch:
640 case instrprof_error::value_site_count_mismatch:
641 Hint = "Make sure that all profile data to be merged is generated "
642 "from the same binary.";
643 break;
644 default:
645 break;
646 }
647 }
648
649 if (!Hint.empty())
650 errs() << Hint << "\n";
651 }
652}
653
654namespace {
655/// A remapper from original symbol names to new symbol names based on a file
656/// containing a list of mappings from old name to new name.
657class SymbolRemapper {
658 std::unique_ptr<MemoryBuffer> File;
659 DenseMap<StringRef, StringRef> RemappingTable;
660
661public:
662 /// Build a SymbolRemapper from a file containing a list of old/new symbols.
663 static Expected<std::unique_ptr<SymbolRemapper>> create(StringRef InputFile) {
664 auto BufOrError = MemoryBuffer::getFileOrSTDIN(Filename: InputFile);
665 if (!BufOrError)
666 return makeError(EC: BufOrError.getError(), Whence: InputFile);
667
668 auto Remapper = std::make_unique<SymbolRemapper>();
669 Remapper->File = std::move(BufOrError.get());
670
671 for (line_iterator LineIt(*Remapper->File, /*SkipBlanks=*/true, '#');
672 !LineIt.is_at_eof(); ++LineIt) {
673 std::pair<StringRef, StringRef> Parts = LineIt->split(Separator: ' ');
674 if (Parts.first.empty() || Parts.second.empty() ||
675 Parts.second.count(C: ' ')) {
676 return makeError(Message: "unexpected line in remapping file",
677 Whence: (InputFile + ":" + Twine(LineIt.line_number())).str(),
678 Hint: "expected 'old_symbol new_symbol'");
679 }
680 Remapper->RemappingTable.insert(KV: Parts);
681 }
682 return std::move(Remapper);
683 }
684
685 /// Attempt to map the given old symbol into a new symbol.
686 ///
687 /// \return The new symbol, or \p Name if no such symbol was found.
688 StringRef operator()(StringRef Name) {
689 StringRef New = RemappingTable.lookup(Val: Name);
690 return New.empty() ? Name : New;
691 }
692
693 FunctionId operator()(FunctionId Name) {
694 // MD5 name cannot be remapped.
695 if (!Name.isStringRef())
696 return Name;
697 StringRef New = RemappingTable.lookup(Val: Name.stringRef());
698 return New.empty() ? Name : FunctionId(New);
699 }
700};
701}
702
703struct WeightedFile {
704 std::string Filename;
705 uint64_t Weight;
706};
707typedef SmallVector<WeightedFile, 5> WeightedFileVector;
708
709/// Keep track of merged data and reported errors.
710struct WriterContext {
711 std::mutex Lock;
712 InstrProfWriter Writer;
713 std::vector<std::pair<Error, std::string>> Errors;
714 std::mutex &ErrLock;
715 SmallSet<instrprof_error, 4> &WriterErrorCodes;
716
717 WriterContext(bool IsSparse, std::mutex &ErrLock,
718 SmallSet<instrprof_error, 4> &WriterErrorCodes,
719 uint64_t ReservoirSize = 0, uint64_t MaxTraceLength = 0)
720 : Writer(IsSparse, ReservoirSize, MaxTraceLength, DoWritePrevVersion,
721 MemProfVersionRequested, MemProfFullSchema,
722 MemprofGenerateRandomHotness, RandomSeed),
723 ErrLock(ErrLock), WriterErrorCodes(WriterErrorCodes) {}
724
725 ~WriterContext() {
726 for (auto &ErrorPair : Errors)
727 consumeError(Err: std::move(ErrorPair.first));
728 }
729};
730
731/// Computer the overlap b/w profile BaseFilename and TestFileName,
732/// and store the program level result to Overlap.
733static void overlapInput(const std::string &BaseFilename,
734 const std::string &TestFilename, WriterContext *WC,
735 OverlapStats &Overlap,
736 const OverlapFuncFilters &FuncFilter,
737 raw_fd_ostream &OS, bool IsCS) {
738 auto FS = vfs::getRealFileSystem();
739 auto ReaderOrErr = InstrProfReader::create(Path: TestFilename, FS&: *FS);
740 if (Error E = ReaderOrErr.takeError()) {
741 // Skip the empty profiles by returning sliently.
742 auto [ErrorCode, Msg] = InstrProfError::take(E: std::move(E));
743 if (ErrorCode != instrprof_error::empty_raw_profile)
744 WC->Errors.emplace_back(args: make_error<InstrProfError>(Args&: ErrorCode, Args&: Msg),
745 args: TestFilename);
746 return;
747 }
748
749 auto Reader = std::move(ReaderOrErr.get());
750 for (auto &I : *Reader) {
751 OverlapStats FuncOverlap(OverlapStats::FunctionLevel);
752 FuncOverlap.setFuncInfo(Name: I.Name, Hash: I.Hash);
753
754 WC->Writer.overlapRecord(Other: std::move(I), Overlap, FuncLevelOverlap&: FuncOverlap, FuncFilter);
755 FuncOverlap.dump(OS);
756 }
757}
758
759/// Load an input into a writer context.
760static Error
761loadInput(const WeightedFile &Input, SymbolRemapper *Remapper,
762 const InstrProfCorrelator *Correlator, const StringRef ProfiledBinary,
763 WriterContext *WC, const object::BuildIDFetcher *BIDFetcher = nullptr,
764 const ProfCorrelatorKind *BIDFetcherCorrelatorKind = nullptr) {
765 std::unique_lock<std::mutex> CtxGuard{WC->Lock};
766
767 // Copy the filename, because llvm::ThreadPool copied the input "const
768 // WeightedFile &" by value, making a reference to the filename within it
769 // invalid outside of this packaged task.
770 std::string Filename = Input.Filename;
771
772 using ::llvm::memprof::RawMemProfReader;
773 if (RawMemProfReader::hasFormat(Path: Input.Filename)) {
774 auto ReaderOrErr = RawMemProfReader::create(Path: Input.Filename, ProfiledBinary);
775 if (!ReaderOrErr)
776 return makeError(E: ReaderOrErr.takeError(), Whence: Input.Filename);
777 std::unique_ptr<RawMemProfReader> Reader = std::move(ReaderOrErr.get());
778 // Check if the profile types can be merged, e.g. clang frontend profiles
779 // should not be merged with memprof profiles.
780 if (Error E = WC->Writer.mergeProfileKind(Other: Reader->getProfileKind())) {
781 consumeError(Err: std::move(E));
782 WC->Errors.emplace_back(
783 args: make_error<StringError>(
784 Args: "Cannot merge MemProf profile with Clang generated profile.",
785 Args: std::error_code()),
786 args&: Filename);
787 return Error::success();
788 }
789
790 auto MemProfError = [&](Error E) {
791 auto [ErrorCode, Msg] = InstrProfError::take(E: std::move(E));
792 WC->Errors.emplace_back(args: make_error<InstrProfError>(Args&: ErrorCode, Args&: Msg),
793 args&: Filename);
794 };
795
796 WC->Writer.addMemProfData(Incoming: Reader->takeMemProfData(), Warn: MemProfError);
797 return Error::success();
798 }
799
800 using ::llvm::memprof::YAMLMemProfReader;
801 if (YAMLMemProfReader::hasFormat(Path: Input.Filename)) {
802 auto ReaderOrErr = YAMLMemProfReader::create(Path: Input.Filename);
803 if (!ReaderOrErr)
804 return makeError(E: ReaderOrErr.takeError(), Whence: Input.Filename);
805 std::unique_ptr<YAMLMemProfReader> Reader = std::move(ReaderOrErr.get());
806 // Check if the profile types can be merged, e.g. clang frontend profiles
807 // should not be merged with memprof profiles.
808 if (Error E = WC->Writer.mergeProfileKind(Other: Reader->getProfileKind())) {
809 consumeError(Err: std::move(E));
810 WC->Errors.emplace_back(
811 args: make_error<StringError>(
812 Args: "Cannot merge MemProf profile with incompatible profile.",
813 Args: std::error_code()),
814 args&: Filename);
815 return Error::success();
816 }
817
818 auto MemProfError = [&](Error E) {
819 auto [ErrorCode, Msg] = InstrProfError::take(E: std::move(E));
820 WC->Errors.emplace_back(args: make_error<InstrProfError>(Args&: ErrorCode, Args&: Msg),
821 args&: Filename);
822 };
823
824 auto MemProfData = Reader->takeMemProfData();
825
826 auto DataAccessProfData = Reader->takeDataAccessProfData();
827
828 // Check for the empty input in case the YAML file is invalid.
829 if (MemProfData.Records.empty() &&
830 (!DataAccessProfData || DataAccessProfData->empty())) {
831 WC->Errors.emplace_back(
832 args: make_error<StringError>(Args: "The profile is empty.", Args: std::error_code()),
833 args&: Filename);
834 }
835
836 WC->Writer.addMemProfData(Incoming: std::move(MemProfData), Warn: MemProfError);
837 WC->Writer.addDataAccessProfData(DataAccessProfile: std::move(DataAccessProfData));
838 return Error::success();
839 }
840
841 auto FS = vfs::getRealFileSystem();
842 // TODO: This only saves the first non-fatal error from InstrProfReader, and
843 // then added to WriterContext::Errors. However, this is not extensible, if
844 // we have more non-fatal errors from InstrProfReader in the future. How
845 // should this interact with different -failure-mode?
846 std::optional<std::pair<Error, std::string>> ReaderWarning;
847 llvm::scope_exit ReaderWarningScope([&] {
848 // If we hit a different error we may still have an error in ReaderWarning.
849 // Consume it now to avoid an assert
850 if (ReaderWarning)
851 consumeError(Err: std::move(ReaderWarning->first));
852 });
853 auto Warn = [&](Error E) {
854 if (ReaderWarning) {
855 consumeError(Err: std::move(E));
856 return;
857 }
858 // Only show the first time an error occurs in this file.
859 auto [ErrCode, Msg] = InstrProfError::take(E: std::move(E));
860 ReaderWarning = {make_error<InstrProfError>(Args&: ErrCode, Args&: Msg), Filename};
861 };
862
863 const ProfCorrelatorKind CorrelatorKind = BIDFetcherCorrelatorKind
864 ? *BIDFetcherCorrelatorKind
865 : ProfCorrelatorKind::NONE;
866 auto ReaderOrErr = InstrProfReader::create(Path: Input.Filename, FS&: *FS, Correlator,
867 BIDFetcher, BIDFetcherCorrelatorKind: CorrelatorKind, Warn);
868 if (Error E = ReaderOrErr.takeError()) {
869 // Skip the empty profiles by returning silently.
870 auto [ErrCode, Msg] = InstrProfError::take(E: std::move(E));
871 if (ErrCode != instrprof_error::empty_raw_profile)
872 WC->Errors.emplace_back(args: make_error<InstrProfError>(Args&: ErrCode, Args&: Msg),
873 args&: Filename);
874 return Error::success();
875 }
876
877 auto Reader = std::move(ReaderOrErr.get());
878 if (Error E = WC->Writer.mergeProfileKind(Other: Reader->getProfileKind())) {
879 WC->Errors.emplace_back(args: std::move(E), args&: Filename);
880 return Error::success();
881 }
882
883 for (auto &I : *Reader) {
884 if (Remapper)
885 I.Name = (*Remapper)(I.Name);
886 const StringRef FuncName = I.Name;
887 bool Reported = false;
888
889 WC->Writer.addRecord(I: std::move(I), Weight: Input.Weight, Warn: [&](Error E) {
890 if (Reported) {
891 consumeError(Err: std::move(E));
892 return;
893 }
894 Reported = true;
895 // Only show hint the first time an error occurs.
896 auto [ErrCode, Msg] = InstrProfError::take(E: std::move(E));
897 std::unique_lock<std::mutex> ErrGuard{WC->ErrLock};
898 bool firstTime = WC->WriterErrorCodes.insert(V: ErrCode).second;
899 handleMergeWriterError(E: make_error<InstrProfError>(Args&: ErrCode, Args&: Msg),
900 WhenceFile: Input.Filename, WhenceFunction: FuncName, ShowHint: firstTime);
901 });
902 }
903
904 if (KeepVTableSymbols) {
905 const InstrProfSymtab &symtab = Reader->getSymtab();
906 const auto &VTableNames = symtab.getVTableNames();
907
908 for (const auto &kv : VTableNames)
909 WC->Writer.addVTableName(VTableName: kv.getKey());
910 }
911
912 if (Reader->hasTemporalProfile()) {
913 auto &Traces = Reader->getTemporalProfTraces(Weight: Input.Weight);
914 if (!Traces.empty())
915 WC->Writer.addTemporalProfileTraces(
916 SrcTraces&: Traces, SrcStreamSize: Reader->getTemporalProfTraceStreamSize());
917 }
918 if (Reader->hasError()) {
919 if (Error E = Reader->getError()) {
920 WC->Errors.emplace_back(args: std::move(E), args&: Filename);
921 return Error::success();
922 }
923 }
924
925 std::vector<llvm::object::BuildID> BinaryIds;
926 if (Error E = Reader->readBinaryIds(BinaryIds)) {
927 WC->Errors.emplace_back(args: std::move(E), args&: Filename);
928 return Error::success();
929 }
930 WC->Writer.addBinaryIds(BIs: BinaryIds);
931
932 if (ReaderWarning) {
933 WC->Errors.emplace_back(args: std::move(ReaderWarning->first),
934 args&: ReaderWarning->second);
935 }
936 return Error::success();
937}
938
939/// Merge the \p Src writer context into \p Dst.
940static Error mergeWriterContexts(WriterContext *Dst, WriterContext *Src) {
941 for (auto &ErrorPair : Src->Errors)
942 Dst->Errors.push_back(x: std::move(ErrorPair));
943 Src->Errors.clear();
944
945 if (Error E = Dst->Writer.mergeProfileKind(Other: Src->Writer.getProfileKind()))
946 return makeError(E: std::move(E));
947
948 Dst->Writer.mergeRecordsFromWriter(IPW: std::move(Src->Writer), Warn: [&](Error E) {
949 auto [ErrorCode, Msg] = InstrProfError::take(E: std::move(E));
950 std::unique_lock<std::mutex> ErrGuard{Dst->ErrLock};
951 bool firstTime = Dst->WriterErrorCodes.insert(V: ErrorCode).second;
952 if (firstTime)
953 warn(Message: toString(E: make_error<InstrProfError>(Args&: ErrorCode, Args&: Msg)));
954 });
955 return Error::success();
956}
957
958static StringRef
959getFuncName(const StringMap<InstrProfWriter::ProfilingData>::value_type &Val) {
960 return Val.first();
961}
962
963static std::string
964getFuncName(const SampleProfileMap::value_type &Val) {
965 return Val.second.getContext().toString();
966}
967
968template <typename T> static Error filterFunctions(T &ProfileMap) {
969 bool hasFilter = !FuncNameFilter.empty();
970 bool hasNegativeFilter = !FuncNameNegativeFilter.empty();
971 if (!hasFilter && !hasNegativeFilter)
972 return Error::success();
973
974 // If filter starts with '?' it is MSVC mangled name, not a regex.
975 llvm::Regex ProbablyMSVCMangledName("[?@$_0-9A-Za-z]+");
976 if (hasFilter && FuncNameFilter[0] == '?' &&
977 ProbablyMSVCMangledName.match(String: FuncNameFilter))
978 FuncNameFilter = llvm::Regex::escape(String: FuncNameFilter);
979 if (hasNegativeFilter && FuncNameNegativeFilter[0] == '?' &&
980 ProbablyMSVCMangledName.match(String: FuncNameNegativeFilter))
981 FuncNameNegativeFilter = llvm::Regex::escape(String: FuncNameNegativeFilter);
982
983 size_t Count = ProfileMap.size();
984 llvm::Regex Pattern(FuncNameFilter);
985 llvm::Regex NegativePattern(FuncNameNegativeFilter);
986 std::string RegexError;
987 if (hasFilter && !Pattern.isValid(Error&: RegexError))
988 return makeError(Message: RegexError);
989 if (hasNegativeFilter && !NegativePattern.isValid(Error&: RegexError))
990 return makeError(Message: RegexError);
991
992 // Handle MD5 profile, so it is still able to match using the original name.
993 std::string MD5Name = std::to_string(val: llvm::MD5Hash(Str: FuncNameFilter));
994 std::string NegativeMD5Name =
995 std::to_string(val: llvm::MD5Hash(Str: FuncNameNegativeFilter));
996
997 ProfileMap.remove_if([&](const auto &Entry) {
998 const auto &FuncName = getFuncName(Entry);
999 // Negative filter has higher precedence than positive filter.
1000 return (hasNegativeFilter &&
1001 (NegativePattern.match(String: FuncName) ||
1002 (FunctionSamples::UseMD5 && NegativeMD5Name == FuncName))) ||
1003 (hasFilter && !(Pattern.match(String: FuncName) ||
1004 (FunctionSamples::UseMD5 && MD5Name == FuncName)));
1005 });
1006
1007 llvm::dbgs() << Count - ProfileMap.size() << " of " << Count << " functions "
1008 << "in the original profile are filtered.\n";
1009 return Error::success();
1010}
1011
1012static Error writeInstrProfile(StringRef OutputFilename,
1013 ProfileFormat OutputFormat,
1014 InstrProfWriter &Writer) {
1015 std::error_code EC;
1016 raw_fd_ostream Output(OutputFilename.data(), EC,
1017 OutputFormat == PF_Text ? sys::fs::OF_TextWithCRLF
1018 : sys::fs::OF_None);
1019 if (EC)
1020 return makeError(EC, Whence: OutputFilename);
1021
1022 if (OutputFormat == PF_Text) {
1023 if (Error E = Writer.writeText(OS&: Output))
1024 warn(E: std::move(E));
1025 } else {
1026 if (Output.is_displayed())
1027 return makeError(
1028 Message: "cannot write a non-text format profile to the terminal");
1029 if (Error E = Writer.write(OS&: Output))
1030 warn(E: std::move(E));
1031 }
1032 return Error::success();
1033}
1034
1035static Error mergeInstrProfile(const WeightedFileVector &Inputs,
1036 SymbolRemapper *Remapper,
1037 int MaxDbgCorrelationWarnings,
1038 const StringRef ProfiledBinary) {
1039 const uint64_t TraceReservoirSize = TemporalProfTraceReservoirSize.getValue();
1040 const uint64_t MaxTraceLength = TemporalProfMaxTraceLength.getValue();
1041 if (OutputFormat == PF_Compact_Binary)
1042 return makeError(Message: "Compact Binary is deprecated");
1043 if (OutputFormat != PF_Binary && OutputFormat != PF_Ext_Binary &&
1044 OutputFormat != PF_Text)
1045 return makeError(Message: "unknown format is specified");
1046
1047 // TODO: Maybe we should support correlation with mixture of different
1048 // correlation modes(w/wo debug-info/object correlation).
1049 if (DebugInfoFilename.empty()) {
1050 if (!BinaryFilename.empty() && (DebugInfod || !DebugFileDirectory.empty()))
1051 return makeError(Message: "Expected only one of -binary-file, -debuginfod or "
1052 "-debug-file-directory");
1053 } else if (!BinaryFilename.empty() || DebugInfod ||
1054 !DebugFileDirectory.empty()) {
1055 return makeError(
1056 Message: "Expected only one of -debug-info, -binary-file, -debuginfod "
1057 "or -debug-file-directory");
1058 }
1059 std::string CorrelateFilename;
1060 ProfCorrelatorKind CorrelateKind = ProfCorrelatorKind::NONE;
1061 if (!DebugInfoFilename.empty()) {
1062 CorrelateFilename = DebugInfoFilename;
1063 CorrelateKind = ProfCorrelatorKind::DEBUG_INFO;
1064 } else if (!BinaryFilename.empty()) {
1065 CorrelateFilename = BinaryFilename;
1066 CorrelateKind = ProfCorrelatorKind::BINARY;
1067 }
1068
1069 std::unique_ptr<InstrProfCorrelator> Correlator;
1070 if (CorrelateKind != InstrProfCorrelator::NONE) {
1071 if (auto Err = InstrProfCorrelator::get(Filename: CorrelateFilename, FileKind: CorrelateKind)
1072 .moveInto(Value&: Correlator))
1073 return makeError(E: std::move(Err), Whence: CorrelateFilename);
1074 if (auto Err = Correlator->correlateProfileData(MaxWarnings: MaxDbgCorrelationWarnings))
1075 return makeError(E: std::move(Err), Whence: CorrelateFilename);
1076 }
1077
1078 ProfCorrelatorKind BIDFetcherCorrelateKind = ProfCorrelatorKind::NONE;
1079 std::unique_ptr<object::BuildIDFetcher> BIDFetcher;
1080 if (DebugInfod) {
1081 llvm::HTTPClient::initialize();
1082 BIDFetcher = std::make_unique<DebuginfodFetcher>(args&: DebugFileDirectory);
1083 if (!BIDFetcherProfileCorrelate)
1084 return makeError(Message: "Expected --correlate when --debuginfod is provided");
1085 BIDFetcherCorrelateKind = BIDFetcherProfileCorrelate;
1086 } else if (!DebugFileDirectory.empty()) {
1087 BIDFetcher = std::make_unique<object::BuildIDFetcher>(args&: DebugFileDirectory);
1088 if (!BIDFetcherProfileCorrelate)
1089 return makeError(Message: "Expected --correlate when --debug-file-directory "
1090 "is provided");
1091 BIDFetcherCorrelateKind = BIDFetcherProfileCorrelate;
1092 } else if (BIDFetcherProfileCorrelate) {
1093 return makeError(Message: "Expected --debuginfod or --debug-file-directory when "
1094 "--correlate is provided");
1095 }
1096
1097 std::mutex ErrorLock;
1098 SmallSet<instrprof_error, 4> WriterErrorCodes;
1099
1100 // If NumThreads is not specified, auto-detect a good default.
1101 if (NumThreads == 0)
1102 NumThreads = std::min(a: hardware_concurrency().compute_thread_count(),
1103 b: unsigned((Inputs.size() + 1) / 2));
1104
1105 // Initialize the writer contexts.
1106 SmallVector<std::unique_ptr<WriterContext>, 4> Contexts;
1107 for (unsigned I = 0; I < NumThreads; ++I)
1108 Contexts.emplace_back(Args: std::make_unique<WriterContext>(
1109 args&: OutputSparse, args&: ErrorLock, args&: WriterErrorCodes, args: TraceReservoirSize,
1110 args: MaxTraceLength));
1111
1112 if (NumThreads == 1) {
1113 for (const auto &Input : Inputs)
1114 if (Error E = loadInput(Input, Remapper, Correlator: Correlator.get(), ProfiledBinary,
1115 WC: Contexts[0].get(), BIDFetcher: BIDFetcher.get(),
1116 BIDFetcherCorrelatorKind: &BIDFetcherCorrelateKind))
1117 return E;
1118 } else {
1119 Error FatalError = Error::success();
1120 auto hasFatalError = [&] {
1121 std::unique_lock<std::mutex> ErrGuard{ErrorLock};
1122 return static_cast<bool>(FatalError);
1123 };
1124
1125 DefaultThreadPool Pool(hardware_concurrency(ThreadCount: NumThreads));
1126 auto Async = [&](auto F, auto &&...Args) {
1127 Pool.async(
1128 [&, F](auto &&...InnerArgs) {
1129 if (hasFatalError())
1130 return;
1131 if (Error E = F(std::forward<decltype(InnerArgs)>(InnerArgs)...)) {
1132 std::unique_lock<std::mutex> ErrGuard{ErrorLock};
1133 if (FatalError)
1134 consumeError(Err: std::move(E));
1135 else
1136 FatalError = std::move(E);
1137 }
1138 },
1139 std::forward<decltype(Args)>(Args)...);
1140 };
1141
1142 // Load the inputs in parallel (N/NumThreads serial steps).
1143 unsigned Ctx = 0;
1144 for (const auto &Input : Inputs) {
1145 if (hasFatalError())
1146 break;
1147 Async(loadInput, Input, Remapper, Correlator.get(), ProfiledBinary,
1148 Contexts[Ctx].get(), BIDFetcher.get(), &BIDFetcherCorrelateKind);
1149 Ctx = (Ctx + 1) % NumThreads;
1150 }
1151 Pool.wait();
1152 if (FatalError)
1153 return FatalError;
1154
1155 // Merge the writer contexts together (~ lg(NumThreads) serial steps).
1156 unsigned Mid = Contexts.size() / 2;
1157 unsigned End = Contexts.size();
1158 assert(Mid > 0 && "Expected more than one context");
1159 do {
1160 for (unsigned I = 0; I < Mid; ++I)
1161 Async(mergeWriterContexts, Contexts[I].get(), Contexts[I + Mid].get());
1162 Pool.wait();
1163 if (FatalError)
1164 return FatalError;
1165 if (End & 1) {
1166 Async(mergeWriterContexts, Contexts[0].get(), Contexts[End - 1].get());
1167 Pool.wait();
1168 if (FatalError)
1169 return FatalError;
1170 }
1171 End = Mid;
1172 Mid /= 2;
1173 } while (Mid > 0);
1174 }
1175
1176 // Handle deferred errors encountered during merging. If the number of errors
1177 // is equal to the number of inputs the merge failed.
1178 unsigned NumErrors = 0;
1179 for (std::unique_ptr<WriterContext> &WC : Contexts) {
1180 for (auto &ErrorPair : WC->Errors) {
1181 ++NumErrors;
1182 warn(Message: toString(E: std::move(ErrorPair.first)), Whence: ErrorPair.second);
1183 }
1184 }
1185 if ((NumErrors == Inputs.size() && FailMode == failIfAllAreInvalid) ||
1186 (NumErrors > 0 && FailMode == failIfAnyAreInvalid))
1187 return makeError(Message: "no profile can be merged");
1188
1189 if (Error E = filterFunctions(ProfileMap&: Contexts[0]->Writer.getProfileData()))
1190 return E;
1191
1192 return writeInstrProfile(OutputFilename, OutputFormat, Writer&: Contexts[0]->Writer);
1193}
1194
1195/// The profile entry for a function in instrumentation profile.
1196struct InstrProfileEntry {
1197 uint64_t MaxCount = 0;
1198 uint64_t NumEdgeCounters = 0;
1199 float ZeroCounterRatio = 0.0;
1200 InstrProfRecord *ProfRecord;
1201 InstrProfileEntry(InstrProfRecord *Record);
1202 InstrProfileEntry() = default;
1203};
1204
1205InstrProfileEntry::InstrProfileEntry(InstrProfRecord *Record) {
1206 ProfRecord = Record;
1207 uint64_t CntNum = Record->Counts.size();
1208 uint64_t ZeroCntNum = 0;
1209 for (size_t I = 0; I < CntNum; ++I) {
1210 MaxCount = std::max(a: MaxCount, b: Record->Counts[I]);
1211 ZeroCntNum += !Record->Counts[I];
1212 }
1213 ZeroCounterRatio = (float)ZeroCntNum / CntNum;
1214 NumEdgeCounters = CntNum;
1215}
1216
1217/// Either set all the counters in the instr profile entry \p IFE to
1218/// -1 / -2 /in order to drop the profile or scale up the
1219/// counters in \p IFP to be above hot / cold threshold. We use
1220/// the ratio of zero counters in the profile of a function to
1221/// decide the profile is helpful or harmful for performance,
1222/// and to choose whether to scale up or drop it.
1223static void updateInstrProfileEntry(InstrProfileEntry &IFE, bool SetToHot,
1224 uint64_t HotInstrThreshold,
1225 uint64_t ColdInstrThreshold,
1226 float ZeroCounterThreshold) {
1227 InstrProfRecord *ProfRecord = IFE.ProfRecord;
1228 if (!IFE.MaxCount || IFE.ZeroCounterRatio > ZeroCounterThreshold) {
1229 // If all or most of the counters of the function are zero, the
1230 // profile is unaccountable and should be dropped. Reset all the
1231 // counters to be -1 / -2 and PGO profile-use will drop the profile.
1232 // All counters being -1 also implies that the function is hot so
1233 // PGO profile-use will also set the entry count metadata to be
1234 // above hot threshold.
1235 // All counters being -2 implies that the function is warm so
1236 // PGO profile-use will also set the entry count metadata to be
1237 // above cold threshold.
1238 auto Kind =
1239 (SetToHot ? InstrProfRecord::PseudoHot : InstrProfRecord::PseudoWarm);
1240 ProfRecord->setPseudoCount(Kind);
1241 return;
1242 }
1243
1244 // Scale up the MaxCount to be multiple times above hot / cold threshold.
1245 const unsigned MultiplyFactor = 3;
1246 uint64_t Threshold = (SetToHot ? HotInstrThreshold : ColdInstrThreshold);
1247 uint64_t Numerator = Threshold * MultiplyFactor;
1248
1249 // Make sure Threshold for warm counters is below the HotInstrThreshold.
1250 if (!SetToHot && Threshold >= HotInstrThreshold) {
1251 Threshold = (HotInstrThreshold + ColdInstrThreshold) / 2;
1252 }
1253
1254 uint64_t Denominator = IFE.MaxCount;
1255 if (Numerator <= Denominator)
1256 return;
1257 ProfRecord->scale(N: Numerator, D: Denominator, Warn: [&](instrprof_error E) {
1258 warn(Message: toString(E: make_error<InstrProfError>(Args&: E)));
1259 });
1260}
1261
1262const uint64_t ColdPercentileIdx = 15;
1263const uint64_t HotPercentileIdx = 11;
1264
1265using sampleprof::FSDiscriminatorPass;
1266
1267// Internal options to set FSDiscriminatorPass. Used in merge and show
1268// commands.
1269static cl::opt<FSDiscriminatorPass> FSDiscriminatorPassOption(
1270 "fs-discriminator-pass", cl::init(Val: PassLast), cl::Hidden,
1271 cl::desc("Zero out the discriminator bits for the FS discrimiantor "
1272 "pass beyond this value. The enum values are defined in "
1273 "Support/Discriminator.h"),
1274 cl::values(clEnumVal(Base, "Use base discriminators only"),
1275 clEnumVal(Pass1, "Use base and pass 1 discriminators"),
1276 clEnumVal(Pass2, "Use base and pass 1-2 discriminators"),
1277 clEnumVal(Pass3, "Use base and pass 1-3 discriminators"),
1278 clEnumVal(PassLast, "Use all discriminator bits (default)")));
1279
1280static unsigned getDiscriminatorMask() {
1281 return getN1Bits(N: getFSPassBitEnd(P: FSDiscriminatorPassOption.getValue()));
1282}
1283
1284/// Adjust the instr profile in \p WC based on the sample profile in
1285/// \p Reader.
1286static void
1287adjustInstrProfile(std::unique_ptr<WriterContext> &WC,
1288 std::unique_ptr<sampleprof::SampleProfileReader> &Reader,
1289 unsigned SupplMinSizeThreshold, float ZeroCounterThreshold,
1290 unsigned InstrProfColdThreshold) {
1291 // Function to its entry in instr profile.
1292 StringMap<InstrProfileEntry> InstrProfileMap;
1293 StringMap<StringRef> StaticFuncMap;
1294 InstrProfSummaryBuilder IPBuilder(ProfileSummaryBuilder::DefaultCutoffs);
1295
1296 auto checkSampleProfileHasFUnique = [&Reader]() {
1297 for (const auto &PD : Reader->getProfiles()) {
1298 auto &FContext = PD.second.getContext();
1299 if (FContext.toString().find(s: FunctionSamples::UniqSuffix) !=
1300 std::string::npos) {
1301 return true;
1302 }
1303 }
1304 return false;
1305 };
1306
1307 bool SampleProfileHasFUnique = checkSampleProfileHasFUnique();
1308
1309 auto buildStaticFuncMap = [&StaticFuncMap,
1310 SampleProfileHasFUnique](const StringRef Name) {
1311 std::string FilePrefixes[] = {".cpp", "cc", ".c", ".hpp", ".h"};
1312 size_t PrefixPos = StringRef::npos;
1313 for (auto &FilePrefix : FilePrefixes) {
1314 std::string NamePrefix = FilePrefix + GlobalIdentifierDelimiter;
1315 PrefixPos = Name.find_insensitive(Str: NamePrefix);
1316 if (PrefixPos == StringRef::npos)
1317 continue;
1318 PrefixPos += NamePrefix.size();
1319 break;
1320 }
1321
1322 if (PrefixPos == StringRef::npos) {
1323 return;
1324 }
1325
1326 StringRef NewName = Name.drop_front(N: PrefixPos);
1327 StringRef FName = Name.substr(Start: 0, N: PrefixPos - 1);
1328 if (NewName.size() == 0) {
1329 return;
1330 }
1331
1332 // This name should have a static linkage.
1333 size_t PostfixPos = NewName.find(Str: FunctionSamples::UniqSuffix);
1334 bool ProfileHasFUnique = (PostfixPos != StringRef::npos);
1335
1336 // If sample profile and instrumented profile do not agree on symbol
1337 // uniqification.
1338 if (SampleProfileHasFUnique != ProfileHasFUnique) {
1339 // If instrumented profile uses -funique-internal-linkage-symbols,
1340 // we need to trim the name.
1341 if (ProfileHasFUnique) {
1342 NewName = NewName.substr(Start: 0, N: PostfixPos);
1343 } else {
1344 // If sample profile uses -funique-internal-linkage-symbols,
1345 // we build the map.
1346 std::string NStr =
1347 NewName.str() + getUniqueInternalLinkagePostfix(FName);
1348 NewName = StringRef(NStr);
1349 StaticFuncMap[NewName] = Name;
1350 return;
1351 }
1352 }
1353
1354 auto [It, Inserted] = StaticFuncMap.try_emplace(Key: NewName, Args: Name);
1355 if (!Inserted)
1356 It->second = DuplicateNameStr;
1357 };
1358
1359 // We need to flatten the SampleFDO profile as the InstrFDO
1360 // profile does not have inlined callsite profiles.
1361 // One caveat is the pre-inlined function -- their samples
1362 // should be collapsed into the caller function.
1363 // Here we do a DFS traversal to get the flatten profile
1364 // info: the sum of entrycount and the max of maxcount.
1365 // Here is the algorithm:
1366 // recursive (FS, root_name) {
1367 // name = FS->getName();
1368 // get samples for FS;
1369 // if (InstrProf.find(name) {
1370 // root_name = name;
1371 // } else {
1372 // if (name is in static_func map) {
1373 // root_name = static_name;
1374 // }
1375 // }
1376 // update the Map entry for root_name;
1377 // for (subfs: FS) {
1378 // recursive(subfs, root_name);
1379 // }
1380 // }
1381 //
1382 // Here is an example.
1383 //
1384 // SampleProfile:
1385 // foo:12345:1000
1386 // 1: 1000
1387 // 2.1: 1000
1388 // 15: 5000
1389 // 4: bar:1000
1390 // 1: 1000
1391 // 2: goo:3000
1392 // 1: 3000
1393 // 8: bar:40000
1394 // 1: 10000
1395 // 2: goo:30000
1396 // 1: 30000
1397 //
1398 // InstrProfile has two entries:
1399 // foo
1400 // bar.cc;bar
1401 //
1402 // After BuildMaxSampleMap, we should have the following in FlattenSampleMap:
1403 // {"foo", {1000, 5000}}
1404 // {"bar.cc;bar", {11000, 30000}}
1405 //
1406 // foo's has an entry count of 1000, and max body count of 5000.
1407 // bar.cc;bar has an entry count of 11000 (sum two callsites of 1000 and
1408 // 10000), and max count of 30000 (from the callsite in line 8).
1409 //
1410 // Note that goo's count will remain in bar.cc;bar() as it does not have an
1411 // entry in InstrProfile.
1412 llvm::StringMap<std::pair<uint64_t, uint64_t>> FlattenSampleMap;
1413 auto BuildMaxSampleMap = [&FlattenSampleMap, &StaticFuncMap,
1414 &InstrProfileMap](const FunctionSamples &FS,
1415 const StringRef &RootName) {
1416 auto BuildMaxSampleMapImpl = [&](const FunctionSamples &FS,
1417 const StringRef &RootName,
1418 auto &BuildImpl) -> void {
1419 std::string NameStr = FS.getFunction().str();
1420 const StringRef Name = NameStr;
1421 const StringRef *NewRootName = &RootName;
1422 uint64_t EntrySample = FS.getHeadSamplesEstimate();
1423 uint64_t MaxBodySample = FS.getMaxCountInside(/* SkipCallSite*/ true);
1424
1425 auto It = InstrProfileMap.find(Key: Name);
1426 if (It != InstrProfileMap.end()) {
1427 NewRootName = &Name;
1428 } else {
1429 auto NewName = StaticFuncMap.find(Key: Name);
1430 if (NewName != StaticFuncMap.end()) {
1431 It = InstrProfileMap.find(Key: NewName->second);
1432 if (NewName->second != DuplicateNameStr) {
1433 NewRootName = &NewName->second;
1434 }
1435 } else {
1436 // Here the EntrySample is of an inlined function, so we should not
1437 // update the EntrySample in the map.
1438 EntrySample = 0;
1439 }
1440 }
1441 EntrySample += FlattenSampleMap[*NewRootName].first;
1442 MaxBodySample =
1443 std::max(a: FlattenSampleMap[*NewRootName].second, b: MaxBodySample);
1444 FlattenSampleMap[*NewRootName] =
1445 std::make_pair(x&: EntrySample, y&: MaxBodySample);
1446
1447 for (const auto &C : FS.getCallsiteSamples())
1448 for (const auto &F : C.second)
1449 BuildImpl(F.second, *NewRootName, BuildImpl);
1450 };
1451 BuildMaxSampleMapImpl(FS, RootName, BuildMaxSampleMapImpl);
1452 };
1453
1454 for (auto &PD : WC->Writer.getProfileData()) {
1455 // Populate IPBuilder.
1456 for (const auto &PDV : PD.getValue()) {
1457 InstrProfRecord Record = PDV.second;
1458 IPBuilder.addRecord(Record);
1459 }
1460
1461 // If a function has multiple entries in instr profile, skip it.
1462 if (PD.getValue().size() != 1)
1463 continue;
1464
1465 // Initialize InstrProfileMap.
1466 InstrProfRecord *R = &PD.getValue().begin()->second;
1467 StringRef FullName = PD.getKey();
1468 InstrProfileMap[FullName] = InstrProfileEntry(R);
1469 buildStaticFuncMap(FullName);
1470 }
1471
1472 for (auto &PD : Reader->getProfiles()) {
1473 sampleprof::FunctionSamples &FS = PD.second;
1474 std::string Name = FS.getFunction().str();
1475 BuildMaxSampleMap(FS, Name);
1476 }
1477
1478 ProfileSummary InstrPS = *IPBuilder.getSummary();
1479 ProfileSummary SamplePS = Reader->getSummary();
1480
1481 // Compute cold thresholds for instr profile and sample profile.
1482 uint64_t HotSampleThreshold =
1483 ProfileSummaryBuilder::getEntryForPercentile(
1484 DS: SamplePS.getDetailedSummary(),
1485 Percentile: ProfileSummaryBuilder::DefaultCutoffs[HotPercentileIdx])
1486 .MinCount;
1487 uint64_t ColdSampleThreshold =
1488 ProfileSummaryBuilder::getEntryForPercentile(
1489 DS: SamplePS.getDetailedSummary(),
1490 Percentile: ProfileSummaryBuilder::DefaultCutoffs[ColdPercentileIdx])
1491 .MinCount;
1492 uint64_t HotInstrThreshold =
1493 ProfileSummaryBuilder::getEntryForPercentile(
1494 DS: InstrPS.getDetailedSummary(),
1495 Percentile: ProfileSummaryBuilder::DefaultCutoffs[HotPercentileIdx])
1496 .MinCount;
1497 uint64_t ColdInstrThreshold =
1498 InstrProfColdThreshold
1499 ? InstrProfColdThreshold
1500 : ProfileSummaryBuilder::getEntryForPercentile(
1501 DS: InstrPS.getDetailedSummary(),
1502 Percentile: ProfileSummaryBuilder::DefaultCutoffs[ColdPercentileIdx])
1503 .MinCount;
1504
1505 // Find hot/warm functions in sample profile which is cold in instr profile
1506 // and adjust the profiles of those functions in the instr profile.
1507 for (const auto &E : FlattenSampleMap) {
1508 uint64_t SampleMaxCount = std::max(a: E.second.first, b: E.second.second);
1509 if (SampleMaxCount < ColdSampleThreshold)
1510 continue;
1511 StringRef Name = E.first();
1512 auto It = InstrProfileMap.find(Key: Name);
1513 if (It == InstrProfileMap.end()) {
1514 auto NewName = StaticFuncMap.find(Key: Name);
1515 if (NewName != StaticFuncMap.end()) {
1516 It = InstrProfileMap.find(Key: NewName->second);
1517 if (NewName->second == DuplicateNameStr) {
1518 WithColor::warning()
1519 << "Static function " << Name
1520 << " has multiple promoted names, cannot adjust profile.\n";
1521 }
1522 }
1523 }
1524 if (It == InstrProfileMap.end() ||
1525 It->second.MaxCount > ColdInstrThreshold ||
1526 It->second.NumEdgeCounters < SupplMinSizeThreshold)
1527 continue;
1528 bool SetToHot = SampleMaxCount >= HotSampleThreshold;
1529 updateInstrProfileEntry(IFE&: It->second, SetToHot, HotInstrThreshold,
1530 ColdInstrThreshold, ZeroCounterThreshold);
1531 }
1532}
1533
1534/// The main function to supplement instr profile with sample profile.
1535/// \Inputs contains the instr profile. \p SampleFilename specifies the
1536/// sample profile. \p OutputFilename specifies the output profile name.
1537/// \p OutputFormat specifies the output profile format. \p OutputSparse
1538/// specifies whether to generate sparse profile. \p SupplMinSizeThreshold
1539/// specifies the minimal size for the functions whose profile will be
1540/// adjusted. \p ZeroCounterThreshold is the threshold to check whether
1541/// a function contains too many zero counters and whether its profile
1542/// should be dropped. \p InstrProfColdThreshold is the user specified
1543/// cold threshold which will override the cold threshold got from the
1544/// instr profile summary.
1545static Error supplementInstrProfile(const WeightedFileVector &Inputs,
1546 StringRef SampleFilename, bool OutputSparse,
1547 unsigned SupplMinSizeThreshold,
1548 float ZeroCounterThreshold,
1549 unsigned InstrProfColdThreshold) {
1550 if (OutputFilename == "-")
1551 return makeError(Message: "cannot write indexed profdata format to stdout");
1552 if (Inputs.size() != 1)
1553 return makeError(Message: "expect one input to be an instr profile");
1554 if (Inputs[0].Weight != 1)
1555 return makeError(Message: "expect instr profile doesn't have weight");
1556
1557 StringRef InstrFilename = Inputs[0].Filename;
1558
1559 // Read sample profile.
1560 LLVMContext Context;
1561 auto FS = vfs::getRealFileSystem();
1562 auto ReaderOrErr = sampleprof::SampleProfileReader::create(
1563 Filename: SampleFilename.str(), C&: Context, FS&: *FS, P: FSDiscriminatorPassOption);
1564 if (std::error_code EC = ReaderOrErr.getError())
1565 return makeError(EC, Whence: SampleFilename);
1566 auto Reader = std::move(ReaderOrErr.get());
1567 if (std::error_code EC = Reader->read())
1568 return makeError(EC, Whence: SampleFilename);
1569
1570 // Read instr profile.
1571 std::mutex ErrorLock;
1572 SmallSet<instrprof_error, 4> WriterErrorCodes;
1573 auto WC = std::make_unique<WriterContext>(args&: OutputSparse, args&: ErrorLock,
1574 args&: WriterErrorCodes);
1575 if (Error E = loadInput(Input: Inputs[0], Remapper: nullptr, Correlator: nullptr, /*ProfiledBinary=*/"",
1576 WC: WC.get()))
1577 return E;
1578 if (!WC->Errors.empty())
1579 return makeError(E: std::move(WC->Errors[0].first), Whence: InstrFilename);
1580
1581 adjustInstrProfile(WC, Reader, SupplMinSizeThreshold, ZeroCounterThreshold,
1582 InstrProfColdThreshold);
1583 return writeInstrProfile(OutputFilename, OutputFormat, Writer&: WC->Writer);
1584}
1585
1586/// Make a copy of the given function samples with all symbol names remapped
1587/// by the provided symbol remapper.
1588static sampleprof::FunctionSamples
1589remapSamples(const sampleprof::FunctionSamples &Samples,
1590 SymbolRemapper &Remapper, sampleprof_error &Error) {
1591 sampleprof::FunctionSamples Result;
1592 Result.setFunction(Remapper(Samples.getFunction()));
1593 Result.addTotalSamples(Num: Samples.getTotalSamples());
1594 Result.addHeadSamples(Num: Samples.getHeadSamples());
1595 Result.reserveBodySamples(NumEntries: Samples.getBodySamples().size());
1596 for (const auto &BodySample : Samples.getBodySamples()) {
1597 uint32_t MaskedDiscriminator =
1598 BodySample.first.Discriminator & getDiscriminatorMask();
1599 Result.addBodySamples(LineOffset: BodySample.first.LineOffset, Discriminator: MaskedDiscriminator,
1600 Num: BodySample.second.getSamples());
1601 for (const auto &Target : BodySample.second.getCallTargets()) {
1602 Result.addCalledTargetSamples(LineOffset: BodySample.first.LineOffset,
1603 Discriminator: MaskedDiscriminator,
1604 Func: Remapper(Target.first), Num: Target.second);
1605 }
1606 }
1607 for (const auto &CallsiteSamples : Samples.getCallsiteSamples()) {
1608 sampleprof::FunctionSamplesMap &Target =
1609 Result.functionSamplesAt(Loc: CallsiteSamples.first);
1610 for (const auto &Callsite : CallsiteSamples.second) {
1611 sampleprof::FunctionSamples Remapped =
1612 remapSamples(Samples: Callsite.second, Remapper, Error);
1613 mergeSampleProfErrors(Accumulator&: Error,
1614 Result: Target[Remapped.getFunction()].merge(Other: Remapped));
1615 }
1616 }
1617 return Result;
1618}
1619
1620static sampleprof::SampleProfileFormat FormatMap[] = {
1621 sampleprof::SPF_None,
1622 sampleprof::SPF_Text,
1623 sampleprof::SPF_None,
1624 sampleprof::SPF_Ext_Binary,
1625 sampleprof::SPF_GCC,
1626 sampleprof::SPF_Binary};
1627
1628static Expected<std::unique_ptr<MemoryBuffer>>
1629getInputFileBuf(const StringRef &InputFile) {
1630 if (InputFile == "")
1631 return {nullptr};
1632
1633 auto BufOrError = MemoryBuffer::getFileOrSTDIN(Filename: InputFile);
1634 if (!BufOrError)
1635 return makeError(EC: BufOrError.getError(), Whence: InputFile);
1636
1637 return std::move(*BufOrError);
1638}
1639
1640static void populateProfileSymbolList(MemoryBuffer *Buffer,
1641 sampleprof::ProfileSymbolList &PSL) {
1642 if (!Buffer)
1643 return;
1644
1645 SmallVector<StringRef, 32> SymbolVec;
1646 StringRef Data = Buffer->getBuffer();
1647 Data.split(A&: SymbolVec, Separator: '\n', /*MaxSplit=*/-1, /*KeepEmpty=*/false);
1648
1649 for (StringRef SymbolStr : SymbolVec)
1650 PSL.add(Name: SymbolStr.trim());
1651}
1652
1653static void handleExtBinaryWriter(sampleprof::SampleProfileWriter &Writer,
1654 ProfileFormat OutputFormat,
1655 MemoryBuffer *Buffer,
1656 sampleprof::ProfileSymbolList &WriterList,
1657 bool CompressAllSections, bool UseMD5,
1658 bool GenPartialProfile) {
1659 if (SplitLayout) {
1660 if (OutputFormat == PF_Binary)
1661 warn(Message: "-split-layout is ignored. Specify -extbinary to enable it");
1662 else
1663 Writer.setUseCtxSplitLayout();
1664 }
1665
1666 populateProfileSymbolList(Buffer, PSL&: WriterList);
1667 if (WriterList.size() > 0 && OutputFormat != PF_Ext_Binary)
1668 warn(Message: "Profile Symbol list is not empty but the output format is not "
1669 "ExtBinary format. The list will be lost in the output. ");
1670
1671 Writer.setProfileSymbolList(&WriterList);
1672
1673 if (CompressAllSections) {
1674 if (OutputFormat != PF_Ext_Binary)
1675 warn(Message: "-compress-all-section is ignored. Specify -extbinary to enable it");
1676 else
1677 Writer.setToCompressAllSections();
1678 }
1679 if (UseMD5) {
1680 if (OutputFormat != PF_Ext_Binary)
1681 warn(Message: "-use-md5 is ignored. Specify -extbinary to enable it");
1682 else
1683 Writer.setUseMD5();
1684 }
1685 if (GenPartialProfile) {
1686 if (OutputFormat != PF_Ext_Binary)
1687 warn(Message: "-gen-partial-profile is ignored. Specify -extbinary to enable it");
1688 else
1689 Writer.setPartialProfile();
1690 }
1691 if (WriteMD5ProfSymList) {
1692 if (OutputFormat != PF_Ext_Binary)
1693 warn(Message: "-md5-prof-sym-list is ignored. Specify -extbinary to enable it");
1694 else
1695 Writer.setUseMD5ProfileSymbolList();
1696 }
1697 if (WriteMD5IndexedTables) {
1698 if (OutputFormat != PF_Ext_Binary)
1699 warn(Message: "-md5-indexed-tables is ignored. Specify -extbinary to enable it");
1700 else
1701 Writer.setUseMD5IndexedTables();
1702 }
1703}
1704
1705static Error mergeSampleProfile(const WeightedFileVector &Inputs,
1706 SymbolRemapper *Remapper,
1707 StringRef ProfileSymbolListFile,
1708 size_t OutputSizeLimit) {
1709 using namespace sampleprof;
1710 SampleProfileMap ProfileMap;
1711 SmallVector<std::unique_ptr<sampleprof::SampleProfileReader>, 5> Readers;
1712 LLVMContext Context;
1713 sampleprof::ProfileSymbolList WriterList;
1714 std::optional<bool> ProfileIsProbeBased;
1715 std::optional<bool> ProfileIsCS;
1716 for (const auto &Input : Inputs) {
1717 auto FS = vfs::getRealFileSystem();
1718 auto ReaderOrErr = SampleProfileReader::create(Filename: Input.Filename, C&: Context, FS&: *FS,
1719 P: FSDiscriminatorPassOption);
1720 if (std::error_code EC = ReaderOrErr.getError()) {
1721 if (Error E = warnOrErrorGivenError(FailMode, EC, Whence: Input.Filename))
1722 return E;
1723 continue;
1724 }
1725
1726 // We need to keep the readers around until after all the files are
1727 // read so that we do not lose the function names stored in each
1728 // reader's memory. The function names are needed to write out the
1729 // merged profile map.
1730 Readers.push_back(Elt: std::move(ReaderOrErr.get()));
1731 const auto Reader = Readers.back().get();
1732 if (std::error_code EC = Reader->read()) {
1733 if (Error E = warnOrErrorGivenError(FailMode, EC, Whence: Input.Filename))
1734 return E;
1735 Readers.pop_back();
1736 continue;
1737 }
1738
1739 // Merging cannot preserve payloads that this reader does not understand,
1740 // so make the otherwise intentional forward-compatible skip visible.
1741 if (Reader->hasUnknownProfileTypes())
1742 warn(Message: "unknown composite profile blocks were ignored and will not be "
1743 "preserved",
1744 Whence: Input.Filename);
1745
1746 SampleProfileMap &Profiles = Reader->getProfiles();
1747 if (ProfileIsProbeBased &&
1748 ProfileIsProbeBased != FunctionSamples::ProfileIsProbeBased)
1749 return makeError(
1750 Message: "cannot merge probe-based profile with non-probe-based profile");
1751 ProfileIsProbeBased = FunctionSamples::ProfileIsProbeBased;
1752 if (ProfileIsCS && ProfileIsCS != FunctionSamples::ProfileIsCS)
1753 return makeError(Message: "cannot merge CS profile with non-CS profile");
1754 ProfileIsCS = FunctionSamples::ProfileIsCS;
1755 for (SampleProfileMap::iterator I = Profiles.begin(), E = Profiles.end();
1756 I != E; ++I) {
1757 sampleprof_error Result = sampleprof_error::success;
1758 FunctionSamples Remapped =
1759 Remapper ? remapSamples(Samples: I->second, Remapper&: *Remapper, Error&: Result)
1760 : FunctionSamples();
1761 FunctionSamples &Samples = Remapper ? Remapped : I->second;
1762 SampleContext FContext = Samples.getContext();
1763 mergeSampleProfErrors(Accumulator&: Result,
1764 Result: ProfileMap[FContext].merge(Other: Samples, Weight: Input.Weight));
1765 if (Result != sampleprof_error::success) {
1766 std::error_code EC = make_error_code(E: Result);
1767 handleMergeWriterError(E: errorCodeToError(EC), WhenceFile: Input.Filename,
1768 WhenceFunction: FContext.toString());
1769 }
1770 }
1771
1772 if (!DropProfileSymbolList) {
1773 std::unique_ptr<sampleprof::ProfileSymbolList> ReaderList =
1774 Reader->getProfileSymbolList();
1775 if (ReaderList)
1776 WriterList.merge(List: *ReaderList);
1777 }
1778 }
1779
1780 if (ProfileIsCS && (SampleMergeColdContext || SampleTrimColdContext)) {
1781 // Use threshold calculated from profile summary unless specified.
1782 SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs);
1783 auto Summary = Builder.computeSummaryForProfiles(Profiles: ProfileMap);
1784 uint64_t SampleProfColdThreshold =
1785 ProfileSummaryBuilder::getColdCountThreshold(
1786 DS: (Summary->getDetailedSummary()));
1787
1788 // Trim and merge cold context profile using cold threshold above;
1789 SampleContextTrimmer(ProfileMap)
1790 .trimAndMergeColdContextProfiles(
1791 ColdCountThreshold: SampleProfColdThreshold, TrimColdContext: SampleTrimColdContext,
1792 MergeColdContext: SampleMergeColdContext, ColdContextFrameLength: SampleColdContextFrameDepth, TrimBaseProfileOnly: false);
1793 }
1794
1795 if (ProfileLayout == llvm::sampleprof::SPL_Flat) {
1796 ProfileConverter::flattenProfile(ProfileMap, ProfileIsCS: FunctionSamples::ProfileIsCS);
1797 ProfileIsCS = FunctionSamples::ProfileIsCS = false;
1798 } else if (ProfileIsCS && ProfileLayout == llvm::sampleprof::SPL_Nest) {
1799 ProfileConverter CSConverter(ProfileMap);
1800 CSConverter.convertCSProfiles();
1801 ProfileIsCS = FunctionSamples::ProfileIsCS = false;
1802 }
1803
1804 if (Error E = filterFunctions(ProfileMap))
1805 return E;
1806
1807 auto WriterOrErr =
1808 SampleProfileWriter::create(Filename: OutputFilename, Format: FormatMap[OutputFormat]);
1809 if (std::error_code EC = WriterOrErr.getError())
1810 return makeError(EC, Whence: OutputFilename);
1811
1812 auto Writer = std::move(WriterOrErr.get());
1813 // WriterList will have StringRef refering to string in Buffer.
1814 // Make sure Buffer lives as long as WriterList.
1815 auto BufferOrErr = getInputFileBuf(InputFile: ProfileSymbolListFile);
1816 if (!BufferOrErr)
1817 return BufferOrErr.takeError();
1818 auto Buffer = std::move(*BufferOrErr);
1819 handleExtBinaryWriter(Writer&: *Writer, OutputFormat, Buffer: Buffer.get(), WriterList,
1820 CompressAllSections, UseMD5, GenPartialProfile);
1821
1822 // If OutputSizeLimit is 0 (default), it is the same as write().
1823 if (std::error_code EC =
1824 Writer->writeWithSizeLimit(ProfileMap, OutputSizeLimit))
1825 return makeError(EC);
1826
1827 return Error::success();
1828}
1829
1830static Expected<WeightedFile>
1831parseWeightedFile(const StringRef &WeightedFilename) {
1832 StringRef WeightStr, FileName;
1833 std::tie(args&: WeightStr, args&: FileName) = WeightedFilename.split(Separator: ',');
1834
1835 uint64_t Weight;
1836 if (WeightStr.getAsInteger(Radix: 10, Result&: Weight) || Weight < 1)
1837 return makeError(Message: "input weight must be a positive integer");
1838
1839 llvm::SmallString<128> ResolvedFileName;
1840 llvm::sys::fs::expand_tilde(path: FileName, output&: ResolvedFileName);
1841
1842 return WeightedFile{.Filename: std::string(ResolvedFileName), .Weight: Weight};
1843}
1844
1845static Error addWeightedInput(WeightedFileVector &WNI, const WeightedFile &WF) {
1846 StringRef Filename = WF.Filename;
1847 uint64_t Weight = WF.Weight;
1848
1849 // If it's STDIN just pass it on.
1850 if (Filename == "-") {
1851 WNI.push_back(Elt: {.Filename: std::string(Filename), .Weight: Weight});
1852 return Error::success();
1853 }
1854
1855 llvm::sys::fs::file_status Status;
1856 llvm::sys::fs::status(path: Filename, result&: Status);
1857 if (!llvm::sys::fs::exists(status: Status))
1858 return makeError(EC: make_error_code(E: errc::no_such_file_or_directory),
1859 Whence: Filename);
1860 // If it's a source file, collect it.
1861 if (llvm::sys::fs::is_regular_file(status: Status)) {
1862 WNI.push_back(Elt: {.Filename: std::string(Filename), .Weight: Weight});
1863 return Error::success();
1864 }
1865
1866 if (llvm::sys::fs::is_directory(status: Status)) {
1867 std::error_code EC;
1868 for (llvm::sys::fs::recursive_directory_iterator F(Filename, EC), E;
1869 F != E && !EC; F.increment(ec&: EC)) {
1870 if (llvm::sys::fs::is_regular_file(Path: F->path())) {
1871 if (Error E = addWeightedInput(WNI, WF: {.Filename: F->path(), .Weight: Weight}))
1872 return E;
1873 }
1874 }
1875 if (EC)
1876 return makeError(EC, Whence: Filename);
1877 }
1878 return Error::success();
1879}
1880
1881static Error parseInputFilenamesFile(MemoryBuffer *Buffer,
1882 WeightedFileVector &WFV) {
1883 if (!Buffer)
1884 return Error::success();
1885
1886 SmallVector<StringRef, 8> Entries;
1887 StringRef Data = Buffer->getBuffer();
1888 Data.split(A&: Entries, Separator: '\n', /*MaxSplit=*/-1, /*KeepEmpty=*/false);
1889 for (const StringRef &FileWeightEntry : Entries) {
1890 StringRef SanitizedEntry = FileWeightEntry.trim(Chars: " \t\v\f\r");
1891 // Skip comments.
1892 if (SanitizedEntry.starts_with(Prefix: "#"))
1893 continue;
1894 // If there's no comma, it's an unweighted profile.
1895 else if (!SanitizedEntry.contains(C: ',')) {
1896 if (Error E = addWeightedInput(WNI&: WFV, WF: {.Filename: std::string(SanitizedEntry), .Weight: 1}))
1897 return E;
1898 } else {
1899 auto WFOrErr = parseWeightedFile(WeightedFilename: SanitizedEntry);
1900 if (!WFOrErr)
1901 return WFOrErr.takeError();
1902 if (Error E = addWeightedInput(WNI&: WFV, WF: *WFOrErr))
1903 return E;
1904 }
1905 }
1906 return Error::success();
1907}
1908
1909static Error merge_main(StringRef ProgName) {
1910 WeightedFileVector WeightedInputs;
1911 for (StringRef Filename : InputFilenames)
1912 if (Error E = addWeightedInput(WNI&: WeightedInputs, WF: {.Filename: std::string(Filename), .Weight: 1}))
1913 return E;
1914 for (StringRef WeightedFilename : WeightedInputFilenames) {
1915 auto WFOrErr = parseWeightedFile(WeightedFilename);
1916 if (!WFOrErr)
1917 return WFOrErr.takeError();
1918 if (Error E = addWeightedInput(WNI&: WeightedInputs, WF: *WFOrErr))
1919 return E;
1920 }
1921
1922 // Make sure that the file buffer stays alive for the duration of the
1923 // weighted input vector's lifetime.
1924 auto BufferOrErr = getInputFileBuf(InputFile: InputFilenamesFile);
1925 if (!BufferOrErr)
1926 return BufferOrErr.takeError();
1927 auto Buffer = std::move(*BufferOrErr);
1928 if (Error E = parseInputFilenamesFile(Buffer: Buffer.get(), WFV&: WeightedInputs))
1929 return E;
1930
1931 if (WeightedInputs.empty())
1932 return makeError(Message: "no input files specified. See " + ProgName +
1933 " merge -help");
1934
1935 if (DumpInputFileList) {
1936 for (auto &WF : WeightedInputs)
1937 outs() << WF.Weight << "," << WF.Filename << "\n";
1938 return Error::success();
1939 }
1940
1941 std::unique_ptr<SymbolRemapper> Remapper;
1942 if (!RemappingFile.empty()) {
1943 auto RemapperOrErr = SymbolRemapper::create(InputFile: RemappingFile);
1944 if (!RemapperOrErr)
1945 return RemapperOrErr.takeError();
1946 Remapper = std::move(*RemapperOrErr);
1947 }
1948
1949 if (!SupplInstrWithSample.empty()) {
1950 if (ProfileKind != instr)
1951 return makeError(
1952 Message: "-supplement-instr-with-sample can only work with -instr. ");
1953
1954 return supplementInstrProfile(Inputs: WeightedInputs, SampleFilename: SupplInstrWithSample,
1955 OutputSparse, SupplMinSizeThreshold,
1956 ZeroCounterThreshold, InstrProfColdThreshold);
1957 }
1958
1959 if (ProfileKind == instr)
1960 return mergeInstrProfile(Inputs: WeightedInputs, Remapper: Remapper.get(),
1961 MaxDbgCorrelationWarnings, ProfiledBinary);
1962
1963 return mergeSampleProfile(Inputs: WeightedInputs, Remapper: Remapper.get(),
1964 ProfileSymbolListFile, OutputSizeLimit);
1965}
1966
1967/// Computer the overlap b/w profile BaseFilename and profile TestFilename.
1968static Error overlapInstrProfile(const std::string &BaseFilename,
1969 const std::string &TestFilename,
1970 const OverlapFuncFilters &FuncFilter,
1971 raw_fd_ostream &OS, bool IsCS) {
1972 std::mutex ErrorLock;
1973 SmallSet<instrprof_error, 4> WriterErrorCodes;
1974 WriterContext Context(false, ErrorLock, WriterErrorCodes);
1975 WeightedFile WeightedInput{.Filename: BaseFilename, .Weight: 1};
1976 OverlapStats Overlap;
1977 Error E = Overlap.accumulateCounts(BaseFilename, TestFilename, IsCS);
1978 if (E)
1979 return makeError(E: std::move(E), Whence: "error in getting profile count sums");
1980 if (Overlap.Base.CountSum < 1.0f) {
1981 OS << "Sum of edge counts for profile " << BaseFilename << " is 0.\n";
1982 return Error::success();
1983 }
1984 if (Overlap.Test.CountSum < 1.0f) {
1985 OS << "Sum of edge counts for profile " << TestFilename << " is 0.\n";
1986 return Error::success();
1987 }
1988 if (Error E = loadInput(Input: WeightedInput, Remapper: nullptr, Correlator: nullptr,
1989 /*ProfiledBinary=*/"", WC: &Context))
1990 return E;
1991 overlapInput(BaseFilename, TestFilename, WC: &Context, Overlap, FuncFilter, OS,
1992 IsCS);
1993 Overlap.dump(OS);
1994 return Error::success();
1995}
1996
1997namespace {
1998struct SampleOverlapStats {
1999 SampleContext BaseName;
2000 SampleContext TestName;
2001 // Number of overlap units
2002 uint64_t OverlapCount = 0;
2003 // Total samples of overlap units
2004 uint64_t OverlapSample = 0;
2005 // Number of and total samples of units that only present in base or test
2006 // profile
2007 uint64_t BaseUniqueCount = 0;
2008 uint64_t BaseUniqueSample = 0;
2009 uint64_t TestUniqueCount = 0;
2010 uint64_t TestUniqueSample = 0;
2011 // Number of units and total samples in base or test profile
2012 uint64_t BaseCount = 0;
2013 uint64_t BaseSample = 0;
2014 uint64_t TestCount = 0;
2015 uint64_t TestSample = 0;
2016 // Number of and total samples of units that present in at least one profile
2017 uint64_t UnionCount = 0;
2018 uint64_t UnionSample = 0;
2019 // Weighted similarity
2020 double Similarity = 0.0;
2021 // For SampleOverlapStats instances representing functions, weights of the
2022 // function in base and test profiles
2023 double BaseWeight = 0.0;
2024 double TestWeight = 0.0;
2025
2026 SampleOverlapStats() = default;
2027};
2028} // end anonymous namespace
2029
2030namespace {
2031struct FuncSampleStats {
2032 uint64_t SampleSum = 0;
2033 uint64_t MaxSample = 0;
2034 uint64_t HotBlockCount = 0;
2035 FuncSampleStats() = default;
2036 FuncSampleStats(uint64_t SampleSum, uint64_t MaxSample,
2037 uint64_t HotBlockCount)
2038 : SampleSum(SampleSum), MaxSample(MaxSample),
2039 HotBlockCount(HotBlockCount) {}
2040};
2041} // end anonymous namespace
2042
2043namespace {
2044enum MatchStatus { MS_Match, MS_FirstUnique, MS_SecondUnique, MS_None };
2045
2046// Class for updating merging steps for two sorted maps. The class should be
2047// instantiated with a map iterator type.
2048template <class T> class MatchStep {
2049public:
2050 MatchStep() = delete;
2051
2052 MatchStep(T FirstIter, T FirstEnd, T SecondIter, T SecondEnd)
2053 : FirstIter(FirstIter), FirstEnd(FirstEnd), SecondIter(SecondIter),
2054 SecondEnd(SecondEnd), Status(MS_None) {}
2055
2056 bool areBothFinished() const {
2057 return (FirstIter == FirstEnd && SecondIter == SecondEnd);
2058 }
2059
2060 bool isFirstFinished() const { return FirstIter == FirstEnd; }
2061
2062 bool isSecondFinished() const { return SecondIter == SecondEnd; }
2063
2064 /// Advance one step based on the previous match status unless the previous
2065 /// status is MS_None. Then update Status based on the comparison between two
2066 /// container iterators at the current step. If the previous status is
2067 /// MS_None, it means two iterators are at the beginning and no comparison has
2068 /// been made, so we simply update Status without advancing the iterators.
2069 void updateOneStep();
2070
2071 T getFirstIter() const { return FirstIter; }
2072
2073 T getSecondIter() const { return SecondIter; }
2074
2075 MatchStatus getMatchStatus() const { return Status; }
2076
2077private:
2078 // Current iterator and end iterator of the first container.
2079 T FirstIter;
2080 T FirstEnd;
2081 // Current iterator and end iterator of the second container.
2082 T SecondIter;
2083 T SecondEnd;
2084 // Match status of the current step.
2085 MatchStatus Status;
2086};
2087} // end anonymous namespace
2088
2089template <class T> void MatchStep<T>::updateOneStep() {
2090 switch (Status) {
2091 case MS_Match:
2092 ++FirstIter;
2093 ++SecondIter;
2094 break;
2095 case MS_FirstUnique:
2096 ++FirstIter;
2097 break;
2098 case MS_SecondUnique:
2099 ++SecondIter;
2100 break;
2101 case MS_None:
2102 break;
2103 }
2104
2105 // Update Status according to iterators at the current step.
2106 if (areBothFinished())
2107 return;
2108 if (FirstIter != FirstEnd &&
2109 (SecondIter == SecondEnd || FirstIter->first < SecondIter->first))
2110 Status = MS_FirstUnique;
2111 else if (SecondIter != SecondEnd &&
2112 (FirstIter == FirstEnd || SecondIter->first < FirstIter->first))
2113 Status = MS_SecondUnique;
2114 else
2115 Status = MS_Match;
2116}
2117
2118// Return the sum of line/block samples, the max line/block sample, and the
2119// number of line/block samples above the given threshold in a function
2120// including its inlinees.
2121static void getFuncSampleStats(const sampleprof::FunctionSamples &Func,
2122 FuncSampleStats &FuncStats,
2123 uint64_t HotThreshold) {
2124 for (const auto &L : Func.getBodySamples()) {
2125 uint64_t Sample = L.second.getSamples();
2126 FuncStats.SampleSum += Sample;
2127 FuncStats.MaxSample = std::max(a: FuncStats.MaxSample, b: Sample);
2128 if (Sample >= HotThreshold)
2129 ++FuncStats.HotBlockCount;
2130 }
2131
2132 for (const auto &C : Func.getCallsiteSamples()) {
2133 for (const auto &F : C.second)
2134 getFuncSampleStats(Func: F.second, FuncStats, HotThreshold);
2135 }
2136}
2137
2138/// Predicate that determines if a function is hot with a given threshold. We
2139/// keep it separate from its callsites for possible extension in the future.
2140static bool isFunctionHot(const FuncSampleStats &FuncStats,
2141 uint64_t HotThreshold) {
2142 // We intentionally compare the maximum sample count in a function with the
2143 // HotThreshold to get an approximate determination on hot functions.
2144 return (FuncStats.MaxSample >= HotThreshold);
2145}
2146
2147namespace {
2148class SampleOverlapAggregator {
2149public:
2150 SampleOverlapAggregator(const std::string &BaseFilename,
2151 const std::string &TestFilename,
2152 double LowSimilarityThreshold, double Epsilon,
2153 const OverlapFuncFilters &FuncFilter)
2154 : BaseFilename(BaseFilename), TestFilename(TestFilename),
2155 LowSimilarityThreshold(LowSimilarityThreshold), Epsilon(Epsilon),
2156 FuncFilter(FuncFilter) {}
2157
2158 /// Detect 0-sample input profile and report to output stream. This interface
2159 /// should be called after loadProfiles().
2160 bool detectZeroSampleProfile(raw_fd_ostream &OS) const;
2161
2162 /// Write out function-level similarity statistics for functions specified by
2163 /// options --function, --value-cutoff, and --similarity-cutoff.
2164 void dumpFuncSimilarity(raw_fd_ostream &OS) const;
2165
2166 /// Write out program-level similarity and overlap statistics.
2167 void dumpProgramSummary(raw_fd_ostream &OS) const;
2168
2169 /// Write out hot-function and hot-block statistics for base_profile,
2170 /// test_profile, and their overlap. For both cases, the overlap HO is
2171 /// calculated as follows:
2172 /// Given the number of functions (or blocks) that are hot in both profiles
2173 /// HCommon and the number of functions (or blocks) that are hot in at
2174 /// least one profile HUnion, HO = HCommon / HUnion.
2175 void dumpHotFuncAndBlockOverlap(raw_fd_ostream &OS) const;
2176
2177 /// This function tries matching functions in base and test profiles. For each
2178 /// pair of matched functions, it aggregates the function-level
2179 /// similarity into a profile-level similarity. It also dump function-level
2180 /// similarity information of functions specified by --function,
2181 /// --value-cutoff, and --similarity-cutoff options. The program-level
2182 /// similarity PS is computed as follows:
2183 /// Given function-level similarity FS(A) for all function A, the
2184 /// weight of function A in base profile WB(A), and the weight of function
2185 /// A in test profile WT(A), compute PS(base_profile, test_profile) =
2186 /// sum_A(FS(A) * avg(WB(A), WT(A))) ranging in [0.0f to 1.0f] with 0.0
2187 /// meaning no-overlap.
2188 void computeSampleProfileOverlap(raw_fd_ostream &OS);
2189
2190 /// Initialize ProfOverlap with the sum of samples in base and test
2191 /// profiles. This function also computes and keeps the sum of samples and
2192 /// max sample counts of each function in BaseStats and TestStats for later
2193 /// use to avoid re-computations.
2194 void initializeSampleProfileOverlap();
2195
2196 /// Load profiles specified by BaseFilename and TestFilename.
2197 Error loadProfiles();
2198
2199 using FuncSampleStatsMap = DenseMap<SampleContext, FuncSampleStats>;
2200
2201private:
2202 SampleOverlapStats ProfOverlap;
2203 SampleOverlapStats HotFuncOverlap;
2204 SampleOverlapStats HotBlockOverlap;
2205 std::string BaseFilename;
2206 std::string TestFilename;
2207 std::unique_ptr<sampleprof::SampleProfileReader> BaseReader;
2208 std::unique_ptr<sampleprof::SampleProfileReader> TestReader;
2209 // BaseStats and TestStats hold FuncSampleStats for each function, with
2210 // function name as the key.
2211 FuncSampleStatsMap BaseStats;
2212 FuncSampleStatsMap TestStats;
2213 // Low similarity threshold in floating point number
2214 double LowSimilarityThreshold;
2215 // Block samples above BaseHotThreshold or TestHotThreshold are considered hot
2216 // for tracking hot blocks.
2217 uint64_t BaseHotThreshold;
2218 uint64_t TestHotThreshold;
2219 // A small threshold used to round the results of floating point accumulations
2220 // to resolve imprecision.
2221 const double Epsilon;
2222 std::multimap<double, SampleOverlapStats, std::greater<double>>
2223 FuncSimilarityDump;
2224 // FuncFilter carries specifications in options --value-cutoff and
2225 // --function.
2226 OverlapFuncFilters FuncFilter;
2227 // Column offsets for printing the function-level details table.
2228 static const unsigned int TestWeightCol = 15;
2229 static const unsigned int SimilarityCol = 30;
2230 static const unsigned int OverlapCol = 43;
2231 static const unsigned int BaseUniqueCol = 53;
2232 static const unsigned int TestUniqueCol = 67;
2233 static const unsigned int BaseSampleCol = 81;
2234 static const unsigned int TestSampleCol = 96;
2235 static const unsigned int FuncNameCol = 111;
2236
2237 /// Return a similarity of two line/block sample counters in the same
2238 /// function in base and test profiles. The line/block-similarity BS(i) is
2239 /// computed as follows:
2240 /// For an offsets i, given the sample count at i in base profile BB(i),
2241 /// the sample count at i in test profile BT(i), the sum of sample counts
2242 /// in this function in base profile SB, and the sum of sample counts in
2243 /// this function in test profile ST, compute BS(i) = 1.0 - fabs(BB(i)/SB -
2244 /// BT(i)/ST), ranging in [0.0f to 1.0f] with 0.0 meaning no-overlap.
2245 double computeBlockSimilarity(uint64_t BaseSample, uint64_t TestSample,
2246 const SampleOverlapStats &FuncOverlap) const;
2247
2248 void updateHotBlockOverlap(uint64_t BaseSample, uint64_t TestSample,
2249 uint64_t HotBlockCount);
2250
2251 void getHotFunctions(const FuncSampleStatsMap &ProfStats,
2252 FuncSampleStatsMap &HotFunc,
2253 uint64_t HotThreshold) const;
2254
2255 void computeHotFuncOverlap();
2256
2257 /// This function updates statistics in FuncOverlap, HotBlockOverlap, and
2258 /// Difference for two sample units in a matched function according to the
2259 /// given match status.
2260 void updateOverlapStatsForFunction(uint64_t BaseSample, uint64_t TestSample,
2261 uint64_t HotBlockCount,
2262 SampleOverlapStats &FuncOverlap,
2263 double &Difference, MatchStatus Status);
2264
2265 /// This function updates statistics in FuncOverlap, HotBlockOverlap, and
2266 /// Difference for unmatched callees that only present in one profile in a
2267 /// matched caller function.
2268 void updateForUnmatchedCallee(const sampleprof::FunctionSamples &Func,
2269 SampleOverlapStats &FuncOverlap,
2270 double &Difference, MatchStatus Status);
2271
2272 /// This function updates sample overlap statistics of an overlap function in
2273 /// base and test profile. It also calculates a function-internal similarity
2274 /// FIS as follows:
2275 /// For offsets i that have samples in at least one profile in this
2276 /// function A, given BS(i) returned by computeBlockSimilarity(), compute
2277 /// FIS(A) = (2.0 - sum_i(1.0 - BS(i))) / 2, ranging in [0.0f to 1.0f] with
2278 /// 0.0 meaning no overlap.
2279 double computeSampleFunctionInternalOverlap(
2280 const sampleprof::FunctionSamples &BaseFunc,
2281 const sampleprof::FunctionSamples &TestFunc,
2282 SampleOverlapStats &FuncOverlap);
2283
2284 /// Function-level similarity (FS) is a weighted value over function internal
2285 /// similarity (FIS). This function computes a function's FS from its FIS by
2286 /// applying the weight.
2287 double weightForFuncSimilarity(double FuncSimilarity, uint64_t BaseFuncSample,
2288 uint64_t TestFuncSample) const;
2289
2290 /// The function-level similarity FS(A) for a function A is computed as
2291 /// follows:
2292 /// Compute a function-internal similarity FIS(A) by
2293 /// computeSampleFunctionInternalOverlap(). Then, with the weight of
2294 /// function A in base profile WB(A), and the weight of function A in test
2295 /// profile WT(A), compute FS(A) = FIS(A) * (1.0 - fabs(WB(A) - WT(A)))
2296 /// ranging in [0.0f to 1.0f] with 0.0 meaning no overlap.
2297 double
2298 computeSampleFunctionOverlap(const sampleprof::FunctionSamples *BaseFunc,
2299 const sampleprof::FunctionSamples *TestFunc,
2300 SampleOverlapStats *FuncOverlap,
2301 uint64_t BaseFuncSample,
2302 uint64_t TestFuncSample);
2303
2304 /// Profile-level similarity (PS) is a weighted aggregate over function-level
2305 /// similarities (FS). This method weights the FS value by the function
2306 /// weights in the base and test profiles for the aggregation.
2307 double weightByImportance(double FuncSimilarity, uint64_t BaseFuncSample,
2308 uint64_t TestFuncSample) const;
2309};
2310} // end anonymous namespace
2311
2312bool SampleOverlapAggregator::detectZeroSampleProfile(
2313 raw_fd_ostream &OS) const {
2314 bool HaveZeroSample = false;
2315 if (ProfOverlap.BaseSample == 0) {
2316 OS << "Sum of sample counts for profile " << BaseFilename << " is 0.\n";
2317 HaveZeroSample = true;
2318 }
2319 if (ProfOverlap.TestSample == 0) {
2320 OS << "Sum of sample counts for profile " << TestFilename << " is 0.\n";
2321 HaveZeroSample = true;
2322 }
2323 return HaveZeroSample;
2324}
2325
2326double SampleOverlapAggregator::computeBlockSimilarity(
2327 uint64_t BaseSample, uint64_t TestSample,
2328 const SampleOverlapStats &FuncOverlap) const {
2329 double BaseFrac = 0.0;
2330 double TestFrac = 0.0;
2331 if (FuncOverlap.BaseSample > 0)
2332 BaseFrac = static_cast<double>(BaseSample) / FuncOverlap.BaseSample;
2333 if (FuncOverlap.TestSample > 0)
2334 TestFrac = static_cast<double>(TestSample) / FuncOverlap.TestSample;
2335 return 1.0 - std::fabs(x: BaseFrac - TestFrac);
2336}
2337
2338void SampleOverlapAggregator::updateHotBlockOverlap(uint64_t BaseSample,
2339 uint64_t TestSample,
2340 uint64_t HotBlockCount) {
2341 bool IsBaseHot = (BaseSample >= BaseHotThreshold);
2342 bool IsTestHot = (TestSample >= TestHotThreshold);
2343 if (!IsBaseHot && !IsTestHot)
2344 return;
2345
2346 HotBlockOverlap.UnionCount += HotBlockCount;
2347 if (IsBaseHot)
2348 HotBlockOverlap.BaseCount += HotBlockCount;
2349 if (IsTestHot)
2350 HotBlockOverlap.TestCount += HotBlockCount;
2351 if (IsBaseHot && IsTestHot)
2352 HotBlockOverlap.OverlapCount += HotBlockCount;
2353}
2354
2355void SampleOverlapAggregator::getHotFunctions(
2356 const FuncSampleStatsMap &ProfStats, FuncSampleStatsMap &HotFunc,
2357 uint64_t HotThreshold) const {
2358 for (const auto &F : ProfStats) {
2359 if (isFunctionHot(FuncStats: F.second, HotThreshold))
2360 HotFunc.try_emplace(Key: F.first, Args: F.second);
2361 }
2362}
2363
2364void SampleOverlapAggregator::computeHotFuncOverlap() {
2365 FuncSampleStatsMap BaseHotFunc;
2366 getHotFunctions(ProfStats: BaseStats, HotFunc&: BaseHotFunc, HotThreshold: BaseHotThreshold);
2367 HotFuncOverlap.BaseCount = BaseHotFunc.size();
2368
2369 FuncSampleStatsMap TestHotFunc;
2370 getHotFunctions(ProfStats: TestStats, HotFunc&: TestHotFunc, HotThreshold: TestHotThreshold);
2371 HotFuncOverlap.TestCount = TestHotFunc.size();
2372 HotFuncOverlap.UnionCount = HotFuncOverlap.TestCount;
2373
2374 for (const auto &F : BaseHotFunc) {
2375 if (TestHotFunc.count(Val: F.first))
2376 ++HotFuncOverlap.OverlapCount;
2377 else
2378 ++HotFuncOverlap.UnionCount;
2379 }
2380}
2381
2382void SampleOverlapAggregator::updateOverlapStatsForFunction(
2383 uint64_t BaseSample, uint64_t TestSample, uint64_t HotBlockCount,
2384 SampleOverlapStats &FuncOverlap, double &Difference, MatchStatus Status) {
2385 assert(Status != MS_None &&
2386 "Match status should be updated before updating overlap statistics");
2387 if (Status == MS_FirstUnique) {
2388 TestSample = 0;
2389 FuncOverlap.BaseUniqueSample += BaseSample;
2390 } else if (Status == MS_SecondUnique) {
2391 BaseSample = 0;
2392 FuncOverlap.TestUniqueSample += TestSample;
2393 } else {
2394 ++FuncOverlap.OverlapCount;
2395 }
2396
2397 FuncOverlap.UnionSample += std::max(a: BaseSample, b: TestSample);
2398 FuncOverlap.OverlapSample += std::min(a: BaseSample, b: TestSample);
2399 Difference +=
2400 1.0 - computeBlockSimilarity(BaseSample, TestSample, FuncOverlap);
2401 updateHotBlockOverlap(BaseSample, TestSample, HotBlockCount);
2402}
2403
2404void SampleOverlapAggregator::updateForUnmatchedCallee(
2405 const sampleprof::FunctionSamples &Func, SampleOverlapStats &FuncOverlap,
2406 double &Difference, MatchStatus Status) {
2407 assert((Status == MS_FirstUnique || Status == MS_SecondUnique) &&
2408 "Status must be either of the two unmatched cases");
2409 FuncSampleStats FuncStats;
2410 if (Status == MS_FirstUnique) {
2411 getFuncSampleStats(Func, FuncStats, HotThreshold: BaseHotThreshold);
2412 updateOverlapStatsForFunction(BaseSample: FuncStats.SampleSum, TestSample: 0,
2413 HotBlockCount: FuncStats.HotBlockCount, FuncOverlap,
2414 Difference, Status);
2415 } else {
2416 getFuncSampleStats(Func, FuncStats, HotThreshold: TestHotThreshold);
2417 updateOverlapStatsForFunction(BaseSample: 0, TestSample: FuncStats.SampleSum,
2418 HotBlockCount: FuncStats.HotBlockCount, FuncOverlap,
2419 Difference, Status);
2420 }
2421}
2422
2423double SampleOverlapAggregator::computeSampleFunctionInternalOverlap(
2424 const sampleprof::FunctionSamples &BaseFunc,
2425 const sampleprof::FunctionSamples &TestFunc,
2426 SampleOverlapStats &FuncOverlap) {
2427
2428 using namespace sampleprof;
2429
2430 double Difference = 0;
2431
2432 // Accumulate Difference for regular line/block samples in the function.
2433 // We match them through sort-merge join algorithm because
2434 // FunctionSamples::getBodySamples() returns a map of sample counters ordered
2435 // by their offsets.
2436 MatchStep<BodySampleMap::const_iterator> BlockIterStep(
2437 BaseFunc.getBodySamples().cbegin(), BaseFunc.getBodySamples().cend(),
2438 TestFunc.getBodySamples().cbegin(), TestFunc.getBodySamples().cend());
2439 BlockIterStep.updateOneStep();
2440 while (!BlockIterStep.areBothFinished()) {
2441 uint64_t BaseSample =
2442 BlockIterStep.isFirstFinished()
2443 ? 0
2444 : BlockIterStep.getFirstIter()->second.getSamples();
2445 uint64_t TestSample =
2446 BlockIterStep.isSecondFinished()
2447 ? 0
2448 : BlockIterStep.getSecondIter()->second.getSamples();
2449 updateOverlapStatsForFunction(BaseSample, TestSample, HotBlockCount: 1, FuncOverlap,
2450 Difference, Status: BlockIterStep.getMatchStatus());
2451
2452 BlockIterStep.updateOneStep();
2453 }
2454
2455 // Accumulate Difference for callsite lines in the function. We match
2456 // them through sort-merge algorithm because
2457 // FunctionSamples::getCallsiteSamples() returns a map of callsite records
2458 // ordered by their offsets.
2459 MatchStep<CallsiteSampleMap::const_iterator> CallsiteIterStep(
2460 BaseFunc.getCallsiteSamples().cbegin(),
2461 BaseFunc.getCallsiteSamples().cend(),
2462 TestFunc.getCallsiteSamples().cbegin(),
2463 TestFunc.getCallsiteSamples().cend());
2464 CallsiteIterStep.updateOneStep();
2465 while (!CallsiteIterStep.areBothFinished()) {
2466 MatchStatus CallsiteStepStatus = CallsiteIterStep.getMatchStatus();
2467 assert(CallsiteStepStatus != MS_None &&
2468 "Match status should be updated before entering loop body");
2469
2470 if (CallsiteStepStatus != MS_Match) {
2471 auto Callsite = (CallsiteStepStatus == MS_FirstUnique)
2472 ? CallsiteIterStep.getFirstIter()
2473 : CallsiteIterStep.getSecondIter();
2474 for (const auto &F : Callsite->second)
2475 updateForUnmatchedCallee(Func: F.second, FuncOverlap, Difference,
2476 Status: CallsiteStepStatus);
2477 } else {
2478 // There may be multiple inlinees at the same offset, so we need to try
2479 // matching all of them. This match is implemented through sort-merge
2480 // algorithm because callsite records at the same offset are ordered by
2481 // function names.
2482 MatchStep<FunctionSamplesMap::const_iterator> CalleeIterStep(
2483 CallsiteIterStep.getFirstIter()->second.cbegin(),
2484 CallsiteIterStep.getFirstIter()->second.cend(),
2485 CallsiteIterStep.getSecondIter()->second.cbegin(),
2486 CallsiteIterStep.getSecondIter()->second.cend());
2487 CalleeIterStep.updateOneStep();
2488 while (!CalleeIterStep.areBothFinished()) {
2489 MatchStatus CalleeStepStatus = CalleeIterStep.getMatchStatus();
2490 if (CalleeStepStatus != MS_Match) {
2491 auto Callee = (CalleeStepStatus == MS_FirstUnique)
2492 ? CalleeIterStep.getFirstIter()
2493 : CalleeIterStep.getSecondIter();
2494 updateForUnmatchedCallee(Func: Callee->second, FuncOverlap, Difference,
2495 Status: CalleeStepStatus);
2496 } else {
2497 // An inlined function can contain other inlinees inside, so compute
2498 // the Difference recursively.
2499 Difference += 2.0 - 2 * computeSampleFunctionInternalOverlap(
2500 BaseFunc: CalleeIterStep.getFirstIter()->second,
2501 TestFunc: CalleeIterStep.getSecondIter()->second,
2502 FuncOverlap);
2503 }
2504 CalleeIterStep.updateOneStep();
2505 }
2506 }
2507 CallsiteIterStep.updateOneStep();
2508 }
2509
2510 // Difference reflects the total differences of line/block samples in this
2511 // function and ranges in [0.0f to 2.0f]. Take (2.0 - Difference) / 2 to
2512 // reflect the similarity between function profiles in [0.0f to 1.0f].
2513 return (2.0 - Difference) / 2;
2514}
2515
2516double SampleOverlapAggregator::weightForFuncSimilarity(
2517 double FuncInternalSimilarity, uint64_t BaseFuncSample,
2518 uint64_t TestFuncSample) const {
2519 // Compute the weight as the distance between the function weights in two
2520 // profiles.
2521 double BaseFrac = 0.0;
2522 double TestFrac = 0.0;
2523 assert(ProfOverlap.BaseSample > 0 &&
2524 "Total samples in base profile should be greater than 0");
2525 BaseFrac = static_cast<double>(BaseFuncSample) / ProfOverlap.BaseSample;
2526 assert(ProfOverlap.TestSample > 0 &&
2527 "Total samples in test profile should be greater than 0");
2528 TestFrac = static_cast<double>(TestFuncSample) / ProfOverlap.TestSample;
2529 double WeightDistance = std::fabs(x: BaseFrac - TestFrac);
2530
2531 // Take WeightDistance into the similarity.
2532 return FuncInternalSimilarity * (1 - WeightDistance);
2533}
2534
2535double
2536SampleOverlapAggregator::weightByImportance(double FuncSimilarity,
2537 uint64_t BaseFuncSample,
2538 uint64_t TestFuncSample) const {
2539
2540 double BaseFrac = 0.0;
2541 double TestFrac = 0.0;
2542 assert(ProfOverlap.BaseSample > 0 &&
2543 "Total samples in base profile should be greater than 0");
2544 BaseFrac = static_cast<double>(BaseFuncSample) / ProfOverlap.BaseSample / 2.0;
2545 assert(ProfOverlap.TestSample > 0 &&
2546 "Total samples in test profile should be greater than 0");
2547 TestFrac = static_cast<double>(TestFuncSample) / ProfOverlap.TestSample / 2.0;
2548 return FuncSimilarity * (BaseFrac + TestFrac);
2549}
2550
2551double SampleOverlapAggregator::computeSampleFunctionOverlap(
2552 const sampleprof::FunctionSamples *BaseFunc,
2553 const sampleprof::FunctionSamples *TestFunc,
2554 SampleOverlapStats *FuncOverlap, uint64_t BaseFuncSample,
2555 uint64_t TestFuncSample) {
2556 // Default function internal similarity before weighted, meaning two functions
2557 // has no overlap.
2558 const double DefaultFuncInternalSimilarity = 0;
2559 double FuncSimilarity;
2560 double FuncInternalSimilarity;
2561
2562 // If BaseFunc or TestFunc is nullptr, it means the functions do not overlap.
2563 // In this case, we use DefaultFuncInternalSimilarity as the function internal
2564 // similarity.
2565 if (!BaseFunc || !TestFunc) {
2566 FuncInternalSimilarity = DefaultFuncInternalSimilarity;
2567 } else {
2568 assert(FuncOverlap != nullptr &&
2569 "FuncOverlap should be provided in this case");
2570 FuncInternalSimilarity = computeSampleFunctionInternalOverlap(
2571 BaseFunc: *BaseFunc, TestFunc: *TestFunc, FuncOverlap&: *FuncOverlap);
2572 // Now, FuncInternalSimilarity may be a little less than 0 due to
2573 // imprecision of floating point accumulations. Make it zero if the
2574 // difference is below Epsilon.
2575 FuncInternalSimilarity = (std::fabs(x: FuncInternalSimilarity - 0) < Epsilon)
2576 ? 0
2577 : FuncInternalSimilarity;
2578 }
2579 FuncSimilarity = weightForFuncSimilarity(FuncInternalSimilarity,
2580 BaseFuncSample, TestFuncSample);
2581 return FuncSimilarity;
2582}
2583
2584void SampleOverlapAggregator::computeSampleProfileOverlap(raw_fd_ostream &OS) {
2585 using namespace sampleprof;
2586
2587 DenseMap<SampleContext, const FunctionSamples *> BaseFuncProf;
2588 const auto &BaseProfiles = BaseReader->getProfiles();
2589 for (const auto &BaseFunc : BaseProfiles) {
2590 BaseFuncProf.try_emplace(Key: BaseFunc.second.getContext(), Args: &(BaseFunc.second));
2591 }
2592 ProfOverlap.UnionCount = BaseFuncProf.size();
2593
2594 const auto &TestProfiles = TestReader->getProfiles();
2595 for (const auto &TestFunc : TestProfiles) {
2596 SampleOverlapStats FuncOverlap;
2597 FuncOverlap.TestName = TestFunc.second.getContext();
2598 assert(TestStats.count(FuncOverlap.TestName) &&
2599 "TestStats should have records for all functions in test profile "
2600 "except inlinees");
2601 FuncOverlap.TestSample = TestStats[FuncOverlap.TestName].SampleSum;
2602
2603 bool Matched = false;
2604 const auto Match = BaseFuncProf.find(Val: FuncOverlap.TestName);
2605 if (Match == BaseFuncProf.end()) {
2606 const FuncSampleStats &FuncStats = TestStats[FuncOverlap.TestName];
2607 ++ProfOverlap.TestUniqueCount;
2608 ProfOverlap.TestUniqueSample += FuncStats.SampleSum;
2609 FuncOverlap.TestUniqueSample = FuncStats.SampleSum;
2610
2611 updateHotBlockOverlap(BaseSample: 0, TestSample: FuncStats.SampleSum, HotBlockCount: FuncStats.HotBlockCount);
2612
2613 double FuncSimilarity = computeSampleFunctionOverlap(
2614 BaseFunc: nullptr, TestFunc: nullptr, FuncOverlap: nullptr, BaseFuncSample: 0, TestFuncSample: FuncStats.SampleSum);
2615 ProfOverlap.Similarity +=
2616 weightByImportance(FuncSimilarity, BaseFuncSample: 0, TestFuncSample: FuncStats.SampleSum);
2617
2618 ++ProfOverlap.UnionCount;
2619 ProfOverlap.UnionSample += FuncStats.SampleSum;
2620 } else {
2621 ++ProfOverlap.OverlapCount;
2622
2623 // Two functions match with each other. Compute function-level overlap and
2624 // aggregate them into profile-level overlap.
2625 FuncOverlap.BaseName = Match->second->getContext();
2626 assert(BaseStats.count(FuncOverlap.BaseName) &&
2627 "BaseStats should have records for all functions in base profile "
2628 "except inlinees");
2629 FuncOverlap.BaseSample = BaseStats[FuncOverlap.BaseName].SampleSum;
2630
2631 FuncOverlap.Similarity = computeSampleFunctionOverlap(
2632 BaseFunc: Match->second, TestFunc: &TestFunc.second, FuncOverlap: &FuncOverlap, BaseFuncSample: FuncOverlap.BaseSample,
2633 TestFuncSample: FuncOverlap.TestSample);
2634 ProfOverlap.Similarity +=
2635 weightByImportance(FuncSimilarity: FuncOverlap.Similarity, BaseFuncSample: FuncOverlap.BaseSample,
2636 TestFuncSample: FuncOverlap.TestSample);
2637 ProfOverlap.OverlapSample += FuncOverlap.OverlapSample;
2638 ProfOverlap.UnionSample += FuncOverlap.UnionSample;
2639
2640 // Accumulate the percentage of base unique and test unique samples into
2641 // ProfOverlap.
2642 ProfOverlap.BaseUniqueSample += FuncOverlap.BaseUniqueSample;
2643 ProfOverlap.TestUniqueSample += FuncOverlap.TestUniqueSample;
2644
2645 // Remove matched base functions for later reporting functions not found
2646 // in test profile.
2647 BaseFuncProf.erase(I: Match);
2648 Matched = true;
2649 }
2650
2651 // Print function-level similarity information if specified by options.
2652 assert(TestStats.count(FuncOverlap.TestName) &&
2653 "TestStats should have records for all functions in test profile "
2654 "except inlinees");
2655 if (TestStats[FuncOverlap.TestName].MaxSample >= FuncFilter.ValueCutoff ||
2656 (Matched && FuncOverlap.Similarity < LowSimilarityThreshold) ||
2657 (Matched && !FuncFilter.NameFilter.empty() &&
2658 FuncOverlap.BaseName.toString().find(str: FuncFilter.NameFilter) !=
2659 std::string::npos)) {
2660 assert(ProfOverlap.BaseSample > 0 &&
2661 "Total samples in base profile should be greater than 0");
2662 FuncOverlap.BaseWeight =
2663 static_cast<double>(FuncOverlap.BaseSample) / ProfOverlap.BaseSample;
2664 assert(ProfOverlap.TestSample > 0 &&
2665 "Total samples in test profile should be greater than 0");
2666 FuncOverlap.TestWeight =
2667 static_cast<double>(FuncOverlap.TestSample) / ProfOverlap.TestSample;
2668 FuncSimilarityDump.emplace(args&: FuncOverlap.BaseWeight, args&: FuncOverlap);
2669 }
2670 }
2671
2672 // Traverse through functions in base profile but not in test profile.
2673 for (const auto &F : BaseFuncProf) {
2674 assert(BaseStats.count(F.second->getContext()) &&
2675 "BaseStats should have records for all functions in base profile "
2676 "except inlinees");
2677 const FuncSampleStats &FuncStats = BaseStats[F.second->getContext()];
2678 ++ProfOverlap.BaseUniqueCount;
2679 ProfOverlap.BaseUniqueSample += FuncStats.SampleSum;
2680
2681 updateHotBlockOverlap(BaseSample: FuncStats.SampleSum, TestSample: 0, HotBlockCount: FuncStats.HotBlockCount);
2682
2683 double FuncSimilarity = computeSampleFunctionOverlap(
2684 BaseFunc: nullptr, TestFunc: nullptr, FuncOverlap: nullptr, BaseFuncSample: FuncStats.SampleSum, TestFuncSample: 0);
2685 ProfOverlap.Similarity +=
2686 weightByImportance(FuncSimilarity, BaseFuncSample: FuncStats.SampleSum, TestFuncSample: 0);
2687
2688 ProfOverlap.UnionSample += FuncStats.SampleSum;
2689 }
2690
2691 // Now, ProfSimilarity may be a little greater than 1 due to imprecision
2692 // of floating point accumulations. Make it 1.0 if the difference is below
2693 // Epsilon.
2694 ProfOverlap.Similarity = (std::fabs(x: ProfOverlap.Similarity - 1) < Epsilon)
2695 ? 1
2696 : ProfOverlap.Similarity;
2697
2698 computeHotFuncOverlap();
2699}
2700
2701void SampleOverlapAggregator::initializeSampleProfileOverlap() {
2702 const auto &BaseProf = BaseReader->getProfiles();
2703 for (const auto &I : BaseProf) {
2704 ++ProfOverlap.BaseCount;
2705 FuncSampleStats FuncStats;
2706 getFuncSampleStats(Func: I.second, FuncStats, HotThreshold: BaseHotThreshold);
2707 ProfOverlap.BaseSample += FuncStats.SampleSum;
2708 BaseStats.try_emplace(Key: I.second.getContext(), Args&: FuncStats);
2709 }
2710
2711 const auto &TestProf = TestReader->getProfiles();
2712 for (const auto &I : TestProf) {
2713 ++ProfOverlap.TestCount;
2714 FuncSampleStats FuncStats;
2715 getFuncSampleStats(Func: I.second, FuncStats, HotThreshold: TestHotThreshold);
2716 ProfOverlap.TestSample += FuncStats.SampleSum;
2717 TestStats.try_emplace(Key: I.second.getContext(), Args&: FuncStats);
2718 }
2719
2720 ProfOverlap.BaseName = StringRef(BaseFilename);
2721 ProfOverlap.TestName = StringRef(TestFilename);
2722}
2723
2724void SampleOverlapAggregator::dumpFuncSimilarity(raw_fd_ostream &OS) const {
2725 using namespace sampleprof;
2726
2727 if (FuncSimilarityDump.empty())
2728 return;
2729
2730 formatted_raw_ostream FOS(OS);
2731 FOS << "Function-level details:\n";
2732 FOS << "Base weight";
2733 FOS.PadToColumn(NewCol: TestWeightCol);
2734 FOS << "Test weight";
2735 FOS.PadToColumn(NewCol: SimilarityCol);
2736 FOS << "Similarity";
2737 FOS.PadToColumn(NewCol: OverlapCol);
2738 FOS << "Overlap";
2739 FOS.PadToColumn(NewCol: BaseUniqueCol);
2740 FOS << "Base unique";
2741 FOS.PadToColumn(NewCol: TestUniqueCol);
2742 FOS << "Test unique";
2743 FOS.PadToColumn(NewCol: BaseSampleCol);
2744 FOS << "Base samples";
2745 FOS.PadToColumn(NewCol: TestSampleCol);
2746 FOS << "Test samples";
2747 FOS.PadToColumn(NewCol: FuncNameCol);
2748 FOS << "Function name\n";
2749 for (const auto &F : FuncSimilarityDump) {
2750 double OverlapPercent =
2751 F.second.UnionSample > 0
2752 ? static_cast<double>(F.second.OverlapSample) / F.second.UnionSample
2753 : 0;
2754 double BaseUniquePercent =
2755 F.second.BaseSample > 0
2756 ? static_cast<double>(F.second.BaseUniqueSample) /
2757 F.second.BaseSample
2758 : 0;
2759 double TestUniquePercent =
2760 F.second.TestSample > 0
2761 ? static_cast<double>(F.second.TestUniqueSample) /
2762 F.second.TestSample
2763 : 0;
2764
2765 FOS << format(Fmt: "%.2f%%", Vals: F.second.BaseWeight * 100);
2766 FOS.PadToColumn(NewCol: TestWeightCol);
2767 FOS << format(Fmt: "%.2f%%", Vals: F.second.TestWeight * 100);
2768 FOS.PadToColumn(NewCol: SimilarityCol);
2769 FOS << format(Fmt: "%.2f%%", Vals: F.second.Similarity * 100);
2770 FOS.PadToColumn(NewCol: OverlapCol);
2771 FOS << format(Fmt: "%.2f%%", Vals: OverlapPercent * 100);
2772 FOS.PadToColumn(NewCol: BaseUniqueCol);
2773 FOS << format(Fmt: "%.2f%%", Vals: BaseUniquePercent * 100);
2774 FOS.PadToColumn(NewCol: TestUniqueCol);
2775 FOS << format(Fmt: "%.2f%%", Vals: TestUniquePercent * 100);
2776 FOS.PadToColumn(NewCol: BaseSampleCol);
2777 FOS << F.second.BaseSample;
2778 FOS.PadToColumn(NewCol: TestSampleCol);
2779 FOS << F.second.TestSample;
2780 FOS.PadToColumn(NewCol: FuncNameCol);
2781 FOS << F.second.TestName.toString() << "\n";
2782 }
2783}
2784
2785void SampleOverlapAggregator::dumpProgramSummary(raw_fd_ostream &OS) const {
2786 OS << "Profile overlap information for base_profile: "
2787 << ProfOverlap.BaseName.toString()
2788 << " and test_profile: " << ProfOverlap.TestName.toString()
2789 << "\nProgram level:\n";
2790
2791 OS << " Whole program profile similarity: "
2792 << format(Fmt: "%.3f%%", Vals: ProfOverlap.Similarity * 100) << "\n";
2793
2794 assert(ProfOverlap.UnionSample > 0 &&
2795 "Total samples in two profile should be greater than 0");
2796 double OverlapPercent =
2797 static_cast<double>(ProfOverlap.OverlapSample) / ProfOverlap.UnionSample;
2798 assert(ProfOverlap.BaseSample > 0 &&
2799 "Total samples in base profile should be greater than 0");
2800 double BaseUniquePercent = static_cast<double>(ProfOverlap.BaseUniqueSample) /
2801 ProfOverlap.BaseSample;
2802 assert(ProfOverlap.TestSample > 0 &&
2803 "Total samples in test profile should be greater than 0");
2804 double TestUniquePercent = static_cast<double>(ProfOverlap.TestUniqueSample) /
2805 ProfOverlap.TestSample;
2806
2807 OS << " Whole program sample overlap: "
2808 << format(Fmt: "%.3f%%", Vals: OverlapPercent * 100) << "\n";
2809 OS << " percentage of samples unique in base profile: "
2810 << format(Fmt: "%.3f%%", Vals: BaseUniquePercent * 100) << "\n";
2811 OS << " percentage of samples unique in test profile: "
2812 << format(Fmt: "%.3f%%", Vals: TestUniquePercent * 100) << "\n";
2813 OS << " total samples in base profile: " << ProfOverlap.BaseSample << "\n"
2814 << " total samples in test profile: " << ProfOverlap.TestSample << "\n";
2815
2816 assert(ProfOverlap.UnionCount > 0 &&
2817 "There should be at least one function in two input profiles");
2818 double FuncOverlapPercent =
2819 static_cast<double>(ProfOverlap.OverlapCount) / ProfOverlap.UnionCount;
2820 OS << " Function overlap: " << format(Fmt: "%.3f%%", Vals: FuncOverlapPercent * 100)
2821 << "\n";
2822 OS << " overlap functions: " << ProfOverlap.OverlapCount << "\n";
2823 OS << " functions unique in base profile: " << ProfOverlap.BaseUniqueCount
2824 << "\n";
2825 OS << " functions unique in test profile: " << ProfOverlap.TestUniqueCount
2826 << "\n";
2827}
2828
2829void SampleOverlapAggregator::dumpHotFuncAndBlockOverlap(
2830 raw_fd_ostream &OS) const {
2831 assert(HotFuncOverlap.UnionCount > 0 &&
2832 "There should be at least one hot function in two input profiles");
2833 OS << " Hot-function overlap: "
2834 << format(Fmt: "%.3f%%", Vals: static_cast<double>(HotFuncOverlap.OverlapCount) /
2835 HotFuncOverlap.UnionCount * 100)
2836 << "\n";
2837 OS << " overlap hot functions: " << HotFuncOverlap.OverlapCount << "\n";
2838 OS << " hot functions unique in base profile: "
2839 << HotFuncOverlap.BaseCount - HotFuncOverlap.OverlapCount << "\n";
2840 OS << " hot functions unique in test profile: "
2841 << HotFuncOverlap.TestCount - HotFuncOverlap.OverlapCount << "\n";
2842
2843 assert(HotBlockOverlap.UnionCount > 0 &&
2844 "There should be at least one hot block in two input profiles");
2845 OS << " Hot-block overlap: "
2846 << format(Fmt: "%.3f%%", Vals: static_cast<double>(HotBlockOverlap.OverlapCount) /
2847 HotBlockOverlap.UnionCount * 100)
2848 << "\n";
2849 OS << " overlap hot blocks: " << HotBlockOverlap.OverlapCount << "\n";
2850 OS << " hot blocks unique in base profile: "
2851 << HotBlockOverlap.BaseCount - HotBlockOverlap.OverlapCount << "\n";
2852 OS << " hot blocks unique in test profile: "
2853 << HotBlockOverlap.TestCount - HotBlockOverlap.OverlapCount << "\n";
2854}
2855
2856Error SampleOverlapAggregator::loadProfiles() {
2857 using namespace sampleprof;
2858
2859 LLVMContext Context;
2860 auto FS = vfs::getRealFileSystem();
2861 auto BaseReaderOrErr = SampleProfileReader::create(Filename: BaseFilename, C&: Context, FS&: *FS,
2862 P: FSDiscriminatorPassOption);
2863 if (std::error_code EC = BaseReaderOrErr.getError())
2864 return makeError(EC, Whence: BaseFilename);
2865
2866 auto TestReaderOrErr = SampleProfileReader::create(Filename: TestFilename, C&: Context, FS&: *FS,
2867 P: FSDiscriminatorPassOption);
2868 if (std::error_code EC = TestReaderOrErr.getError())
2869 return makeError(EC, Whence: TestFilename);
2870
2871 BaseReader = std::move(BaseReaderOrErr.get());
2872 TestReader = std::move(TestReaderOrErr.get());
2873
2874 if (std::error_code EC = BaseReader->read())
2875 return makeError(EC, Whence: BaseFilename);
2876 if (std::error_code EC = TestReader->read())
2877 return makeError(EC, Whence: TestFilename);
2878 if (BaseReader->profileIsProbeBased() != TestReader->profileIsProbeBased())
2879 return makeError(
2880 Message: "cannot compare probe-based profile with non-probe-based profile");
2881 if (BaseReader->profileIsCS() != TestReader->profileIsCS())
2882 return makeError(Message: "cannot compare CS profile with non-CS profile");
2883
2884 // Load BaseHotThreshold and TestHotThreshold as 99-percentile threshold in
2885 // profile summary.
2886 ProfileSummary &BasePS = BaseReader->getSummary();
2887 ProfileSummary &TestPS = TestReader->getSummary();
2888 BaseHotThreshold =
2889 ProfileSummaryBuilder::getHotCountThreshold(DS: BasePS.getDetailedSummary());
2890 TestHotThreshold =
2891 ProfileSummaryBuilder::getHotCountThreshold(DS: TestPS.getDetailedSummary());
2892
2893 return Error::success();
2894}
2895
2896static Error overlapSampleProfile(const std::string &BaseFilename,
2897 const std::string &TestFilename,
2898 const OverlapFuncFilters &FuncFilter,
2899 uint64_t SimilarityCutoff,
2900 raw_fd_ostream &OS) {
2901 using namespace sampleprof;
2902
2903 // We use 0.000005 to initialize OverlapAggr.Epsilon because the final metrics
2904 // report 2--3 places after decimal point in percentage numbers.
2905 SampleOverlapAggregator OverlapAggr(
2906 BaseFilename, TestFilename,
2907 static_cast<double>(SimilarityCutoff) / 1000000, 0.000005, FuncFilter);
2908 if (Error E = OverlapAggr.loadProfiles())
2909 return E;
2910
2911 OverlapAggr.initializeSampleProfileOverlap();
2912 if (OverlapAggr.detectZeroSampleProfile(OS))
2913 return Error::success();
2914
2915 OverlapAggr.computeSampleProfileOverlap(OS);
2916
2917 OverlapAggr.dumpProgramSummary(OS);
2918 OverlapAggr.dumpHotFuncAndBlockOverlap(OS);
2919 OverlapAggr.dumpFuncSimilarity(OS);
2920 return Error::success();
2921}
2922
2923static Error overlap_main() {
2924 std::error_code EC;
2925 raw_fd_ostream OS(OutputFilename.data(), EC, sys::fs::OF_TextWithCRLF);
2926 if (EC)
2927 return makeError(EC, Whence: OutputFilename);
2928
2929 if (ProfileKind == instr)
2930 return overlapInstrProfile(
2931 BaseFilename, TestFilename,
2932 FuncFilter: OverlapFuncFilters{.ValueCutoff: OverlapValueCutoff, .NameFilter: FuncNameFilter}, OS, IsCS);
2933
2934 return overlapSampleProfile(
2935 BaseFilename, TestFilename,
2936 FuncFilter: OverlapFuncFilters{.ValueCutoff: OverlapValueCutoff, .NameFilter: FuncNameFilter}, SimilarityCutoff,
2937 OS);
2938}
2939
2940namespace {
2941struct ValueSitesStats {
2942 ValueSitesStats() = default;
2943 uint64_t TotalNumValueSites = 0;
2944 uint64_t TotalNumValueSitesWithValueProfile = 0;
2945 uint64_t TotalNumValues = 0;
2946 std::vector<unsigned> ValueSitesHistogram;
2947};
2948} // namespace
2949
2950static void traverseAllValueSites(const InstrProfRecord &Func, uint32_t VK,
2951 ValueSitesStats &Stats, raw_fd_ostream &OS,
2952 InstrProfSymtab *Symtab) {
2953 uint32_t NS = Func.getNumValueSites(ValueKind: VK);
2954 Stats.TotalNumValueSites += NS;
2955 for (size_t I = 0; I < NS; ++I) {
2956 auto VD = Func.getValueArrayForSite(ValueKind: VK, Site: I);
2957 uint32_t NV = VD.size();
2958 if (NV == 0)
2959 continue;
2960 Stats.TotalNumValues += NV;
2961 Stats.TotalNumValueSitesWithValueProfile++;
2962 if (NV > Stats.ValueSitesHistogram.size())
2963 Stats.ValueSitesHistogram.resize(new_size: NV, x: 0);
2964 Stats.ValueSitesHistogram[NV - 1]++;
2965
2966 uint64_t SiteSum = 0;
2967 for (const auto &V : VD)
2968 SiteSum += V.Count;
2969 if (SiteSum == 0)
2970 SiteSum = 1;
2971
2972 for (const auto &V : VD) {
2973 OS << "\t[ " << format(Fmt: "%2u", Vals: I) << ", ";
2974 if (Symtab == nullptr)
2975 OS << format(Fmt: "%4" PRIu64, Vals: V.Value);
2976 else
2977 OS << Symtab->getFuncOrVarName(MD5Hash: V.Value);
2978 OS << ", " << format(Fmt: "%10" PRId64, Vals: V.Count) << " ] ("
2979 << format(Fmt: "%.2f%%", Vals: (V.Count * 100.0 / SiteSum)) << ")\n";
2980 }
2981 }
2982}
2983
2984static void showValueSitesStats(raw_fd_ostream &OS, uint32_t VK,
2985 ValueSitesStats &Stats) {
2986 OS << " Total number of sites: " << Stats.TotalNumValueSites << "\n";
2987 OS << " Total number of sites with values: "
2988 << Stats.TotalNumValueSitesWithValueProfile << "\n";
2989 OS << " Total number of profiled values: " << Stats.TotalNumValues << "\n";
2990
2991 OS << " Value sites histogram:\n\tNumTargets, SiteCount\n";
2992 for (unsigned I = 0; I < Stats.ValueSitesHistogram.size(); I++) {
2993 if (Stats.ValueSitesHistogram[I] > 0)
2994 OS << "\t" << I + 1 << ", " << Stats.ValueSitesHistogram[I] << "\n";
2995 }
2996}
2997
2998static Error showInstrProfile(ShowFormat SFormat, raw_fd_ostream &OS) {
2999 if (SFormat == ShowFormat::Json)
3000 return makeError(Message: "JSON output is not supported for instr profiles");
3001 if (SFormat == ShowFormat::Yaml)
3002 return makeError(Message: "YAML output is not supported for instr profiles");
3003 auto FS = vfs::getRealFileSystem();
3004 auto ReaderOrErr = InstrProfReader::create(Path: Filename, FS&: *FS);
3005 std::vector<uint32_t> Cutoffs = std::move(DetailedSummaryCutoffs);
3006 if (Cutoffs.empty() && (ShowDetailedSummary || ShowHotFuncList))
3007 Cutoffs = ProfileSummaryBuilder::DefaultCutoffs;
3008 InstrProfSummaryBuilder Builder(std::move(Cutoffs));
3009 if (Error E = ReaderOrErr.takeError())
3010 return makeError(E: std::move(E), Whence: Filename);
3011
3012 auto Reader = std::move(ReaderOrErr.get());
3013 bool IsIRInstr = Reader->isIRLevelProfile();
3014 size_t ShownFunctions = 0;
3015 size_t BelowCutoffFunctions = 0;
3016 int NumVPKind = IPVK_Last - IPVK_First + 1;
3017 std::vector<ValueSitesStats> VPStats(NumVPKind);
3018
3019 std::vector<std::pair<StringRef, uint64_t>> NameAndMaxCount;
3020
3021 if (!TextFormat && OnlyListBelow) {
3022 OS << "The list of functions with the maximum counter less than "
3023 << ShowValueCutoff << ":\n";
3024 }
3025
3026 // Add marker so that IR-level instrumentation round-trips properly.
3027 if (TextFormat && IsIRInstr)
3028 OS << ":ir\n";
3029
3030 for (const auto &Func : *Reader) {
3031 if (Reader->isIRLevelProfile()) {
3032 bool FuncIsCS = NamedInstrProfRecord::hasCSFlagInHash(FuncHash: Func.Hash);
3033 if (FuncIsCS != ShowCS)
3034 continue;
3035 }
3036 bool Show = ShowAllFunctions ||
3037 (!FuncNameFilter.empty() && Func.Name.contains(Other: FuncNameFilter));
3038
3039 bool doTextFormatDump = (Show && TextFormat);
3040
3041 if (doTextFormatDump) {
3042 InstrProfSymtab &Symtab = Reader->getSymtab();
3043 InstrProfWriter::writeRecordInText(Name: Func.Name, Hash: Func.Hash, Counters: Func, Symtab,
3044 OS);
3045 continue;
3046 }
3047
3048 assert(Func.Counts.size() > 0 && "function missing entry counter");
3049 Builder.addRecord(Func);
3050
3051 if (ShowCovered) {
3052 if (llvm::any_of(Range: Func.Counts, P: [](uint64_t C) { return C; }))
3053 OS << Func.Name << "\n";
3054 continue;
3055 }
3056
3057 uint64_t FuncMax = 0;
3058 uint64_t FuncSum = 0;
3059
3060 auto PseudoKind = Func.getCountPseudoKind();
3061 if (PseudoKind != InstrProfRecord::NotPseudo) {
3062 if (Show) {
3063 if (!ShownFunctions)
3064 OS << "Counters:\n";
3065 ++ShownFunctions;
3066 OS << " " << Func.Name << ":\n"
3067 << " Hash: " << format(Fmt: "0x%016" PRIx64, Vals: Func.Hash) << "\n"
3068 << " Counters: " << Func.Counts.size();
3069 if (PseudoKind == InstrProfRecord::PseudoHot)
3070 OS << " <PseudoHot>\n";
3071 else if (PseudoKind == InstrProfRecord::PseudoWarm)
3072 OS << " <PseudoWarm>\n";
3073 else
3074 llvm_unreachable("Unknown PseudoKind");
3075 }
3076 continue;
3077 }
3078
3079 for (uint64_t Count : Func.Counts) {
3080 FuncMax = std::max(a: FuncMax, b: Count);
3081 FuncSum += Count;
3082 }
3083
3084 if (FuncMax < ShowValueCutoff) {
3085 ++BelowCutoffFunctions;
3086 if (OnlyListBelow) {
3087 OS << " " << Func.Name << ": (Max = " << FuncMax
3088 << " Sum = " << FuncSum << ")\n";
3089 }
3090 continue;
3091 } else if (OnlyListBelow)
3092 continue;
3093
3094 if (TopNFunctions || ShowHotFuncList)
3095 NameAndMaxCount.emplace_back(args: Func.Name, args&: FuncMax);
3096
3097 if (Show) {
3098 if (!ShownFunctions)
3099 OS << "Counters:\n";
3100
3101 ++ShownFunctions;
3102
3103 OS << " " << Func.Name << ":\n"
3104 << " Hash: " << format(Fmt: "0x%016" PRIx64, Vals: Func.Hash) << "\n"
3105 << " Counters: " << Func.Counts.size() << "\n";
3106 if (!IsIRInstr)
3107 OS << " Function count: " << Func.Counts[0] << "\n";
3108
3109 if (ShowIndirectCallTargets)
3110 OS << " Indirect Call Site Count: "
3111 << Func.getNumValueSites(ValueKind: IPVK_IndirectCallTarget) << "\n";
3112
3113 if (ShowVTables)
3114 OS << " Number of instrumented vtables: "
3115 << Func.getNumValueSites(ValueKind: IPVK_VTableTarget) << "\n";
3116
3117 uint32_t NumMemOPCalls = Func.getNumValueSites(ValueKind: IPVK_MemOPSize);
3118 if (ShowMemOPSizes && NumMemOPCalls > 0)
3119 OS << " Number of Memory Intrinsics Calls: " << NumMemOPCalls
3120 << "\n";
3121
3122 if (ShowCounts) {
3123 OS << " Block counts: [";
3124 size_t Start = (IsIRInstr ? 0 : 1);
3125 for (size_t I = Start, E = Func.Counts.size(); I < E; ++I) {
3126 OS << (I == Start ? "" : ", ") << Func.Counts[I];
3127 }
3128 OS << "]\n";
3129
3130 // Show uniformity bits if present
3131 if (!Func.UniformityBits.empty()) {
3132 OS << " Block uniformity: [";
3133 for (size_t I = Start, E = Func.Counts.size(); I < E; ++I) {
3134 bool IsUniform = Func.isBlockUniform(BlockIdx: I);
3135 OS << (I == Start ? "" : ", ") << (IsUniform ? "U" : "D");
3136 }
3137 OS << "]\n";
3138 }
3139 }
3140
3141 if (ShowIndirectCallTargets) {
3142 OS << " Indirect Target Results:\n";
3143 traverseAllValueSites(Func, VK: IPVK_IndirectCallTarget,
3144 Stats&: VPStats[IPVK_IndirectCallTarget], OS,
3145 Symtab: &(Reader->getSymtab()));
3146 }
3147
3148 if (ShowVTables) {
3149 OS << " VTable Results:\n";
3150 traverseAllValueSites(Func, VK: IPVK_VTableTarget,
3151 Stats&: VPStats[IPVK_VTableTarget], OS,
3152 Symtab: &(Reader->getSymtab()));
3153 }
3154
3155 if (ShowMemOPSizes && NumMemOPCalls > 0) {
3156 OS << " Memory Intrinsic Size Results:\n";
3157 traverseAllValueSites(Func, VK: IPVK_MemOPSize, Stats&: VPStats[IPVK_MemOPSize], OS,
3158 Symtab: nullptr);
3159 }
3160 }
3161 }
3162 if (Reader->hasError())
3163 return makeError(E: Reader->getError(), Whence: Filename);
3164
3165 if (TextFormat || ShowCovered)
3166 return Error::success();
3167 std::unique_ptr<ProfileSummary> PS(Builder.getSummary());
3168 bool IsIR = Reader->isIRLevelProfile();
3169 OS << "Instrumentation level: " << (IsIR ? "IR" : "Front-end");
3170 if (IsIR) {
3171 OS << " entry_first = " << Reader->instrEntryBBEnabled();
3172 OS << " instrument_loop_entries = " << Reader->instrLoopEntriesEnabled();
3173 }
3174 OS << "\n";
3175 if (ShowAllFunctions || !FuncNameFilter.empty())
3176 OS << "Functions shown: " << ShownFunctions << "\n";
3177 PS->printSummary(OS);
3178 if (ShowValueCutoff > 0) {
3179 OS << "Number of functions with maximum count (< " << ShowValueCutoff
3180 << "): " << BelowCutoffFunctions << "\n";
3181 OS << "Number of functions with maximum count (>= " << ShowValueCutoff
3182 << "): " << PS->getNumFunctions() - BelowCutoffFunctions << "\n";
3183 }
3184
3185 // Sort by MaxCount in decreasing order
3186 llvm::stable_sort(Range&: NameAndMaxCount, C: [](const auto &L, const auto &R) {
3187 return L.second > R.second;
3188 });
3189 if (TopNFunctions) {
3190 OS << "Top " << TopNFunctions
3191 << " functions with the largest internal block counts: \n";
3192 auto TopFuncs = ArrayRef(NameAndMaxCount).take_front(N: TopNFunctions);
3193 for (auto [Name, MaxCount] : TopFuncs)
3194 OS << " " << Name << ", max count = " << MaxCount << "\n";
3195 }
3196
3197 if (ShowHotFuncList) {
3198 auto HotCountThreshold =
3199 ProfileSummaryBuilder::getHotCountThreshold(DS: PS->getDetailedSummary());
3200 OS << "# Hot count threshold: " << HotCountThreshold << "\n";
3201 for (auto [Name, MaxCount] : NameAndMaxCount) {
3202 if (MaxCount < HotCountThreshold)
3203 break;
3204 OS << Name << "\n";
3205 }
3206 }
3207
3208 if (ShownFunctions && ShowIndirectCallTargets) {
3209 OS << "Statistics for indirect call sites profile:\n";
3210 showValueSitesStats(OS, VK: IPVK_IndirectCallTarget,
3211 Stats&: VPStats[IPVK_IndirectCallTarget]);
3212 }
3213
3214 if (ShownFunctions && ShowVTables) {
3215 OS << "Statistics for vtable profile:\n";
3216 showValueSitesStats(OS, VK: IPVK_VTableTarget, Stats&: VPStats[IPVK_VTableTarget]);
3217 }
3218
3219 if (ShownFunctions && ShowMemOPSizes) {
3220 OS << "Statistics for memory intrinsic calls sizes profile:\n";
3221 showValueSitesStats(OS, VK: IPVK_MemOPSize, Stats&: VPStats[IPVK_MemOPSize]);
3222 }
3223
3224 if (ShowDetailedSummary)
3225 PS->printDetailedSummary(OS);
3226
3227 if (ShowBinaryIds)
3228 if (Error E = Reader->printBinaryIds(OS))
3229 return makeError(E: std::move(E), Whence: Filename);
3230
3231 if (ShowProfileVersion)
3232 OS << "Profile version: " << Reader->getVersion() << "\n";
3233
3234 if (ShowTemporalProfTraces) {
3235 auto &Traces = Reader->getTemporalProfTraces();
3236 OS << "Temporal Profile Traces (samples=" << Traces.size()
3237 << " seen=" << Reader->getTemporalProfTraceStreamSize() << "):\n";
3238 for (unsigned i = 0; i < Traces.size(); i++) {
3239 OS << " Temporal Profile Trace " << i << " (weight=" << Traces[i].Weight
3240 << " count=" << Traces[i].FunctionNameRefs.size() << "):\n";
3241 for (auto &NameRef : Traces[i].FunctionNameRefs)
3242 OS << " " << Reader->getSymtab().getFuncOrVarName(MD5Hash: NameRef) << "\n";
3243 }
3244 }
3245
3246 return Error::success();
3247}
3248
3249static void showSectionInfo(sampleprof::SampleProfileReader *Reader,
3250 raw_fd_ostream &OS) {
3251 if (!Reader->dumpSectionInfo(OS)) {
3252 WithColor::warning() << "-show-sec-info-only is only supported for "
3253 << "sample profile in extbinary format and is "
3254 << "ignored for other formats.\n";
3255 return;
3256 }
3257}
3258
3259namespace {
3260struct HotFuncInfo {
3261 std::string FuncName;
3262 uint64_t TotalCount = 0;
3263 double TotalCountPercent = 0.0f;
3264 uint64_t MaxCount = 0;
3265 uint64_t EntryCount = 0;
3266
3267 HotFuncInfo() = default;
3268
3269 HotFuncInfo(StringRef FN, uint64_t TS, double TSP, uint64_t MS, uint64_t ES)
3270 : FuncName(FN.begin(), FN.end()), TotalCount(TS), TotalCountPercent(TSP),
3271 MaxCount(MS), EntryCount(ES) {}
3272};
3273} // namespace
3274
3275// Print out detailed information about hot functions in PrintValues vector.
3276// Users specify titles and offset of every columns through ColumnTitle and
3277// ColumnOffset. The size of ColumnTitle and ColumnOffset need to be the same
3278// and at least 4. Besides, users can optionally give a HotFuncMetric string to
3279// print out or let it be an empty string.
3280static void dumpHotFunctionList(const std::vector<std::string> &ColumnTitle,
3281 const std::vector<int> &ColumnOffset,
3282 const std::vector<HotFuncInfo> &PrintValues,
3283 uint64_t HotFuncCount, uint64_t TotalFuncCount,
3284 uint64_t HotProfCount, uint64_t TotalProfCount,
3285 const std::string &HotFuncMetric,
3286 uint32_t TopNFunctions, raw_fd_ostream &OS) {
3287 assert(ColumnOffset.size() == ColumnTitle.size() &&
3288 "ColumnOffset and ColumnTitle should have the same size");
3289 assert(ColumnTitle.size() >= 4 &&
3290 "ColumnTitle should have at least 4 elements");
3291 assert(TotalFuncCount > 0 &&
3292 "There should be at least one function in the profile");
3293 double TotalProfPercent = 0;
3294 if (TotalProfCount > 0)
3295 TotalProfPercent = static_cast<double>(HotProfCount) / TotalProfCount * 100;
3296
3297 formatted_raw_ostream FOS(OS);
3298 FOS << HotFuncCount << " out of " << TotalFuncCount
3299 << " functions with profile ("
3300 << format(Fmt: "%.2f%%",
3301 Vals: (static_cast<double>(HotFuncCount) / TotalFuncCount * 100))
3302 << ") are considered hot functions";
3303 if (!HotFuncMetric.empty())
3304 FOS << " (" << HotFuncMetric << ")";
3305 FOS << ".\n";
3306 FOS << HotProfCount << " out of " << TotalProfCount << " profile counts ("
3307 << format(Fmt: "%.2f%%", Vals: TotalProfPercent) << ") are from hot functions.\n";
3308
3309 for (size_t I = 0; I < ColumnTitle.size(); ++I) {
3310 FOS.PadToColumn(NewCol: ColumnOffset[I]);
3311 FOS << ColumnTitle[I];
3312 }
3313 FOS << "\n";
3314
3315 uint32_t Count = 0;
3316 for (const auto &R : PrintValues) {
3317 if (TopNFunctions && (Count++ == TopNFunctions))
3318 break;
3319 FOS.PadToColumn(NewCol: ColumnOffset[0]);
3320 FOS << R.TotalCount << " (" << format(Fmt: "%.2f%%", Vals: R.TotalCountPercent) << ")";
3321 FOS.PadToColumn(NewCol: ColumnOffset[1]);
3322 FOS << R.MaxCount;
3323 FOS.PadToColumn(NewCol: ColumnOffset[2]);
3324 FOS << R.EntryCount;
3325 FOS.PadToColumn(NewCol: ColumnOffset[3]);
3326 FOS << R.FuncName << "\n";
3327 }
3328}
3329
3330static int showHotFunctionList(const sampleprof::SampleProfileMap &Profiles,
3331 ProfileSummary &PS, uint32_t TopN,
3332 raw_fd_ostream &OS) {
3333 using namespace sampleprof;
3334
3335 const uint32_t HotFuncCutoff = 990000;
3336 auto &SummaryVector = PS.getDetailedSummary();
3337 uint64_t MinCountThreshold = 0;
3338 for (const ProfileSummaryEntry &SummaryEntry : SummaryVector) {
3339 if (SummaryEntry.Cutoff == HotFuncCutoff) {
3340 MinCountThreshold = SummaryEntry.MinCount;
3341 break;
3342 }
3343 }
3344
3345 // Traverse all functions in the profile and keep only hot functions.
3346 // The following loop also calculates the sum of total samples of all
3347 // functions.
3348 std::multimap<uint64_t, std::pair<const FunctionSamples *, const uint64_t>,
3349 std::greater<uint64_t>>
3350 HotFunc;
3351 uint64_t ProfileTotalSample = 0;
3352 uint64_t HotFuncSample = 0;
3353 uint64_t HotFuncCount = 0;
3354
3355 for (const auto &I : Profiles) {
3356 FuncSampleStats FuncStats;
3357 const FunctionSamples &FuncProf = I.second;
3358 ProfileTotalSample += FuncProf.getTotalSamples();
3359 getFuncSampleStats(Func: FuncProf, FuncStats, HotThreshold: MinCountThreshold);
3360
3361 if (isFunctionHot(FuncStats, HotThreshold: MinCountThreshold)) {
3362 HotFunc.emplace(args: FuncProf.getTotalSamples(),
3363 args: std::make_pair(x: &(I.second), y&: FuncStats.MaxSample));
3364 HotFuncSample += FuncProf.getTotalSamples();
3365 ++HotFuncCount;
3366 }
3367 }
3368
3369 std::vector<std::string> ColumnTitle{"Total sample (%)", "Max sample",
3370 "Entry sample", "Function name"};
3371 std::vector<int> ColumnOffset{0, 24, 42, 58};
3372 std::string Metric =
3373 std::string("max sample >= ") + std::to_string(val: MinCountThreshold);
3374 std::vector<HotFuncInfo> PrintValues;
3375 for (const auto &FuncPair : HotFunc) {
3376 const FunctionSamples &Func = *FuncPair.second.first;
3377 double TotalSamplePercent =
3378 (ProfileTotalSample > 0)
3379 ? (Func.getTotalSamples() * 100.0) / ProfileTotalSample
3380 : 0;
3381 PrintValues.emplace_back(
3382 args: HotFuncInfo(Func.getContext().toString(), Func.getTotalSamples(),
3383 TotalSamplePercent, FuncPair.second.second,
3384 Func.getHeadSamplesEstimate()));
3385 }
3386 dumpHotFunctionList(ColumnTitle, ColumnOffset, PrintValues, HotFuncCount,
3387 TotalFuncCount: Profiles.size(), HotProfCount: HotFuncSample, TotalProfCount: ProfileTotalSample,
3388 HotFuncMetric: Metric, TopNFunctions: TopN, OS);
3389
3390 return 0;
3391}
3392
3393static Error showSampleProfile(ShowFormat SFormat, raw_fd_ostream &OS) {
3394 if (SFormat == ShowFormat::Yaml)
3395 return makeError(Message: "YAML output is not supported for sample profiles");
3396 if (ShowSectionInfoOnly && ShowCompositeInfoOnly)
3397 return makeError(Message: "-show-sec-info-only and "
3398 "-show-composite-info-only cannot be used together");
3399
3400 using namespace sampleprof;
3401 LLVMContext Context;
3402 auto FS = vfs::getRealFileSystem();
3403 auto ReaderOrErr = SampleProfileReader::create(Filename, C&: Context, FS&: *FS,
3404 P: FSDiscriminatorPassOption);
3405 if (std::error_code EC = ReaderOrErr.getError())
3406 return makeError(EC, Whence: Filename);
3407
3408 auto Reader = std::move(ReaderOrErr.get());
3409 if (ShowSectionInfoOnly) {
3410 showSectionInfo(Reader: Reader.get(), OS);
3411 return Error::success();
3412 }
3413
3414 if (ShowCompositeInfoOnly) {
3415 if (!Reader->hasCompositeProfileSection()) {
3416 WithColor::warning() << "no composite profile section; nothing to show\n";
3417 return Error::success();
3418 }
3419 if (std::error_code EC = Reader->dumpProfileTypeInfo(OS)) {
3420 OS.flush();
3421 return makeError(EC, Whence: Filename);
3422 }
3423 return Error::success();
3424 }
3425
3426 if (std::error_code EC = Reader->read())
3427 return makeError(EC, Whence: Filename);
3428
3429 if (ShowAllFunctions || FuncNameFilter.empty()) {
3430 if (SFormat == ShowFormat::Json)
3431 Reader->dumpJson(OS);
3432 else
3433 Reader->dump(OS);
3434 } else {
3435 if (SFormat == ShowFormat::Json)
3436 return makeError(
3437 Message: "the JSON format is supported only when all functions are to "
3438 "be printed");
3439
3440 // TODO: parse context string to support filtering by contexts.
3441 FunctionSamples *FS = Reader->getSamplesFor(Fname: StringRef(FuncNameFilter));
3442 Reader->dumpFunctionProfile(FS: FS ? *FS : FunctionSamples(), OS);
3443 }
3444
3445 if (ShowProfileSymbolList) {
3446 std::unique_ptr<sampleprof::ProfileSymbolList> ReaderList =
3447 Reader->getProfileSymbolList();
3448 ReaderList->dump(OS);
3449 }
3450
3451 if (ShowDetailedSummary) {
3452 auto &PS = Reader->getSummary();
3453 PS.printSummary(OS);
3454 PS.printDetailedSummary(OS);
3455 }
3456
3457 if (ShowHotFuncList || TopNFunctions)
3458 showHotFunctionList(Profiles: Reader->getProfiles(), PS&: Reader->getSummary(),
3459 TopN: TopNFunctions, OS);
3460
3461 return Error::success();
3462}
3463
3464static Error showMemProfProfile(ShowFormat SFormat, raw_fd_ostream &OS) {
3465 if (SFormat == ShowFormat::Json)
3466 return makeError(Message: "JSON output is not supported for MemProf");
3467
3468 // Show the raw profile in YAML.
3469 if (memprof::RawMemProfReader::hasFormat(Path: Filename)) {
3470 auto ReaderOr = llvm::memprof::RawMemProfReader::create(
3471 Path: Filename, ProfiledBinary, /*KeepNames=*/KeepName: true);
3472 if (Error E = ReaderOr.takeError()) {
3473 // Since the error can be related to the profile or the binary we do not
3474 // pass whence. Instead additional context is provided where necessary in
3475 // the error message.
3476 return makeError(E: std::move(E), /*Whence*/ "");
3477 }
3478
3479 std::unique_ptr<llvm::memprof::RawMemProfReader> Reader(
3480 ReaderOr.get().release());
3481
3482 Reader->printYAML(OS);
3483 return Error::success();
3484 }
3485
3486 // Show the indexed MemProf profile in YAML.
3487 auto FS = vfs::getRealFileSystem();
3488 auto ReaderOrErr = IndexedInstrProfReader::create(Path: Filename, FS&: *FS);
3489 if (Error E = ReaderOrErr.takeError())
3490 return makeError(E: std::move(E), Whence: Filename);
3491
3492 auto Reader = std::move(ReaderOrErr.get());
3493 memprof::AllMemProfData Data = Reader->getAllMemProfData();
3494
3495 // For v4 and above the summary is serialized in the indexed profile, and can
3496 // be accessed from the reader. Earlier versions build the summary below.
3497 // The summary is emitted as YAML comments at the start of the output.
3498 if (auto *MemProfSum = Reader->getMemProfSummary()) {
3499 MemProfSum->printSummaryYaml(OS);
3500 } else {
3501 memprof::MemProfSummaryBuilder MemProfSumBuilder;
3502 for (auto &Pair : Data.HeapProfileRecords)
3503 MemProfSumBuilder.addRecord(Pair.Record);
3504 MemProfSumBuilder.getSummary()->printSummaryYaml(OS);
3505 }
3506 // Construct yaml::Output with the maximum column width of 80 so that each
3507 // Frame fits in one line.
3508 yaml::Output Yout(OS, nullptr, 80);
3509 Yout << Data;
3510
3511 return Error::success();
3512}
3513
3514static Error showDebugInfoCorrelation(const std::string &Filename,
3515 ShowFormat SFormat, raw_fd_ostream &OS) {
3516 if (SFormat == ShowFormat::Json)
3517 return makeError(Message: "JSON output is not supported for debug info correlation");
3518 std::unique_ptr<InstrProfCorrelator> Correlator;
3519 if (auto Err =
3520 InstrProfCorrelator::get(Filename, FileKind: InstrProfCorrelator::DEBUG_INFO)
3521 .moveInto(Value&: Correlator))
3522 return makeError(E: std::move(Err), Whence: Filename);
3523 if (SFormat == ShowFormat::Yaml) {
3524 if (auto Err = Correlator->dumpYaml(MaxWarnings: MaxDbgCorrelationWarnings, OS))
3525 return makeError(E: std::move(Err), Whence: Filename);
3526 return Error::success();
3527 }
3528
3529 if (auto Err = Correlator->correlateProfileData(MaxWarnings: MaxDbgCorrelationWarnings))
3530 return makeError(E: std::move(Err), Whence: Filename);
3531
3532 InstrProfSymtab Symtab;
3533 if (auto Err = Symtab.create(
3534 NameStrings: StringRef(Correlator->getNamesPointer(), Correlator->getNamesSize())))
3535 return makeError(E: std::move(Err), Whence: Filename);
3536
3537 if (ShowProfileSymbolList)
3538 Symtab.dumpNames(OS);
3539 // TODO: Read "Profile Data Type" from debug info to compute and show how many
3540 // counters the section holds.
3541 if (ShowDetailedSummary)
3542 OS << "Counters section size: 0x"
3543 << Twine::utohexstr(Val: Correlator->getCountersSectionSize()) << " bytes\n";
3544 OS << "Found " << Correlator->getDataSize() << " functions\n";
3545
3546 return Error::success();
3547}
3548
3549static Error show_main(StringRef ProgName) {
3550 if (Filename.empty() && DebugInfoFilename.empty())
3551 return makeError(
3552 Message: "the positional argument '<profdata-file>' is required unless '--" +
3553 DebugInfoFilename.ArgStr + "' is provided");
3554
3555 if (Filename == OutputFilename)
3556 return makeError(
3557 Message: "Input file name cannot be the same as the output file name!",
3558 Whence: (ProgName + " show").str());
3559 if (JsonFormat)
3560 SFormat = ShowFormat::Json;
3561
3562 std::error_code EC;
3563 raw_fd_ostream OS(OutputFilename.data(), EC, sys::fs::OF_TextWithCRLF);
3564 if (EC)
3565 return makeError(EC, Whence: OutputFilename);
3566
3567 if (ShowAllFunctions && !FuncNameFilter.empty())
3568 WithColor::warning() << "-function argument ignored: showing all functions\n";
3569
3570 if (!DebugInfoFilename.empty())
3571 return showDebugInfoCorrelation(Filename: DebugInfoFilename, SFormat, OS);
3572
3573 if (ShowProfileKind == instr)
3574 return showInstrProfile(SFormat, OS);
3575 if (ShowProfileKind == sample)
3576 return showSampleProfile(SFormat, OS);
3577 return showMemProfProfile(SFormat, OS);
3578}
3579
3580static Error order_main() {
3581 std::error_code EC;
3582 raw_fd_ostream OS(OutputFilename.data(), EC, sys::fs::OF_TextWithCRLF);
3583 if (EC)
3584 return makeError(EC, Whence: OutputFilename);
3585 auto FS = vfs::getRealFileSystem();
3586 auto ReaderOrErr = InstrProfReader::create(Path: Filename, FS&: *FS);
3587 if (Error E = ReaderOrErr.takeError())
3588 return makeError(E: std::move(E), Whence: Filename);
3589
3590 auto Reader = std::move(ReaderOrErr.get());
3591 for (auto &I : *Reader) {
3592 // Read all entries
3593 (void)I;
3594 }
3595 ArrayRef Traces = Reader->getTemporalProfTraces();
3596 if (NumTestTraces && NumTestTraces >= Traces.size())
3597 return makeError(
3598 Message: "--" + NumTestTraces.ArgStr +
3599 " must be smaller than the total number of traces: expected: < " +
3600 Twine(Traces.size()) + ", actual: " + Twine(NumTestTraces));
3601 ArrayRef TestTraces = Traces.take_back(N: NumTestTraces);
3602 Traces = Traces.drop_back(N: NumTestTraces);
3603
3604 std::vector<BPFunctionNode> Nodes;
3605 TemporalProfTraceTy::createBPFunctionNodes(Traces, Nodes);
3606 BalancedPartitioningConfig Config;
3607 BalancedPartitioning BP(Config);
3608 BP.run(Nodes);
3609
3610 OS << "# Ordered " << Nodes.size() << " functions\n";
3611 if (!TestTraces.empty()) {
3612 // Since we don't know the symbol sizes, we assume 32 functions per page.
3613 DenseMap<BPFunctionNode::IDT, unsigned> IdToPageNumber;
3614 for (auto &Node : Nodes)
3615 IdToPageNumber[Node.Id] = IdToPageNumber.size() / 32;
3616
3617 SmallSet<unsigned, 0> TouchedPages;
3618 unsigned Area = 0;
3619 for (auto &Trace : TestTraces) {
3620 for (auto Id : Trace.FunctionNameRefs) {
3621 auto It = IdToPageNumber.find(Val: Id);
3622 if (It == IdToPageNumber.end())
3623 continue;
3624 TouchedPages.insert(V: It->getSecond());
3625 Area += TouchedPages.size();
3626 }
3627 TouchedPages.clear();
3628 }
3629 OS << "# Total area under the page fault curve: " << (float)Area << "\n";
3630 }
3631 OS << "# Warning: Mach-O may prefix symbols with \"_\" depending on the "
3632 "linkage and this output does not take that into account. Some "
3633 "post-processing may be required before passing to the linker via "
3634 "-order_file.\n";
3635 for (auto &N : Nodes) {
3636 auto [Filename, ParsedFuncName] =
3637 getParsedIRPGOName(IRPGOName: Reader->getSymtab().getFuncOrVarName(MD5Hash: N.Id));
3638 if (!Filename.empty())
3639 OS << "# " << Filename << "\n";
3640 OS << ParsedFuncName << "\n";
3641 }
3642 return Error::success();
3643}
3644
3645int main(int argc, const char *argv[]) {
3646 InitLLVM X(argc, argv);
3647 StringRef ProgName(sys::path::filename(path: argv[0]));
3648
3649 if (argc < 2) {
3650 errs()
3651 << ProgName
3652 << ": No subcommand specified! Run llvm-profdata --help for usage.\n";
3653 return 1;
3654 }
3655
3656 cl::ParseCommandLineOptions(argc, argv, Overview: "LLVM profile data\n");
3657
3658 if (ShowSubcommand)
3659 return reportError(E: show_main(ProgName));
3660
3661 if (OrderSubcommand)
3662 return reportError(E: order_main());
3663
3664 if (OverlapSubcommand)
3665 return reportError(E: overlap_main());
3666
3667 if (MergeSubcommand)
3668 return reportError(E: merge_main(ProgName));
3669
3670 errs() << ProgName
3671 << ": Unknown command. Run llvm-profdata --help for usage.\n";
3672 return 1;
3673}
3674