1//===- llvm-profdata.cpp - LLVM profile data tool -------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// llvm-profdata merges .profdata files.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/ADT/DenseMap.h"
14#include "llvm/ADT/ScopeExit.h"
15#include "llvm/ADT/SmallSet.h"
16#include "llvm/ADT/SmallVector.h"
17#include "llvm/ADT/StringRef.h"
18#include "llvm/HTTP/HTTPClient.h"
19#include "llvm/IR/LLVMContext.h"
20#include "llvm/Object/Binary.h"
21#include "llvm/ProfileData/DataAccessProf.h"
22#include "llvm/ProfileData/InstrProfCorrelator.h"
23#include "llvm/ProfileData/InstrProfReader.h"
24#include "llvm/ProfileData/InstrProfWriter.h"
25#include "llvm/ProfileData/MemProf.h"
26#include "llvm/ProfileData/MemProfReader.h"
27#include "llvm/ProfileData/MemProfSummaryBuilder.h"
28#include "llvm/ProfileData/MemProfYAML.h"
29#include "llvm/ProfileData/ProfileCommon.h"
30#include "llvm/ProfileData/SampleProfReader.h"
31#include "llvm/ProfileData/SampleProfWriter.h"
32#include "llvm/Support/BalancedPartitioning.h"
33#include "llvm/Support/CommandLine.h"
34#include "llvm/Support/Discriminator.h"
35#include "llvm/Support/Errc.h"
36#include "llvm/Support/FileSystem.h"
37#include "llvm/Support/Format.h"
38#include "llvm/Support/FormattedStream.h"
39#include "llvm/Support/InitLLVM.h"
40#include "llvm/Support/MD5.h"
41#include "llvm/Support/MemoryBuffer.h"
42#include "llvm/Support/Path.h"
43#include "llvm/Support/Regex.h"
44#include "llvm/Support/ThreadPool.h"
45#include "llvm/Support/Threading.h"
46#include "llvm/Support/VirtualFileSystem.h"
47#include "llvm/Support/WithColor.h"
48#include "llvm/Support/raw_ostream.h"
49#include <algorithm>
50#include <cmath>
51#include <optional>
52
53#if LLVM_ADDRESS_SANITIZER_BUILD || LLVM_HWADDRESS_SANITIZER_BUILD
54#include <sanitizer/lsan_interface.h>
55static int SkipLeakCheck;
56LLVM_ATTRIBUTE_USED int __lsan_is_turned_off() { return SkipLeakCheck; }
57static void skipLeakCheck() { SkipLeakCheck = 1; }
58#else
59static void skipLeakCheck() {}
60#endif
61
62using namespace llvm;
63using ProfCorrelatorKind = InstrProfCorrelator::ProfCorrelatorKind;
64
65// https://llvm.org/docs/CommandGuide/llvm-profdata.html has documentations
66// on each subcommand.
67cl::SubCommand ShowSubcommand(
68 "show",
69 "Takes a profile data file and displays the profiles. See detailed "
70 "documentation in "
71 "https://llvm.org/docs/CommandGuide/llvm-profdata.html#profdata-show");
72cl::SubCommand OrderSubcommand(
73 "order",
74 "Reads temporal profiling traces from a profile and outputs a function "
75 "order that reduces the number of page faults for those traces. See "
76 "detailed documentation in "
77 "https://llvm.org/docs/CommandGuide/llvm-profdata.html#profdata-order");
78cl::SubCommand OverlapSubcommand(
79 "overlap",
80 "Computes and displays the overlap between two profiles. See detailed "
81 "documentation in "
82 "https://llvm.org/docs/CommandGuide/llvm-profdata.html#profdata-overlap");
83cl::SubCommand MergeSubcommand(
84 "merge",
85 "Takes several profiles and merge them together. See detailed "
86 "documentation in "
87 "https://llvm.org/docs/CommandGuide/llvm-profdata.html#profdata-merge");
88
89namespace {
90enum ProfileKinds { instr, sample, memory };
91enum FailureMode { warnOnly, failIfAnyAreInvalid, failIfAllAreInvalid };
92
93enum ProfileFormat {
94 PF_None = 0,
95 PF_Text,
96 PF_Compact_Binary, // Deprecated
97 PF_Ext_Binary,
98 PF_GCC,
99 PF_Binary
100};
101
102enum class ShowFormat { Text, Json, Yaml };
103} // namespace
104
105// Common options.
106cl::opt<std::string> OutputFilename("output", cl::value_desc("output"),
107 cl::init(Val: "-"), cl::desc("Output file"),
108 cl::sub(ShowSubcommand),
109 cl::sub(OrderSubcommand),
110 cl::sub(OverlapSubcommand),
111 cl::sub(MergeSubcommand));
112// NOTE: cl::alias must not have cl::sub(), since aliased option's cl::sub()
113// will be used. llvm::cl::alias::done() method asserts this condition.
114static cl::alias OutputFilenameA("o", cl::desc("Alias for --output"),
115 cl::aliasopt(OutputFilename));
116
117// Options common to at least two commands.
118static cl::opt<ProfileKinds> ProfileKind(
119 cl::desc("Profile kind:"), cl::sub(MergeSubcommand),
120 cl::sub(OverlapSubcommand), cl::init(Val: instr),
121 cl::values(clEnumVal(instr, "Instrumentation profile (default)"),
122 clEnumVal(sample, "Sample profile")));
123static cl::opt<std::string> Filename(cl::Positional,
124 cl::desc("<profdata-file>"),
125 cl::sub(ShowSubcommand),
126 cl::sub(OrderSubcommand));
127static cl::opt<unsigned> MaxDbgCorrelationWarnings(
128 "max-debug-info-correlation-warnings",
129 cl::desc("The maximum number of warnings to emit when correlating "
130 "profile from debug info (0 = no limit)"),
131 cl::sub(MergeSubcommand), cl::sub(ShowSubcommand), cl::init(Val: 5));
132static cl::opt<std::string> ProfiledBinary(
133 "profiled-binary", cl::init(Val: ""),
134 cl::desc("Path to binary from which the profile was collected."),
135 cl::sub(ShowSubcommand), cl::sub(MergeSubcommand));
136static cl::opt<std::string> DebugInfoFilename(
137 "debug-info", cl::init(Val: ""),
138 cl::desc(
139 "For show, read and extract profile metadata from debug info and show "
140 "the functions it found. For merge, use the provided debug info to "
141 "correlate the raw profile."),
142 cl::sub(ShowSubcommand), cl::sub(MergeSubcommand));
143static cl::opt<std::string>
144 BinaryFilename("binary-file", cl::init(Val: ""),
145 cl::desc("For merge, use the provided unstripped binary to "
146 "correlate the raw profile."),
147 cl::sub(MergeSubcommand));
148static cl::list<std::string> DebugFileDirectory(
149 "debug-file-directory",
150 cl::desc("Directories to search for object files by build ID"));
151static cl::opt<bool> DebugInfod("debuginfod", cl::init(Val: false), cl::Hidden,
152 cl::sub(MergeSubcommand),
153 cl::desc("Enable debuginfod"));
154static cl::opt<ProfCorrelatorKind> BIDFetcherProfileCorrelate(
155 "correlate",
156 cl::desc("Use debug-info or binary correlation to correlate profiles with "
157 "build id fetcher"),
158 cl::init(Val: InstrProfCorrelator::NONE),
159 cl::values(clEnumValN(InstrProfCorrelator::NONE, "",
160 "No profile correlation"),
161 clEnumValN(InstrProfCorrelator::DEBUG_INFO, "debug-info",
162 "Use debug info to correlate"),
163 clEnumValN(InstrProfCorrelator::BINARY, "binary",
164 "Use binary to correlate")));
165static cl::opt<std::string> FuncNameFilter(
166 "function",
167 cl::desc("Only functions matching the filter are shown in the output. For "
168 "overlapping CSSPGO, this takes a function name with calling "
169 "context."),
170 cl::sub(ShowSubcommand), cl::sub(OverlapSubcommand),
171 cl::sub(MergeSubcommand));
172
173// TODO: Consider creating a template class (e.g., MergeOption, ShowOption) to
174// factor out the common cl::sub in cl::opt constructor for subcommand-specific
175// options.
176
177// Options specific to merge subcommand.
178static cl::list<std::string> InputFilenames(cl::Positional,
179 cl::sub(MergeSubcommand),
180 cl::desc("<filename...>"));
181static cl::list<std::string>
182 WeightedInputFilenames("weighted-input", cl::sub(MergeSubcommand),
183 cl::desc("<weight>,<filename>"));
184static cl::opt<ProfileFormat> OutputFormat(
185 cl::desc("Format of output profile"), cl::sub(MergeSubcommand),
186 cl::init(Val: PF_Ext_Binary),
187 cl::values(clEnumValN(PF_Binary, "binary", "Binary encoding"),
188 clEnumValN(PF_Ext_Binary, "extbinary",
189 "Extensible binary encoding "
190 "(default)"),
191 clEnumValN(PF_Text, "text", "Text encoding"),
192 clEnumValN(PF_GCC, "gcc",
193 "GCC encoding (only meaningful for -sample)")));
194static cl::opt<std::string>
195 InputFilenamesFile("input-files", cl::init(Val: ""), cl::sub(MergeSubcommand),
196 cl::desc("Path to file containing newline-separated "
197 "[<weight>,]<filename> entries"));
198static cl::alias InputFilenamesFileA("f", cl::desc("Alias for --input-files"),
199 cl::aliasopt(InputFilenamesFile));
200static cl::opt<bool> DumpInputFileList(
201 "dump-input-file-list", cl::init(Val: false), cl::Hidden,
202 cl::sub(MergeSubcommand),
203 cl::desc("Dump the list of input files and their weights, then exit"));
204static cl::opt<std::string> RemappingFile("remapping-file",
205 cl::value_desc("file"),
206 cl::sub(MergeSubcommand),
207 cl::desc("Symbol remapping file"));
208static cl::alias RemappingFileA("r", cl::desc("Alias for --remapping-file"),
209 cl::aliasopt(RemappingFile));
210static cl::opt<bool>
211 UseMD5("use-md5", cl::init(Val: false), cl::Hidden,
212 cl::desc("Choose to use MD5 to represent string in name table (only "
213 "meaningful for -extbinary)"),
214 cl::sub(MergeSubcommand));
215static cl::opt<bool> CompressAllSections(
216 "compress-all-sections", cl::init(Val: false), cl::Hidden,
217 cl::sub(MergeSubcommand),
218 cl::desc("Compress all sections when writing the profile (only "
219 "meaningful for -extbinary)"));
220static cl::opt<bool> SampleMergeColdContext(
221 "sample-merge-cold-context", cl::init(Val: false), cl::Hidden,
222 cl::sub(MergeSubcommand),
223 cl::desc(
224 "Merge context sample profiles whose count is below cold threshold"));
225static cl::opt<bool> SampleTrimColdContext(
226 "sample-trim-cold-context", cl::init(Val: false), cl::Hidden,
227 cl::sub(MergeSubcommand),
228 cl::desc(
229 "Trim context sample profiles whose count is below cold threshold"));
230static cl::opt<uint32_t> SampleColdContextFrameDepth(
231 "sample-frame-depth-for-cold-context", cl::init(Val: 1),
232 cl::sub(MergeSubcommand),
233 cl::desc("Keep the last K frames while merging cold profile. 1 means the "
234 "context-less base profile"));
235static cl::opt<size_t> OutputSizeLimit(
236 "output-size-limit", cl::init(Val: 0), cl::Hidden, cl::sub(MergeSubcommand),
237 cl::desc("Trim cold functions until profile size is below specified "
238 "limit in bytes. This uses a heursitic and functions may be "
239 "excessively trimmed"));
240static cl::opt<bool> GenPartialProfile(
241 "gen-partial-profile", cl::init(Val: false), cl::Hidden,
242 cl::sub(MergeSubcommand),
243 cl::desc("Generate a partial profile (only meaningful for -extbinary)"));
244static cl::opt<bool> SplitLayout(
245 "split-layout", cl::init(Val: false), cl::Hidden, cl::sub(MergeSubcommand),
246 cl::desc("Split the profile to two sections with one containing sample "
247 "profiles with inlined functions and the other without (only "
248 "meaningful for -extbinary)"));
249static cl::opt<bool>
250 WriteMD5ProfSymList("md5-prof-sym-list", cl::init(Val: false), cl::Hidden,
251 cl::sub(MergeSubcommand),
252 cl::desc("Write ProfileSymbolList (Cold Symbols) as "
253 "64-bit MD5 hashes in Eytzinger layout"));
254static cl::opt<bool> WriteMD5IndexedTables(
255 "md5-indexed-tables", cl::init(Val: false), cl::Hidden, cl::sub(MergeSubcommand),
256 cl::desc("Write MD5-based indexed NameTable and parallel "
257 "FuncOffsetTable in Eytzinger layout (only meaningful for "
258 "-extbinary)"));
259static cl::opt<std::string> SupplInstrWithSample(
260 "supplement-instr-with-sample", cl::init(Val: ""), cl::Hidden,
261 cl::sub(MergeSubcommand),
262 cl::desc("Supplement an instr profile with sample profile, to correct "
263 "the profile unrepresentativeness issue. The sample "
264 "profile is the input of the flag. Output will be in instr "
265 "format (The flag only works with -instr)"));
266static cl::opt<float> ZeroCounterThreshold(
267 "zero-counter-threshold", cl::init(Val: 0.7), cl::Hidden,
268 cl::sub(MergeSubcommand),
269 cl::desc("For the function which is cold in instr profile but hot in "
270 "sample profile, if the ratio of the number of zero counters "
271 "divided by the total number of counters is above the "
272 "threshold, the profile of the function will be regarded as "
273 "being harmful for performance and will be dropped."));
274static cl::opt<unsigned> SupplMinSizeThreshold(
275 "suppl-min-size-threshold", cl::init(Val: 10), cl::Hidden,
276 cl::sub(MergeSubcommand),
277 cl::desc("If the size of a function is smaller than the threshold, "
278 "assume it can be inlined by PGO early inliner and it won't "
279 "be adjusted based on sample profile."));
280static cl::opt<unsigned> InstrProfColdThreshold(
281 "instr-prof-cold-threshold", cl::init(Val: 0), cl::Hidden,
282 cl::sub(MergeSubcommand),
283 cl::desc("User specified cold threshold for instr profile which will "
284 "override the cold threshold got from profile summary. "));
285// WARNING: This reservoir size value is propagated to any input indexed
286// profiles for simplicity. Changing this value between invocations could
287// result in sample bias.
288static cl::opt<uint64_t> TemporalProfTraceReservoirSize(
289 "temporal-profile-trace-reservoir-size", cl::init(Val: 100),
290 cl::sub(MergeSubcommand),
291 cl::desc("The maximum number of stored temporal profile traces (default: "
292 "100)"));
293static cl::opt<uint64_t> TemporalProfMaxTraceLength(
294 "temporal-profile-max-trace-length", cl::init(Val: 10000),
295 cl::sub(MergeSubcommand),
296 cl::desc("The maximum length of a single temporal profile trace "
297 "(default: 10000)"));
298static cl::opt<std::string> FuncNameNegativeFilter(
299 "no-function", cl::init(Val: ""), cl::sub(MergeSubcommand),
300 cl::desc("Exclude functions matching the filter from the output."));
301
302static cl::opt<FailureMode>
303 FailMode("failure-mode", cl::init(Val: failIfAnyAreInvalid),
304 cl::desc("Failure mode:"), cl::sub(MergeSubcommand),
305 cl::values(clEnumValN(warnOnly, "warn",
306 "Do not fail and just print warnings."),
307 clEnumValN(failIfAnyAreInvalid, "any",
308 "Fail if any profile is invalid."),
309 clEnumValN(failIfAllAreInvalid, "all",
310 "Fail only if all profiles are invalid.")));
311
312static cl::opt<bool> OutputSparse(
313 "sparse", cl::init(Val: false), cl::sub(MergeSubcommand),
314 cl::desc("Generate a sparse profile (only meaningful for -instr)"));
315static cl::opt<unsigned> NumThreads(
316 "num-threads", cl::init(Val: 0), cl::sub(MergeSubcommand),
317 cl::desc("Number of merge threads to use (default: autodetect)"));
318static cl::alias NumThreadsA("j", cl::desc("Alias for --num-threads"),
319 cl::aliasopt(NumThreads));
320
321static cl::opt<std::string> ProfileSymbolListFile(
322 "prof-sym-list", cl::init(Val: ""), cl::sub(MergeSubcommand),
323 cl::desc("Path to file containing the list of function symbols "
324 "used to populate profile symbol list"));
325
326static cl::opt<SampleProfileLayout> ProfileLayout(
327 "convert-sample-profile-layout",
328 cl::desc("Convert the generated profile to a profile with a new layout"),
329 cl::sub(MergeSubcommand), cl::init(Val: SPL_None),
330 cl::values(
331 clEnumValN(SPL_Nest, "nest",
332 "Nested profile, the input should be CS flat profile"),
333 clEnumValN(SPL_Flat, "flat",
334 "Profile with nested inlinee flatten out")));
335
336static cl::opt<bool> DropProfileSymbolList(
337 "drop-profile-symbol-list", cl::init(Val: false), cl::Hidden,
338 cl::sub(MergeSubcommand),
339 cl::desc("Drop the profile symbol list when merging AutoFDO profiles "
340 "(only meaningful for -sample)"));
341
342static cl::opt<bool> KeepVTableSymbols(
343 "keep-vtable-symbols", cl::init(Val: false), cl::Hidden,
344 cl::sub(MergeSubcommand),
345 cl::desc("If true, keep the vtable symbols in indexed profiles"));
346
347// Temporary support for writing the previous version of the format, to enable
348// some forward compatibility.
349// TODO: Consider enabling this with future version changes as well, to ease
350// deployment of newer versions of llvm-profdata.
351static cl::opt<bool> DoWritePrevVersion(
352 "write-prev-version", cl::init(Val: false), cl::Hidden,
353 cl::desc("Write the previous version of indexed format, to enable "
354 "some forward compatibility."));
355
356static cl::opt<memprof::IndexedVersion> MemProfVersionRequested(
357 "memprof-version", cl::Hidden, cl::sub(MergeSubcommand),
358 cl::desc("Specify the version of the memprof format to use"),
359 cl::init(Val: memprof::Version3),
360 cl::values(clEnumValN(memprof::Version2, "2", "version 2"),
361 clEnumValN(memprof::Version3, "3", "version 3"),
362 clEnumValN(memprof::Version4, "4", "version 4")));
363
364static cl::opt<bool> MemProfFullSchema(
365 "memprof-full-schema", cl::Hidden, cl::sub(MergeSubcommand),
366 cl::desc("Use the full schema for serialization"), cl::init(Val: false));
367
368static cl::opt<bool> MemprofGenerateRandomHotness(
369 "memprof-random-hotness", cl::init(Val: false), cl::Hidden,
370 cl::sub(MergeSubcommand),
371 cl::desc("Generate random hotness values. Use -random-seed to set the seed "
372 "value, otherwise the constant default seed is used"));
373static cl::opt<unsigned>
374 RandomSeed("random-seed", cl::init(Val: 0), cl::Hidden, cl::sub(MergeSubcommand),
375 cl::desc("Seed for the random number generator used by "
376 "-memprof-random-hotness and temporal profile "
377 "reservoir sampling"));
378static cl::alias MemprofGenerateRandomHotnessSeed(
379 "memprof-random-hotness-seed", cl::Hidden,
380 cl::desc("Alias for -random-seed. Deprecated, please use -random-seed"),
381 cl::aliasopt(RandomSeed));
382
383// Options specific to overlap subcommand.
384static cl::opt<std::string> BaseFilename(cl::Positional, cl::Required,
385 cl::desc("<base profile file>"),
386 cl::sub(OverlapSubcommand));
387static cl::opt<std::string> TestFilename(cl::Positional, cl::Required,
388 cl::desc("<test profile file>"),
389 cl::sub(OverlapSubcommand));
390
391static cl::opt<unsigned long long> SimilarityCutoff(
392 "similarity-cutoff", cl::init(Val: 0),
393 cl::desc("For sample profiles, list function names (with calling context "
394 "for csspgo) for overlapped functions "
395 "with similarities below the cutoff (percentage times 10000)."),
396 cl::sub(OverlapSubcommand));
397
398static cl::opt<bool> IsCS(
399 "cs", cl::init(Val: false),
400 cl::desc("For context sensitive PGO counts. Does not work with CSSPGO."),
401 cl::sub(OverlapSubcommand));
402
403static cl::opt<unsigned long long> OverlapValueCutoff(
404 "value-cutoff", cl::init(Val: -1),
405 cl::desc(
406 "Function level overlap information for every function (with calling "
407 "context for csspgo) in test "
408 "profile with max count value greater than the parameter value"),
409 cl::sub(OverlapSubcommand));
410
411// Options specific to show subcommand.
412static cl::opt<bool>
413 ShowCounts("counts", cl::init(Val: false),
414 cl::desc("Show counter values for shown functions"),
415 cl::sub(ShowSubcommand));
416static cl::opt<ShowFormat>
417 SFormat("show-format", cl::init(Val: ShowFormat::Text),
418 cl::desc("Emit output in the selected format if supported"),
419 cl::sub(ShowSubcommand),
420 cl::values(clEnumValN(ShowFormat::Text, "text",
421 "emit normal text output (default)"),
422 clEnumValN(ShowFormat::Json, "json", "emit JSON"),
423 clEnumValN(ShowFormat::Yaml, "yaml", "emit YAML")));
424// TODO: Consider replacing this with `--show-format=text-encoding`.
425static cl::opt<bool>
426 TextFormat("text", cl::init(Val: false),
427 cl::desc("Show instr profile data in text dump format"),
428 cl::sub(ShowSubcommand));
429static cl::opt<bool>
430 JsonFormat("json",
431 cl::desc("Show sample profile data in the JSON format "
432 "(deprecated, please use --show-format=json)"),
433 cl::sub(ShowSubcommand));
434static cl::opt<bool> ShowIndirectCallTargets(
435 "ic-targets", cl::init(Val: false),
436 cl::desc("Show indirect call site target values for shown functions"),
437 cl::sub(ShowSubcommand));
438static cl::opt<bool>
439 ShowVTables("show-vtables", cl::init(Val: false),
440 cl::desc("Show vtable names for shown functions"),
441 cl::sub(ShowSubcommand));
442static cl::opt<bool> ShowMemOPSizes(
443 "memop-sizes", cl::init(Val: false),
444 cl::desc("Show the profiled sizes of the memory intrinsic calls "
445 "for shown functions"),
446 cl::sub(ShowSubcommand));
447static cl::opt<bool>
448 ShowDetailedSummary("detailed-summary", cl::init(Val: false),
449 cl::desc("Show detailed profile summary"),
450 cl::sub(ShowSubcommand));
451static cl::list<uint32_t> DetailedSummaryCutoffs(
452 cl::CommaSeparated, "detailed-summary-cutoffs",
453 cl::desc(
454 "Cutoff percentages (times 10000) for generating detailed summary"),
455 cl::value_desc("800000,901000,999999"), cl::sub(ShowSubcommand));
456static cl::opt<bool>
457 ShowHotFuncList("hot-func-list", cl::init(Val: false),
458 cl::desc("Show profile summary of a list of hot functions"),
459 cl::sub(ShowSubcommand));
460static cl::opt<bool>
461 ShowAllFunctions("all-functions", cl::init(Val: false),
462 cl::desc("Details for each and every function"),
463 cl::sub(ShowSubcommand));
464static cl::opt<bool> ShowCS("showcs", cl::init(Val: false),
465 cl::desc("Show context sensitive counts"),
466 cl::sub(ShowSubcommand));
467static cl::opt<ProfileKinds> ShowProfileKind(
468 cl::desc("Profile kind supported by show:"), cl::sub(ShowSubcommand),
469 cl::init(Val: instr),
470 cl::values(clEnumVal(instr, "Instrumentation profile (default)"),
471 clEnumVal(sample, "Sample profile"),
472 clEnumVal(memory, "MemProf memory access profile")));
473static cl::opt<uint32_t> TopNFunctions(
474 "topn", cl::init(Val: 0),
475 cl::desc("Show the list of functions with the largest internal counts"),
476 cl::sub(ShowSubcommand));
477static cl::opt<uint32_t> ShowValueCutoff(
478 "value-cutoff", cl::init(Val: 0),
479 cl::desc("Set the count value cutoff. Functions with the maximum count "
480 "less than this value will not be printed out. (Default is 0)"),
481 cl::sub(ShowSubcommand));
482static cl::opt<bool> OnlyListBelow(
483 "list-below-cutoff", cl::init(Val: false),
484 cl::desc("Only output names of functions whose max count values are "
485 "below the cutoff value"),
486 cl::sub(ShowSubcommand));
487static cl::opt<bool> ShowProfileSymbolList(
488 "show-prof-sym-list", cl::init(Val: false),
489 cl::desc("Show profile symbol list if it exists in the profile. "),
490 cl::sub(ShowSubcommand));
491static cl::opt<bool> ShowSectionInfoOnly(
492 "show-sec-info-only", cl::init(Val: false),
493 cl::desc("Show the information of each section in the sample profile. "
494 "The flag is only usable when the sample profile is in "
495 "extbinary format"),
496 cl::sub(ShowSubcommand));
497static cl::opt<bool> ShowCompositeInfoOnly(
498 "show-composite-info-only", cl::init(Val: false),
499 cl::desc("Show type IDs and payload sizes in a composite sample profile"),
500 cl::sub(ShowSubcommand));
501static cl::opt<bool> ShowBinaryIds("binary-ids", cl::init(Val: false),
502 cl::desc("Show binary ids in the profile. "),
503 cl::sub(ShowSubcommand));
504static cl::opt<bool> ShowTemporalProfTraces(
505 "temporal-profile-traces",
506 cl::desc("Show temporal profile traces in the profile."),
507 cl::sub(ShowSubcommand));
508
509static cl::opt<bool>
510 ShowCovered("covered", cl::init(Val: false),
511 cl::desc("Show only the functions that have been executed."),
512 cl::sub(ShowSubcommand));
513
514static cl::opt<bool> ShowProfileVersion("profile-version", cl::init(Val: false),
515 cl::desc("Show profile version. "),
516 cl::sub(ShowSubcommand));
517
518// Options specific to order subcommand.
519static cl::opt<unsigned>
520 NumTestTraces("num-test-traces", cl::init(Val: 0),
521 cl::desc("Keep aside the last <num-test-traces> traces in "
522 "the profile when computing the function order and "
523 "instead use them to evaluate that order"),
524 cl::sub(OrderSubcommand));
525
526// We use this string to indicate that there are
527// multiple static functions map to the same name.
528const std::string DuplicateNameStr = "----";
529
530static void warn(Twine Message, StringRef Whence = "", StringRef Hint = "") {
531 WithColor::warning();
532 if (!Whence.empty())
533 errs() << Whence << ": ";
534 errs() << Message << "\n";
535 if (!Hint.empty())
536 WithColor::note() << Hint << "\n";
537}
538
539static void warn(Error E, StringRef Whence = "") {
540 if (E.isA<InstrProfError>()) {
541 handleAllErrors(E: std::move(E), Handlers: [&](const InstrProfError &IPE) {
542 warn(Message: IPE.message(), Whence);
543 });
544 }
545}
546
547static void exitWithError(Twine Message, StringRef Whence = "",
548 StringRef Hint = "") {
549 WithColor::error();
550 if (!Whence.empty())
551 errs() << Whence << ": ";
552 errs() << Message << "\n";
553 if (!Hint.empty())
554 WithColor::note() << Hint << "\n";
555 // exit() terminates without unwinding the stack or running destructors, and
556 // there is no guaranty that pointers to allocations will be preserved, so
557 // LSan reports in-flight heap allocations as leaks at atexit.
558 skipLeakCheck();
559 ::exit(status: 1);
560}
561
562static void exitWithError(Error E, StringRef Whence = "") {
563 if (E.isA<InstrProfError>()) {
564 handleAllErrors(E: std::move(E), Handlers: [&](const InstrProfError &IPE) {
565 instrprof_error instrError = IPE.get();
566 StringRef Hint = "";
567 if (instrError == instrprof_error::unrecognized_format) {
568 // Hint in case user missed specifying the profile type.
569 Hint = "Perhaps you forgot to use the --sample or --memory option?";
570 }
571 exitWithError(Message: IPE.message(), Whence, Hint);
572 });
573 return;
574 }
575
576 exitWithError(Message: toString(E: std::move(E)), Whence);
577}
578
579static void exitWithErrorCode(std::error_code EC, StringRef Whence = "") {
580 exitWithError(Message: EC.message(), Whence);
581}
582
583static void warnOrExitGivenError(FailureMode FailMode, std::error_code EC,
584 StringRef Whence = "") {
585 if (FailMode == failIfAnyAreInvalid)
586 exitWithErrorCode(EC, Whence);
587 else
588 warn(Message: EC.message(), Whence);
589}
590
591static void handleMergeWriterError(Error E, StringRef WhenceFile = "",
592 StringRef WhenceFunction = "",
593 bool ShowHint = true) {
594 if (!WhenceFile.empty())
595 errs() << WhenceFile << ": ";
596 if (!WhenceFunction.empty())
597 errs() << WhenceFunction << ": ";
598
599 auto IPE = instrprof_error::success;
600 E = handleErrors(E: std::move(E),
601 Hs: [&IPE](std::unique_ptr<InstrProfError> E) -> Error {
602 IPE = E->get();
603 return Error(std::move(E));
604 });
605 errs() << toString(E: std::move(E)) << "\n";
606
607 if (ShowHint) {
608 StringRef Hint = "";
609 if (IPE != instrprof_error::success) {
610 switch (IPE) {
611 case instrprof_error::hash_mismatch:
612 case instrprof_error::count_mismatch:
613 case instrprof_error::value_site_count_mismatch:
614 Hint = "Make sure that all profile data to be merged is generated "
615 "from the same binary.";
616 break;
617 default:
618 break;
619 }
620 }
621
622 if (!Hint.empty())
623 errs() << Hint << "\n";
624 }
625}
626
627namespace {
628/// A remapper from original symbol names to new symbol names based on a file
629/// containing a list of mappings from old name to new name.
630class SymbolRemapper {
631 std::unique_ptr<MemoryBuffer> File;
632 DenseMap<StringRef, StringRef> RemappingTable;
633
634public:
635 /// Build a SymbolRemapper from a file containing a list of old/new symbols.
636 static std::unique_ptr<SymbolRemapper> create(StringRef InputFile) {
637 auto BufOrError = MemoryBuffer::getFileOrSTDIN(Filename: InputFile);
638 if (!BufOrError)
639 exitWithErrorCode(EC: BufOrError.getError(), Whence: InputFile);
640
641 auto Remapper = std::make_unique<SymbolRemapper>();
642 Remapper->File = std::move(BufOrError.get());
643
644 for (line_iterator LineIt(*Remapper->File, /*SkipBlanks=*/true, '#');
645 !LineIt.is_at_eof(); ++LineIt) {
646 std::pair<StringRef, StringRef> Parts = LineIt->split(Separator: ' ');
647 if (Parts.first.empty() || Parts.second.empty() ||
648 Parts.second.count(C: ' ')) {
649 exitWithError(Message: "unexpected line in remapping file",
650 Whence: (InputFile + ":" + Twine(LineIt.line_number())).str(),
651 Hint: "expected 'old_symbol new_symbol'");
652 }
653 Remapper->RemappingTable.insert(KV: Parts);
654 }
655 return Remapper;
656 }
657
658 /// Attempt to map the given old symbol into a new symbol.
659 ///
660 /// \return The new symbol, or \p Name if no such symbol was found.
661 StringRef operator()(StringRef Name) {
662 StringRef New = RemappingTable.lookup(Val: Name);
663 return New.empty() ? Name : New;
664 }
665
666 FunctionId operator()(FunctionId Name) {
667 // MD5 name cannot be remapped.
668 if (!Name.isStringRef())
669 return Name;
670 StringRef New = RemappingTable.lookup(Val: Name.stringRef());
671 return New.empty() ? Name : FunctionId(New);
672 }
673};
674}
675
676struct WeightedFile {
677 std::string Filename;
678 uint64_t Weight;
679};
680typedef SmallVector<WeightedFile, 5> WeightedFileVector;
681
682/// Keep track of merged data and reported errors.
683struct WriterContext {
684 std::mutex Lock;
685 InstrProfWriter Writer;
686 std::vector<std::pair<Error, std::string>> Errors;
687 std::mutex &ErrLock;
688 SmallSet<instrprof_error, 4> &WriterErrorCodes;
689
690 WriterContext(bool IsSparse, std::mutex &ErrLock,
691 SmallSet<instrprof_error, 4> &WriterErrorCodes,
692 uint64_t ReservoirSize = 0, uint64_t MaxTraceLength = 0)
693 : Writer(IsSparse, ReservoirSize, MaxTraceLength, DoWritePrevVersion,
694 MemProfVersionRequested, MemProfFullSchema,
695 MemprofGenerateRandomHotness, RandomSeed),
696 ErrLock(ErrLock), WriterErrorCodes(WriterErrorCodes) {}
697};
698
699/// Computer the overlap b/w profile BaseFilename and TestFileName,
700/// and store the program level result to Overlap.
701static void overlapInput(const std::string &BaseFilename,
702 const std::string &TestFilename, WriterContext *WC,
703 OverlapStats &Overlap,
704 const OverlapFuncFilters &FuncFilter,
705 raw_fd_ostream &OS, bool IsCS) {
706 auto FS = vfs::getRealFileSystem();
707 auto ReaderOrErr = InstrProfReader::create(Path: TestFilename, FS&: *FS);
708 if (Error E = ReaderOrErr.takeError()) {
709 // Skip the empty profiles by returning sliently.
710 auto [ErrorCode, Msg] = InstrProfError::take(E: std::move(E));
711 if (ErrorCode != instrprof_error::empty_raw_profile)
712 WC->Errors.emplace_back(args: make_error<InstrProfError>(Args&: ErrorCode, Args&: Msg),
713 args: TestFilename);
714 return;
715 }
716
717 auto Reader = std::move(ReaderOrErr.get());
718 for (auto &I : *Reader) {
719 OverlapStats FuncOverlap(OverlapStats::FunctionLevel);
720 FuncOverlap.setFuncInfo(Name: I.Name, Hash: I.Hash);
721
722 WC->Writer.overlapRecord(Other: std::move(I), Overlap, FuncLevelOverlap&: FuncOverlap, FuncFilter);
723 FuncOverlap.dump(OS);
724 }
725}
726
727/// Load an input into a writer context.
728static void
729loadInput(const WeightedFile &Input, SymbolRemapper *Remapper,
730 const InstrProfCorrelator *Correlator, const StringRef ProfiledBinary,
731 WriterContext *WC, const object::BuildIDFetcher *BIDFetcher = nullptr,
732 const ProfCorrelatorKind *BIDFetcherCorrelatorKind = nullptr) {
733 std::unique_lock<std::mutex> CtxGuard{WC->Lock};
734
735 // Copy the filename, because llvm::ThreadPool copied the input "const
736 // WeightedFile &" by value, making a reference to the filename within it
737 // invalid outside of this packaged task.
738 std::string Filename = Input.Filename;
739
740 using ::llvm::memprof::RawMemProfReader;
741 if (RawMemProfReader::hasFormat(Path: Input.Filename)) {
742 auto ReaderOrErr = RawMemProfReader::create(Path: Input.Filename, ProfiledBinary);
743 if (!ReaderOrErr) {
744 exitWithError(E: ReaderOrErr.takeError(), Whence: Input.Filename);
745 }
746 std::unique_ptr<RawMemProfReader> Reader = std::move(ReaderOrErr.get());
747 // Check if the profile types can be merged, e.g. clang frontend profiles
748 // should not be merged with memprof profiles.
749 if (Error E = WC->Writer.mergeProfileKind(Other: Reader->getProfileKind())) {
750 consumeError(Err: std::move(E));
751 WC->Errors.emplace_back(
752 args: make_error<StringError>(
753 Args: "Cannot merge MemProf profile with Clang generated profile.",
754 Args: std::error_code()),
755 args&: Filename);
756 return;
757 }
758
759 auto MemProfError = [&](Error E) {
760 auto [ErrorCode, Msg] = InstrProfError::take(E: std::move(E));
761 WC->Errors.emplace_back(args: make_error<InstrProfError>(Args&: ErrorCode, Args&: Msg),
762 args&: Filename);
763 };
764
765 WC->Writer.addMemProfData(Incoming: Reader->takeMemProfData(), Warn: MemProfError);
766 return;
767 }
768
769 using ::llvm::memprof::YAMLMemProfReader;
770 if (YAMLMemProfReader::hasFormat(Path: Input.Filename)) {
771 auto ReaderOrErr = YAMLMemProfReader::create(Path: Input.Filename);
772 if (!ReaderOrErr)
773 exitWithError(E: ReaderOrErr.takeError(), Whence: Input.Filename);
774 std::unique_ptr<YAMLMemProfReader> Reader = std::move(ReaderOrErr.get());
775 // Check if the profile types can be merged, e.g. clang frontend profiles
776 // should not be merged with memprof profiles.
777 if (Error E = WC->Writer.mergeProfileKind(Other: Reader->getProfileKind())) {
778 consumeError(Err: std::move(E));
779 WC->Errors.emplace_back(
780 args: make_error<StringError>(
781 Args: "Cannot merge MemProf profile with incompatible profile.",
782 Args: std::error_code()),
783 args&: Filename);
784 return;
785 }
786
787 auto MemProfError = [&](Error E) {
788 auto [ErrorCode, Msg] = InstrProfError::take(E: std::move(E));
789 WC->Errors.emplace_back(args: make_error<InstrProfError>(Args&: ErrorCode, Args&: Msg),
790 args&: Filename);
791 };
792
793 auto MemProfData = Reader->takeMemProfData();
794
795 auto DataAccessProfData = Reader->takeDataAccessProfData();
796
797 // Check for the empty input in case the YAML file is invalid.
798 if (MemProfData.Records.empty() &&
799 (!DataAccessProfData || DataAccessProfData->empty())) {
800 WC->Errors.emplace_back(
801 args: make_error<StringError>(Args: "The profile is empty.", Args: std::error_code()),
802 args&: Filename);
803 }
804
805 WC->Writer.addMemProfData(Incoming: std::move(MemProfData), Warn: MemProfError);
806 WC->Writer.addDataAccessProfData(DataAccessProfile: std::move(DataAccessProfData));
807 return;
808 }
809
810 auto FS = vfs::getRealFileSystem();
811 // TODO: This only saves the first non-fatal error from InstrProfReader, and
812 // then added to WriterContext::Errors. However, this is not extensible, if
813 // we have more non-fatal errors from InstrProfReader in the future. How
814 // should this interact with different -failure-mode?
815 std::optional<std::pair<Error, std::string>> ReaderWarning;
816 llvm::scope_exit ReaderWarningScope([&] {
817 // If we hit a different error we may still have an error in ReaderWarning.
818 // Consume it now to avoid an assert
819 if (ReaderWarning)
820 consumeError(Err: std::move(ReaderWarning->first));
821 });
822 auto Warn = [&](Error E) {
823 if (ReaderWarning) {
824 consumeError(Err: std::move(E));
825 return;
826 }
827 // Only show the first time an error occurs in this file.
828 auto [ErrCode, Msg] = InstrProfError::take(E: std::move(E));
829 ReaderWarning = {make_error<InstrProfError>(Args&: ErrCode, Args&: Msg), Filename};
830 };
831
832 const ProfCorrelatorKind CorrelatorKind = BIDFetcherCorrelatorKind
833 ? *BIDFetcherCorrelatorKind
834 : ProfCorrelatorKind::NONE;
835 auto ReaderOrErr = InstrProfReader::create(Path: Input.Filename, FS&: *FS, Correlator,
836 BIDFetcher, BIDFetcherCorrelatorKind: CorrelatorKind, Warn);
837 if (Error E = ReaderOrErr.takeError()) {
838 // Skip the empty profiles by returning silently.
839 auto [ErrCode, Msg] = InstrProfError::take(E: std::move(E));
840 if (ErrCode != instrprof_error::empty_raw_profile)
841 WC->Errors.emplace_back(args: make_error<InstrProfError>(Args&: ErrCode, Args&: Msg),
842 args&: Filename);
843 return;
844 }
845
846 auto Reader = std::move(ReaderOrErr.get());
847 if (Error E = WC->Writer.mergeProfileKind(Other: Reader->getProfileKind())) {
848 WC->Errors.emplace_back(args: std::move(E), args&: Filename);
849 return;
850 }
851
852 for (auto &I : *Reader) {
853 if (Remapper)
854 I.Name = (*Remapper)(I.Name);
855 const StringRef FuncName = I.Name;
856 bool Reported = false;
857
858 WC->Writer.addRecord(I: std::move(I), Weight: Input.Weight, Warn: [&](Error E) {
859 if (Reported) {
860 consumeError(Err: std::move(E));
861 return;
862 }
863 Reported = true;
864 // Only show hint the first time an error occurs.
865 auto [ErrCode, Msg] = InstrProfError::take(E: std::move(E));
866 std::unique_lock<std::mutex> ErrGuard{WC->ErrLock};
867 bool firstTime = WC->WriterErrorCodes.insert(V: ErrCode).second;
868 handleMergeWriterError(E: make_error<InstrProfError>(Args&: ErrCode, Args&: Msg),
869 WhenceFile: Input.Filename, WhenceFunction: FuncName, ShowHint: firstTime);
870 });
871 }
872
873 if (KeepVTableSymbols) {
874 const InstrProfSymtab &symtab = Reader->getSymtab();
875 const auto &VTableNames = symtab.getVTableNames();
876
877 for (const auto &kv : VTableNames)
878 WC->Writer.addVTableName(VTableName: kv.getKey());
879 }
880
881 if (Reader->hasTemporalProfile()) {
882 auto &Traces = Reader->getTemporalProfTraces(Weight: Input.Weight);
883 if (!Traces.empty())
884 WC->Writer.addTemporalProfileTraces(
885 SrcTraces&: Traces, SrcStreamSize: Reader->getTemporalProfTraceStreamSize());
886 }
887 if (Reader->hasError()) {
888 if (Error E = Reader->getError()) {
889 WC->Errors.emplace_back(args: std::move(E), args&: Filename);
890 return;
891 }
892 }
893
894 std::vector<llvm::object::BuildID> BinaryIds;
895 if (Error E = Reader->readBinaryIds(BinaryIds)) {
896 WC->Errors.emplace_back(args: std::move(E), args&: Filename);
897 return;
898 }
899 WC->Writer.addBinaryIds(BIs: BinaryIds);
900
901 if (ReaderWarning) {
902 WC->Errors.emplace_back(args: std::move(ReaderWarning->first),
903 args&: ReaderWarning->second);
904 }
905}
906
907/// Merge the \p Src writer context into \p Dst.
908static void mergeWriterContexts(WriterContext *Dst, WriterContext *Src) {
909 for (auto &ErrorPair : Src->Errors)
910 Dst->Errors.push_back(x: std::move(ErrorPair));
911 Src->Errors.clear();
912
913 if (Error E = Dst->Writer.mergeProfileKind(Other: Src->Writer.getProfileKind()))
914 exitWithError(E: std::move(E));
915
916 Dst->Writer.mergeRecordsFromWriter(IPW: std::move(Src->Writer), Warn: [&](Error E) {
917 auto [ErrorCode, Msg] = InstrProfError::take(E: std::move(E));
918 std::unique_lock<std::mutex> ErrGuard{Dst->ErrLock};
919 bool firstTime = Dst->WriterErrorCodes.insert(V: ErrorCode).second;
920 if (firstTime)
921 warn(Message: toString(E: make_error<InstrProfError>(Args&: ErrorCode, Args&: Msg)));
922 });
923}
924
925static StringRef
926getFuncName(const StringMap<InstrProfWriter::ProfilingData>::value_type &Val) {
927 return Val.first();
928}
929
930static std::string
931getFuncName(const SampleProfileMap::value_type &Val) {
932 return Val.second.getContext().toString();
933}
934
935template <typename T>
936static void filterFunctions(T &ProfileMap) {
937 bool hasFilter = !FuncNameFilter.empty();
938 bool hasNegativeFilter = !FuncNameNegativeFilter.empty();
939 if (!hasFilter && !hasNegativeFilter)
940 return;
941
942 // If filter starts with '?' it is MSVC mangled name, not a regex.
943 llvm::Regex ProbablyMSVCMangledName("[?@$_0-9A-Za-z]+");
944 if (hasFilter && FuncNameFilter[0] == '?' &&
945 ProbablyMSVCMangledName.match(String: FuncNameFilter))
946 FuncNameFilter = llvm::Regex::escape(String: FuncNameFilter);
947 if (hasNegativeFilter && FuncNameNegativeFilter[0] == '?' &&
948 ProbablyMSVCMangledName.match(String: FuncNameNegativeFilter))
949 FuncNameNegativeFilter = llvm::Regex::escape(String: FuncNameNegativeFilter);
950
951 size_t Count = ProfileMap.size();
952 llvm::Regex Pattern(FuncNameFilter);
953 llvm::Regex NegativePattern(FuncNameNegativeFilter);
954 std::string Error;
955 if (hasFilter && !Pattern.isValid(Error))
956 exitWithError(Message: Error);
957 if (hasNegativeFilter && !NegativePattern.isValid(Error))
958 exitWithError(Message: Error);
959
960 // Handle MD5 profile, so it is still able to match using the original name.
961 std::string MD5Name = std::to_string(val: llvm::MD5Hash(Str: FuncNameFilter));
962 std::string NegativeMD5Name =
963 std::to_string(val: llvm::MD5Hash(Str: FuncNameNegativeFilter));
964
965 ProfileMap.remove_if([&](const auto &Entry) {
966 const auto &FuncName = getFuncName(Entry);
967 // Negative filter has higher precedence than positive filter.
968 return (hasNegativeFilter &&
969 (NegativePattern.match(String: FuncName) ||
970 (FunctionSamples::UseMD5 && NegativeMD5Name == FuncName))) ||
971 (hasFilter && !(Pattern.match(String: FuncName) ||
972 (FunctionSamples::UseMD5 && MD5Name == FuncName)));
973 });
974
975 llvm::dbgs() << Count - ProfileMap.size() << " of " << Count << " functions "
976 << "in the original profile are filtered.\n";
977}
978
979static void writeInstrProfile(StringRef OutputFilename,
980 ProfileFormat OutputFormat,
981 InstrProfWriter &Writer) {
982 std::error_code EC;
983 raw_fd_ostream Output(OutputFilename.data(), EC,
984 OutputFormat == PF_Text ? sys::fs::OF_TextWithCRLF
985 : sys::fs::OF_None);
986 if (EC)
987 exitWithErrorCode(EC, Whence: OutputFilename);
988
989 if (OutputFormat == PF_Text) {
990 if (Error E = Writer.writeText(OS&: Output))
991 warn(E: std::move(E));
992 } else {
993 if (Output.is_displayed())
994 exitWithError(Message: "cannot write a non-text format profile to the terminal");
995 if (Error E = Writer.write(OS&: Output))
996 warn(E: std::move(E));
997 }
998}
999
1000static void mergeInstrProfile(const WeightedFileVector &Inputs,
1001 SymbolRemapper *Remapper,
1002 int MaxDbgCorrelationWarnings,
1003 const StringRef ProfiledBinary) {
1004 const uint64_t TraceReservoirSize = TemporalProfTraceReservoirSize.getValue();
1005 const uint64_t MaxTraceLength = TemporalProfMaxTraceLength.getValue();
1006 if (OutputFormat == PF_Compact_Binary)
1007 exitWithError(Message: "Compact Binary is deprecated");
1008 if (OutputFormat != PF_Binary && OutputFormat != PF_Ext_Binary &&
1009 OutputFormat != PF_Text)
1010 exitWithError(Message: "unknown format is specified");
1011
1012 // TODO: Maybe we should support correlation with mixture of different
1013 // correlation modes(w/wo debug-info/object correlation).
1014 if (DebugInfoFilename.empty()) {
1015 if (!BinaryFilename.empty() && (DebugInfod || !DebugFileDirectory.empty()))
1016 exitWithError(Message: "Expected only one of -binary-file, -debuginfod or "
1017 "-debug-file-directory");
1018 } else if (!BinaryFilename.empty() || DebugInfod ||
1019 !DebugFileDirectory.empty()) {
1020 exitWithError(Message: "Expected only one of -debug-info, -binary-file, -debuginfod "
1021 "or -debug-file-directory");
1022 }
1023 std::string CorrelateFilename;
1024 ProfCorrelatorKind CorrelateKind = ProfCorrelatorKind::NONE;
1025 if (!DebugInfoFilename.empty()) {
1026 CorrelateFilename = DebugInfoFilename;
1027 CorrelateKind = ProfCorrelatorKind::DEBUG_INFO;
1028 } else if (!BinaryFilename.empty()) {
1029 CorrelateFilename = BinaryFilename;
1030 CorrelateKind = ProfCorrelatorKind::BINARY;
1031 }
1032
1033 std::unique_ptr<InstrProfCorrelator> Correlator;
1034 if (CorrelateKind != InstrProfCorrelator::NONE) {
1035 if (auto Err = InstrProfCorrelator::get(Filename: CorrelateFilename, FileKind: CorrelateKind)
1036 .moveInto(Value&: Correlator))
1037 exitWithError(E: std::move(Err), Whence: CorrelateFilename);
1038 if (auto Err = Correlator->correlateProfileData(MaxWarnings: MaxDbgCorrelationWarnings))
1039 exitWithError(E: std::move(Err), Whence: CorrelateFilename);
1040 }
1041
1042 ProfCorrelatorKind BIDFetcherCorrelateKind = ProfCorrelatorKind::NONE;
1043 std::unique_ptr<object::BuildIDFetcher> BIDFetcher;
1044 if (DebugInfod) {
1045 llvm::HTTPClient::initialize();
1046 BIDFetcher = std::make_unique<DebuginfodFetcher>(args&: DebugFileDirectory);
1047 if (!BIDFetcherProfileCorrelate)
1048 exitWithError(Message: "Expected --correlate when --debuginfod is provided");
1049 BIDFetcherCorrelateKind = BIDFetcherProfileCorrelate;
1050 } else if (!DebugFileDirectory.empty()) {
1051 BIDFetcher = std::make_unique<object::BuildIDFetcher>(args&: DebugFileDirectory);
1052 if (!BIDFetcherProfileCorrelate)
1053 exitWithError(Message: "Expected --correlate when --debug-file-directory "
1054 "is provided");
1055 BIDFetcherCorrelateKind = BIDFetcherProfileCorrelate;
1056 } else if (BIDFetcherProfileCorrelate) {
1057 exitWithError(Message: "Expected --debuginfod or --debug-file-directory when "
1058 "--correlate is provided");
1059 }
1060
1061 std::mutex ErrorLock;
1062 SmallSet<instrprof_error, 4> WriterErrorCodes;
1063
1064 // If NumThreads is not specified, auto-detect a good default.
1065 if (NumThreads == 0)
1066 NumThreads = std::min(a: hardware_concurrency().compute_thread_count(),
1067 b: unsigned((Inputs.size() + 1) / 2));
1068
1069 // Initialize the writer contexts.
1070 SmallVector<std::unique_ptr<WriterContext>, 4> Contexts;
1071 for (unsigned I = 0; I < NumThreads; ++I)
1072 Contexts.emplace_back(Args: std::make_unique<WriterContext>(
1073 args&: OutputSparse, args&: ErrorLock, args&: WriterErrorCodes, args: TraceReservoirSize,
1074 args: MaxTraceLength));
1075
1076 if (NumThreads == 1) {
1077 for (const auto &Input : Inputs)
1078 loadInput(Input, Remapper, Correlator: Correlator.get(), ProfiledBinary,
1079 WC: Contexts[0].get(), BIDFetcher: BIDFetcher.get(), BIDFetcherCorrelatorKind: &BIDFetcherCorrelateKind);
1080 } else {
1081 DefaultThreadPool Pool(hardware_concurrency(ThreadCount: NumThreads));
1082
1083 // Load the inputs in parallel (N/NumThreads serial steps).
1084 unsigned Ctx = 0;
1085 for (const auto &Input : Inputs) {
1086 Pool.async(F&: loadInput, ArgList: Input, ArgList&: Remapper, ArgList: Correlator.get(), ArgList: ProfiledBinary,
1087 ArgList: Contexts[Ctx].get(), ArgList: BIDFetcher.get(),
1088 ArgList: &BIDFetcherCorrelateKind);
1089 Ctx = (Ctx + 1) % NumThreads;
1090 }
1091 Pool.wait();
1092
1093 // Merge the writer contexts together (~ lg(NumThreads) serial steps).
1094 unsigned Mid = Contexts.size() / 2;
1095 unsigned End = Contexts.size();
1096 assert(Mid > 0 && "Expected more than one context");
1097 do {
1098 for (unsigned I = 0; I < Mid; ++I)
1099 Pool.async(F&: mergeWriterContexts, ArgList: Contexts[I].get(),
1100 ArgList: Contexts[I + Mid].get());
1101 Pool.wait();
1102 if (End & 1) {
1103 Pool.async(F&: mergeWriterContexts, ArgList: Contexts[0].get(),
1104 ArgList: Contexts[End - 1].get());
1105 Pool.wait();
1106 }
1107 End = Mid;
1108 Mid /= 2;
1109 } while (Mid > 0);
1110 }
1111
1112 // Handle deferred errors encountered during merging. If the number of errors
1113 // is equal to the number of inputs the merge failed.
1114 unsigned NumErrors = 0;
1115 for (std::unique_ptr<WriterContext> &WC : Contexts) {
1116 for (auto &ErrorPair : WC->Errors) {
1117 ++NumErrors;
1118 warn(Message: toString(E: std::move(ErrorPair.first)), Whence: ErrorPair.second);
1119 }
1120 }
1121 if ((NumErrors == Inputs.size() && FailMode == failIfAllAreInvalid) ||
1122 (NumErrors > 0 && FailMode == failIfAnyAreInvalid))
1123 exitWithError(Message: "no profile can be merged");
1124
1125 filterFunctions(ProfileMap&: Contexts[0]->Writer.getProfileData());
1126
1127 writeInstrProfile(OutputFilename, OutputFormat, Writer&: Contexts[0]->Writer);
1128}
1129
1130/// The profile entry for a function in instrumentation profile.
1131struct InstrProfileEntry {
1132 uint64_t MaxCount = 0;
1133 uint64_t NumEdgeCounters = 0;
1134 float ZeroCounterRatio = 0.0;
1135 InstrProfRecord *ProfRecord;
1136 InstrProfileEntry(InstrProfRecord *Record);
1137 InstrProfileEntry() = default;
1138};
1139
1140InstrProfileEntry::InstrProfileEntry(InstrProfRecord *Record) {
1141 ProfRecord = Record;
1142 uint64_t CntNum = Record->Counts.size();
1143 uint64_t ZeroCntNum = 0;
1144 for (size_t I = 0; I < CntNum; ++I) {
1145 MaxCount = std::max(a: MaxCount, b: Record->Counts[I]);
1146 ZeroCntNum += !Record->Counts[I];
1147 }
1148 ZeroCounterRatio = (float)ZeroCntNum / CntNum;
1149 NumEdgeCounters = CntNum;
1150}
1151
1152/// Either set all the counters in the instr profile entry \p IFE to
1153/// -1 / -2 /in order to drop the profile or scale up the
1154/// counters in \p IFP to be above hot / cold threshold. We use
1155/// the ratio of zero counters in the profile of a function to
1156/// decide the profile is helpful or harmful for performance,
1157/// and to choose whether to scale up or drop it.
1158static void updateInstrProfileEntry(InstrProfileEntry &IFE, bool SetToHot,
1159 uint64_t HotInstrThreshold,
1160 uint64_t ColdInstrThreshold,
1161 float ZeroCounterThreshold) {
1162 InstrProfRecord *ProfRecord = IFE.ProfRecord;
1163 if (!IFE.MaxCount || IFE.ZeroCounterRatio > ZeroCounterThreshold) {
1164 // If all or most of the counters of the function are zero, the
1165 // profile is unaccountable and should be dropped. Reset all the
1166 // counters to be -1 / -2 and PGO profile-use will drop the profile.
1167 // All counters being -1 also implies that the function is hot so
1168 // PGO profile-use will also set the entry count metadata to be
1169 // above hot threshold.
1170 // All counters being -2 implies that the function is warm so
1171 // PGO profile-use will also set the entry count metadata to be
1172 // above cold threshold.
1173 auto Kind =
1174 (SetToHot ? InstrProfRecord::PseudoHot : InstrProfRecord::PseudoWarm);
1175 ProfRecord->setPseudoCount(Kind);
1176 return;
1177 }
1178
1179 // Scale up the MaxCount to be multiple times above hot / cold threshold.
1180 const unsigned MultiplyFactor = 3;
1181 uint64_t Threshold = (SetToHot ? HotInstrThreshold : ColdInstrThreshold);
1182 uint64_t Numerator = Threshold * MultiplyFactor;
1183
1184 // Make sure Threshold for warm counters is below the HotInstrThreshold.
1185 if (!SetToHot && Threshold >= HotInstrThreshold) {
1186 Threshold = (HotInstrThreshold + ColdInstrThreshold) / 2;
1187 }
1188
1189 uint64_t Denominator = IFE.MaxCount;
1190 if (Numerator <= Denominator)
1191 return;
1192 ProfRecord->scale(N: Numerator, D: Denominator, Warn: [&](instrprof_error E) {
1193 warn(Message: toString(E: make_error<InstrProfError>(Args&: E)));
1194 });
1195}
1196
1197const uint64_t ColdPercentileIdx = 15;
1198const uint64_t HotPercentileIdx = 11;
1199
1200using sampleprof::FSDiscriminatorPass;
1201
1202// Internal options to set FSDiscriminatorPass. Used in merge and show
1203// commands.
1204static cl::opt<FSDiscriminatorPass> FSDiscriminatorPassOption(
1205 "fs-discriminator-pass", cl::init(Val: PassLast), cl::Hidden,
1206 cl::desc("Zero out the discriminator bits for the FS discrimiantor "
1207 "pass beyond this value. The enum values are defined in "
1208 "Support/Discriminator.h"),
1209 cl::values(clEnumVal(Base, "Use base discriminators only"),
1210 clEnumVal(Pass1, "Use base and pass 1 discriminators"),
1211 clEnumVal(Pass2, "Use base and pass 1-2 discriminators"),
1212 clEnumVal(Pass3, "Use base and pass 1-3 discriminators"),
1213 clEnumVal(PassLast, "Use all discriminator bits (default)")));
1214
1215static unsigned getDiscriminatorMask() {
1216 return getN1Bits(N: getFSPassBitEnd(P: FSDiscriminatorPassOption.getValue()));
1217}
1218
1219/// Adjust the instr profile in \p WC based on the sample profile in
1220/// \p Reader.
1221static void
1222adjustInstrProfile(std::unique_ptr<WriterContext> &WC,
1223 std::unique_ptr<sampleprof::SampleProfileReader> &Reader,
1224 unsigned SupplMinSizeThreshold, float ZeroCounterThreshold,
1225 unsigned InstrProfColdThreshold) {
1226 // Function to its entry in instr profile.
1227 StringMap<InstrProfileEntry> InstrProfileMap;
1228 StringMap<StringRef> StaticFuncMap;
1229 InstrProfSummaryBuilder IPBuilder(ProfileSummaryBuilder::DefaultCutoffs);
1230
1231 auto checkSampleProfileHasFUnique = [&Reader]() {
1232 for (const auto &PD : Reader->getProfiles()) {
1233 auto &FContext = PD.second.getContext();
1234 if (FContext.toString().find(s: FunctionSamples::UniqSuffix) !=
1235 std::string::npos) {
1236 return true;
1237 }
1238 }
1239 return false;
1240 };
1241
1242 bool SampleProfileHasFUnique = checkSampleProfileHasFUnique();
1243
1244 auto buildStaticFuncMap = [&StaticFuncMap,
1245 SampleProfileHasFUnique](const StringRef Name) {
1246 std::string FilePrefixes[] = {".cpp", "cc", ".c", ".hpp", ".h"};
1247 size_t PrefixPos = StringRef::npos;
1248 for (auto &FilePrefix : FilePrefixes) {
1249 std::string NamePrefix = FilePrefix + GlobalIdentifierDelimiter;
1250 PrefixPos = Name.find_insensitive(Str: NamePrefix);
1251 if (PrefixPos == StringRef::npos)
1252 continue;
1253 PrefixPos += NamePrefix.size();
1254 break;
1255 }
1256
1257 if (PrefixPos == StringRef::npos) {
1258 return;
1259 }
1260
1261 StringRef NewName = Name.drop_front(N: PrefixPos);
1262 StringRef FName = Name.substr(Start: 0, N: PrefixPos - 1);
1263 if (NewName.size() == 0) {
1264 return;
1265 }
1266
1267 // This name should have a static linkage.
1268 size_t PostfixPos = NewName.find(Str: FunctionSamples::UniqSuffix);
1269 bool ProfileHasFUnique = (PostfixPos != StringRef::npos);
1270
1271 // If sample profile and instrumented profile do not agree on symbol
1272 // uniqification.
1273 if (SampleProfileHasFUnique != ProfileHasFUnique) {
1274 // If instrumented profile uses -funique-internal-linkage-symbols,
1275 // we need to trim the name.
1276 if (ProfileHasFUnique) {
1277 NewName = NewName.substr(Start: 0, N: PostfixPos);
1278 } else {
1279 // If sample profile uses -funique-internal-linkage-symbols,
1280 // we build the map.
1281 std::string NStr =
1282 NewName.str() + getUniqueInternalLinkagePostfix(FName);
1283 NewName = StringRef(NStr);
1284 StaticFuncMap[NewName] = Name;
1285 return;
1286 }
1287 }
1288
1289 auto [It, Inserted] = StaticFuncMap.try_emplace(Key: NewName, Args: Name);
1290 if (!Inserted)
1291 It->second = DuplicateNameStr;
1292 };
1293
1294 // We need to flatten the SampleFDO profile as the InstrFDO
1295 // profile does not have inlined callsite profiles.
1296 // One caveat is the pre-inlined function -- their samples
1297 // should be collapsed into the caller function.
1298 // Here we do a DFS traversal to get the flatten profile
1299 // info: the sum of entrycount and the max of maxcount.
1300 // Here is the algorithm:
1301 // recursive (FS, root_name) {
1302 // name = FS->getName();
1303 // get samples for FS;
1304 // if (InstrProf.find(name) {
1305 // root_name = name;
1306 // } else {
1307 // if (name is in static_func map) {
1308 // root_name = static_name;
1309 // }
1310 // }
1311 // update the Map entry for root_name;
1312 // for (subfs: FS) {
1313 // recursive(subfs, root_name);
1314 // }
1315 // }
1316 //
1317 // Here is an example.
1318 //
1319 // SampleProfile:
1320 // foo:12345:1000
1321 // 1: 1000
1322 // 2.1: 1000
1323 // 15: 5000
1324 // 4: bar:1000
1325 // 1: 1000
1326 // 2: goo:3000
1327 // 1: 3000
1328 // 8: bar:40000
1329 // 1: 10000
1330 // 2: goo:30000
1331 // 1: 30000
1332 //
1333 // InstrProfile has two entries:
1334 // foo
1335 // bar.cc;bar
1336 //
1337 // After BuildMaxSampleMap, we should have the following in FlattenSampleMap:
1338 // {"foo", {1000, 5000}}
1339 // {"bar.cc;bar", {11000, 30000}}
1340 //
1341 // foo's has an entry count of 1000, and max body count of 5000.
1342 // bar.cc;bar has an entry count of 11000 (sum two callsites of 1000 and
1343 // 10000), and max count of 30000 (from the callsite in line 8).
1344 //
1345 // Note that goo's count will remain in bar.cc;bar() as it does not have an
1346 // entry in InstrProfile.
1347 llvm::StringMap<std::pair<uint64_t, uint64_t>> FlattenSampleMap;
1348 auto BuildMaxSampleMap = [&FlattenSampleMap, &StaticFuncMap,
1349 &InstrProfileMap](const FunctionSamples &FS,
1350 const StringRef &RootName) {
1351 auto BuildMaxSampleMapImpl = [&](const FunctionSamples &FS,
1352 const StringRef &RootName,
1353 auto &BuildImpl) -> void {
1354 std::string NameStr = FS.getFunction().str();
1355 const StringRef Name = NameStr;
1356 const StringRef *NewRootName = &RootName;
1357 uint64_t EntrySample = FS.getHeadSamplesEstimate();
1358 uint64_t MaxBodySample = FS.getMaxCountInside(/* SkipCallSite*/ true);
1359
1360 auto It = InstrProfileMap.find(Key: Name);
1361 if (It != InstrProfileMap.end()) {
1362 NewRootName = &Name;
1363 } else {
1364 auto NewName = StaticFuncMap.find(Key: Name);
1365 if (NewName != StaticFuncMap.end()) {
1366 It = InstrProfileMap.find(Key: NewName->second);
1367 if (NewName->second != DuplicateNameStr) {
1368 NewRootName = &NewName->second;
1369 }
1370 } else {
1371 // Here the EntrySample is of an inlined function, so we should not
1372 // update the EntrySample in the map.
1373 EntrySample = 0;
1374 }
1375 }
1376 EntrySample += FlattenSampleMap[*NewRootName].first;
1377 MaxBodySample =
1378 std::max(a: FlattenSampleMap[*NewRootName].second, b: MaxBodySample);
1379 FlattenSampleMap[*NewRootName] =
1380 std::make_pair(x&: EntrySample, y&: MaxBodySample);
1381
1382 for (const auto &C : FS.getCallsiteSamples())
1383 for (const auto &F : C.second)
1384 BuildImpl(F.second, *NewRootName, BuildImpl);
1385 };
1386 BuildMaxSampleMapImpl(FS, RootName, BuildMaxSampleMapImpl);
1387 };
1388
1389 for (auto &PD : WC->Writer.getProfileData()) {
1390 // Populate IPBuilder.
1391 for (const auto &PDV : PD.getValue()) {
1392 InstrProfRecord Record = PDV.second;
1393 IPBuilder.addRecord(Record);
1394 }
1395
1396 // If a function has multiple entries in instr profile, skip it.
1397 if (PD.getValue().size() != 1)
1398 continue;
1399
1400 // Initialize InstrProfileMap.
1401 InstrProfRecord *R = &PD.getValue().begin()->second;
1402 StringRef FullName = PD.getKey();
1403 InstrProfileMap[FullName] = InstrProfileEntry(R);
1404 buildStaticFuncMap(FullName);
1405 }
1406
1407 for (auto &PD : Reader->getProfiles()) {
1408 sampleprof::FunctionSamples &FS = PD.second;
1409 std::string Name = FS.getFunction().str();
1410 BuildMaxSampleMap(FS, Name);
1411 }
1412
1413 ProfileSummary InstrPS = *IPBuilder.getSummary();
1414 ProfileSummary SamplePS = Reader->getSummary();
1415
1416 // Compute cold thresholds for instr profile and sample profile.
1417 uint64_t HotSampleThreshold =
1418 ProfileSummaryBuilder::getEntryForPercentile(
1419 DS: SamplePS.getDetailedSummary(),
1420 Percentile: ProfileSummaryBuilder::DefaultCutoffs[HotPercentileIdx])
1421 .MinCount;
1422 uint64_t ColdSampleThreshold =
1423 ProfileSummaryBuilder::getEntryForPercentile(
1424 DS: SamplePS.getDetailedSummary(),
1425 Percentile: ProfileSummaryBuilder::DefaultCutoffs[ColdPercentileIdx])
1426 .MinCount;
1427 uint64_t HotInstrThreshold =
1428 ProfileSummaryBuilder::getEntryForPercentile(
1429 DS: InstrPS.getDetailedSummary(),
1430 Percentile: ProfileSummaryBuilder::DefaultCutoffs[HotPercentileIdx])
1431 .MinCount;
1432 uint64_t ColdInstrThreshold =
1433 InstrProfColdThreshold
1434 ? InstrProfColdThreshold
1435 : ProfileSummaryBuilder::getEntryForPercentile(
1436 DS: InstrPS.getDetailedSummary(),
1437 Percentile: ProfileSummaryBuilder::DefaultCutoffs[ColdPercentileIdx])
1438 .MinCount;
1439
1440 // Find hot/warm functions in sample profile which is cold in instr profile
1441 // and adjust the profiles of those functions in the instr profile.
1442 for (const auto &E : FlattenSampleMap) {
1443 uint64_t SampleMaxCount = std::max(a: E.second.first, b: E.second.second);
1444 if (SampleMaxCount < ColdSampleThreshold)
1445 continue;
1446 StringRef Name = E.first();
1447 auto It = InstrProfileMap.find(Key: Name);
1448 if (It == InstrProfileMap.end()) {
1449 auto NewName = StaticFuncMap.find(Key: Name);
1450 if (NewName != StaticFuncMap.end()) {
1451 It = InstrProfileMap.find(Key: NewName->second);
1452 if (NewName->second == DuplicateNameStr) {
1453 WithColor::warning()
1454 << "Static function " << Name
1455 << " has multiple promoted names, cannot adjust profile.\n";
1456 }
1457 }
1458 }
1459 if (It == InstrProfileMap.end() ||
1460 It->second.MaxCount > ColdInstrThreshold ||
1461 It->second.NumEdgeCounters < SupplMinSizeThreshold)
1462 continue;
1463 bool SetToHot = SampleMaxCount >= HotSampleThreshold;
1464 updateInstrProfileEntry(IFE&: It->second, SetToHot, HotInstrThreshold,
1465 ColdInstrThreshold, ZeroCounterThreshold);
1466 }
1467}
1468
1469/// The main function to supplement instr profile with sample profile.
1470/// \Inputs contains the instr profile. \p SampleFilename specifies the
1471/// sample profile. \p OutputFilename specifies the output profile name.
1472/// \p OutputFormat specifies the output profile format. \p OutputSparse
1473/// specifies whether to generate sparse profile. \p SupplMinSizeThreshold
1474/// specifies the minimal size for the functions whose profile will be
1475/// adjusted. \p ZeroCounterThreshold is the threshold to check whether
1476/// a function contains too many zero counters and whether its profile
1477/// should be dropped. \p InstrProfColdThreshold is the user specified
1478/// cold threshold which will override the cold threshold got from the
1479/// instr profile summary.
1480static void supplementInstrProfile(const WeightedFileVector &Inputs,
1481 StringRef SampleFilename, bool OutputSparse,
1482 unsigned SupplMinSizeThreshold,
1483 float ZeroCounterThreshold,
1484 unsigned InstrProfColdThreshold) {
1485 if (OutputFilename == "-")
1486 exitWithError(Message: "cannot write indexed profdata format to stdout");
1487 if (Inputs.size() != 1)
1488 exitWithError(Message: "expect one input to be an instr profile");
1489 if (Inputs[0].Weight != 1)
1490 exitWithError(Message: "expect instr profile doesn't have weight");
1491
1492 StringRef InstrFilename = Inputs[0].Filename;
1493
1494 // Read sample profile.
1495 LLVMContext Context;
1496 auto FS = vfs::getRealFileSystem();
1497 auto ReaderOrErr = sampleprof::SampleProfileReader::create(
1498 Filename: SampleFilename.str(), C&: Context, FS&: *FS, P: FSDiscriminatorPassOption);
1499 if (std::error_code EC = ReaderOrErr.getError())
1500 exitWithErrorCode(EC, Whence: SampleFilename);
1501 auto Reader = std::move(ReaderOrErr.get());
1502 if (std::error_code EC = Reader->read())
1503 exitWithErrorCode(EC, Whence: SampleFilename);
1504
1505 // Read instr profile.
1506 std::mutex ErrorLock;
1507 SmallSet<instrprof_error, 4> WriterErrorCodes;
1508 auto WC = std::make_unique<WriterContext>(args&: OutputSparse, args&: ErrorLock,
1509 args&: WriterErrorCodes);
1510 loadInput(Input: Inputs[0], Remapper: nullptr, Correlator: nullptr, /*ProfiledBinary=*/"", WC: WC.get());
1511 if (WC->Errors.size() > 0)
1512 exitWithError(E: std::move(WC->Errors[0].first), Whence: InstrFilename);
1513
1514 adjustInstrProfile(WC, Reader, SupplMinSizeThreshold, ZeroCounterThreshold,
1515 InstrProfColdThreshold);
1516 writeInstrProfile(OutputFilename, OutputFormat, Writer&: WC->Writer);
1517}
1518
1519/// Make a copy of the given function samples with all symbol names remapped
1520/// by the provided symbol remapper.
1521static sampleprof::FunctionSamples
1522remapSamples(const sampleprof::FunctionSamples &Samples,
1523 SymbolRemapper &Remapper, sampleprof_error &Error) {
1524 sampleprof::FunctionSamples Result;
1525 Result.setFunction(Remapper(Samples.getFunction()));
1526 Result.addTotalSamples(Num: Samples.getTotalSamples());
1527 Result.addHeadSamples(Num: Samples.getHeadSamples());
1528 Result.reserveBodySamples(NumEntries: Samples.getBodySamples().size());
1529 for (const auto &BodySample : Samples.getBodySamples()) {
1530 uint32_t MaskedDiscriminator =
1531 BodySample.first.Discriminator & getDiscriminatorMask();
1532 Result.addBodySamples(LineOffset: BodySample.first.LineOffset, Discriminator: MaskedDiscriminator,
1533 Num: BodySample.second.getSamples());
1534 for (const auto &Target : BodySample.second.getCallTargets()) {
1535 Result.addCalledTargetSamples(LineOffset: BodySample.first.LineOffset,
1536 Discriminator: MaskedDiscriminator,
1537 Func: Remapper(Target.first), Num: Target.second);
1538 }
1539 }
1540 for (const auto &CallsiteSamples : Samples.getCallsiteSamples()) {
1541 sampleprof::FunctionSamplesMap &Target =
1542 Result.functionSamplesAt(Loc: CallsiteSamples.first);
1543 for (const auto &Callsite : CallsiteSamples.second) {
1544 sampleprof::FunctionSamples Remapped =
1545 remapSamples(Samples: Callsite.second, Remapper, Error);
1546 mergeSampleProfErrors(Accumulator&: Error,
1547 Result: Target[Remapped.getFunction()].merge(Other: Remapped));
1548 }
1549 }
1550 return Result;
1551}
1552
1553static sampleprof::SampleProfileFormat FormatMap[] = {
1554 sampleprof::SPF_None,
1555 sampleprof::SPF_Text,
1556 sampleprof::SPF_None,
1557 sampleprof::SPF_Ext_Binary,
1558 sampleprof::SPF_GCC,
1559 sampleprof::SPF_Binary};
1560
1561static std::unique_ptr<MemoryBuffer>
1562getInputFileBuf(const StringRef &InputFile) {
1563 if (InputFile == "")
1564 return {};
1565
1566 auto BufOrError = MemoryBuffer::getFileOrSTDIN(Filename: InputFile);
1567 if (!BufOrError)
1568 exitWithErrorCode(EC: BufOrError.getError(), Whence: InputFile);
1569
1570 return std::move(*BufOrError);
1571}
1572
1573static void populateProfileSymbolList(MemoryBuffer *Buffer,
1574 sampleprof::ProfileSymbolList &PSL) {
1575 if (!Buffer)
1576 return;
1577
1578 SmallVector<StringRef, 32> SymbolVec;
1579 StringRef Data = Buffer->getBuffer();
1580 Data.split(A&: SymbolVec, Separator: '\n', /*MaxSplit=*/-1, /*KeepEmpty=*/false);
1581
1582 for (StringRef SymbolStr : SymbolVec)
1583 PSL.add(Name: SymbolStr.trim());
1584}
1585
1586static void handleExtBinaryWriter(sampleprof::SampleProfileWriter &Writer,
1587 ProfileFormat OutputFormat,
1588 MemoryBuffer *Buffer,
1589 sampleprof::ProfileSymbolList &WriterList,
1590 bool CompressAllSections, bool UseMD5,
1591 bool GenPartialProfile) {
1592 if (SplitLayout) {
1593 if (OutputFormat == PF_Binary)
1594 warn(Message: "-split-layout is ignored. Specify -extbinary to enable it");
1595 else
1596 Writer.setUseCtxSplitLayout();
1597 }
1598
1599 populateProfileSymbolList(Buffer, PSL&: WriterList);
1600 if (WriterList.size() > 0 && OutputFormat != PF_Ext_Binary)
1601 warn(Message: "Profile Symbol list is not empty but the output format is not "
1602 "ExtBinary format. The list will be lost in the output. ");
1603
1604 Writer.setProfileSymbolList(&WriterList);
1605
1606 if (CompressAllSections) {
1607 if (OutputFormat != PF_Ext_Binary)
1608 warn(Message: "-compress-all-section is ignored. Specify -extbinary to enable it");
1609 else
1610 Writer.setToCompressAllSections();
1611 }
1612 if (UseMD5) {
1613 if (OutputFormat != PF_Ext_Binary)
1614 warn(Message: "-use-md5 is ignored. Specify -extbinary to enable it");
1615 else
1616 Writer.setUseMD5();
1617 }
1618 if (GenPartialProfile) {
1619 if (OutputFormat != PF_Ext_Binary)
1620 warn(Message: "-gen-partial-profile is ignored. Specify -extbinary to enable it");
1621 else
1622 Writer.setPartialProfile();
1623 }
1624 if (WriteMD5ProfSymList) {
1625 if (OutputFormat != PF_Ext_Binary)
1626 warn(Message: "-md5-prof-sym-list is ignored. Specify -extbinary to enable it");
1627 else
1628 Writer.setUseMD5ProfileSymbolList();
1629 }
1630 if (WriteMD5IndexedTables) {
1631 if (OutputFormat != PF_Ext_Binary)
1632 warn(Message: "-md5-indexed-tables is ignored. Specify -extbinary to enable it");
1633 else
1634 Writer.setUseMD5IndexedTables();
1635 }
1636}
1637
1638static void mergeSampleProfile(const WeightedFileVector &Inputs,
1639 SymbolRemapper *Remapper,
1640 StringRef ProfileSymbolListFile,
1641 size_t OutputSizeLimit) {
1642 using namespace sampleprof;
1643 SampleProfileMap ProfileMap;
1644 SmallVector<std::unique_ptr<sampleprof::SampleProfileReader>, 5> Readers;
1645 LLVMContext Context;
1646 sampleprof::ProfileSymbolList WriterList;
1647 std::optional<bool> ProfileIsProbeBased;
1648 std::optional<bool> ProfileIsCS;
1649 for (const auto &Input : Inputs) {
1650 auto FS = vfs::getRealFileSystem();
1651 auto ReaderOrErr = SampleProfileReader::create(Filename: Input.Filename, C&: Context, FS&: *FS,
1652 P: FSDiscriminatorPassOption);
1653 if (std::error_code EC = ReaderOrErr.getError()) {
1654 warnOrExitGivenError(FailMode, EC, Whence: Input.Filename);
1655 continue;
1656 }
1657
1658 // We need to keep the readers around until after all the files are
1659 // read so that we do not lose the function names stored in each
1660 // reader's memory. The function names are needed to write out the
1661 // merged profile map.
1662 Readers.push_back(Elt: std::move(ReaderOrErr.get()));
1663 const auto Reader = Readers.back().get();
1664 if (std::error_code EC = Reader->read()) {
1665 warnOrExitGivenError(FailMode, EC, Whence: Input.Filename);
1666 Readers.pop_back();
1667 continue;
1668 }
1669
1670 // Merging cannot preserve payloads that this reader does not understand,
1671 // so make the otherwise intentional forward-compatible skip visible.
1672 if (Reader->hasUnknownProfileTypes())
1673 warn(Message: "unknown composite profile blocks were ignored and will not be "
1674 "preserved",
1675 Whence: Input.Filename);
1676
1677 SampleProfileMap &Profiles = Reader->getProfiles();
1678 if (ProfileIsProbeBased &&
1679 ProfileIsProbeBased != FunctionSamples::ProfileIsProbeBased)
1680 exitWithError(
1681 Message: "cannot merge probe-based profile with non-probe-based profile");
1682 ProfileIsProbeBased = FunctionSamples::ProfileIsProbeBased;
1683 if (ProfileIsCS && ProfileIsCS != FunctionSamples::ProfileIsCS)
1684 exitWithError(Message: "cannot merge CS profile with non-CS profile");
1685 ProfileIsCS = FunctionSamples::ProfileIsCS;
1686 for (SampleProfileMap::iterator I = Profiles.begin(), E = Profiles.end();
1687 I != E; ++I) {
1688 sampleprof_error Result = sampleprof_error::success;
1689 FunctionSamples Remapped =
1690 Remapper ? remapSamples(Samples: I->second, Remapper&: *Remapper, Error&: Result)
1691 : FunctionSamples();
1692 FunctionSamples &Samples = Remapper ? Remapped : I->second;
1693 SampleContext FContext = Samples.getContext();
1694 mergeSampleProfErrors(Accumulator&: Result,
1695 Result: ProfileMap[FContext].merge(Other: Samples, Weight: Input.Weight));
1696 if (Result != sampleprof_error::success) {
1697 std::error_code EC = make_error_code(E: Result);
1698 handleMergeWriterError(E: errorCodeToError(EC), WhenceFile: Input.Filename,
1699 WhenceFunction: FContext.toString());
1700 }
1701 }
1702
1703 if (!DropProfileSymbolList) {
1704 std::unique_ptr<sampleprof::ProfileSymbolList> ReaderList =
1705 Reader->getProfileSymbolList();
1706 if (ReaderList)
1707 WriterList.merge(List: *ReaderList);
1708 }
1709 }
1710
1711 if (ProfileIsCS && (SampleMergeColdContext || SampleTrimColdContext)) {
1712 // Use threshold calculated from profile summary unless specified.
1713 SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs);
1714 auto Summary = Builder.computeSummaryForProfiles(Profiles: ProfileMap);
1715 uint64_t SampleProfColdThreshold =
1716 ProfileSummaryBuilder::getColdCountThreshold(
1717 DS: (Summary->getDetailedSummary()));
1718
1719 // Trim and merge cold context profile using cold threshold above;
1720 SampleContextTrimmer(ProfileMap)
1721 .trimAndMergeColdContextProfiles(
1722 ColdCountThreshold: SampleProfColdThreshold, TrimColdContext: SampleTrimColdContext,
1723 MergeColdContext: SampleMergeColdContext, ColdContextFrameLength: SampleColdContextFrameDepth, TrimBaseProfileOnly: false);
1724 }
1725
1726 if (ProfileLayout == llvm::sampleprof::SPL_Flat) {
1727 ProfileConverter::flattenProfile(ProfileMap, ProfileIsCS: FunctionSamples::ProfileIsCS);
1728 ProfileIsCS = FunctionSamples::ProfileIsCS = false;
1729 } else if (ProfileIsCS && ProfileLayout == llvm::sampleprof::SPL_Nest) {
1730 ProfileConverter CSConverter(ProfileMap);
1731 CSConverter.convertCSProfiles();
1732 ProfileIsCS = FunctionSamples::ProfileIsCS = false;
1733 }
1734
1735 filterFunctions(ProfileMap);
1736
1737 auto WriterOrErr =
1738 SampleProfileWriter::create(Filename: OutputFilename, Format: FormatMap[OutputFormat]);
1739 if (std::error_code EC = WriterOrErr.getError())
1740 exitWithErrorCode(EC, Whence: OutputFilename);
1741
1742 auto Writer = std::move(WriterOrErr.get());
1743 // WriterList will have StringRef refering to string in Buffer.
1744 // Make sure Buffer lives as long as WriterList.
1745 auto Buffer = getInputFileBuf(InputFile: ProfileSymbolListFile);
1746 handleExtBinaryWriter(Writer&: *Writer, OutputFormat, Buffer: Buffer.get(), WriterList,
1747 CompressAllSections, UseMD5, GenPartialProfile);
1748
1749 // If OutputSizeLimit is 0 (default), it is the same as write().
1750 if (std::error_code EC =
1751 Writer->writeWithSizeLimit(ProfileMap, OutputSizeLimit))
1752 exitWithErrorCode(EC);
1753}
1754
1755static WeightedFile parseWeightedFile(const StringRef &WeightedFilename) {
1756 StringRef WeightStr, FileName;
1757 std::tie(args&: WeightStr, args&: FileName) = WeightedFilename.split(Separator: ',');
1758
1759 uint64_t Weight;
1760 if (WeightStr.getAsInteger(Radix: 10, Result&: Weight) || Weight < 1)
1761 exitWithError(Message: "input weight must be a positive integer");
1762
1763 llvm::SmallString<128> ResolvedFileName;
1764 llvm::sys::fs::expand_tilde(path: FileName, output&: ResolvedFileName);
1765
1766 return {.Filename: std::string(ResolvedFileName), .Weight: Weight};
1767}
1768
1769static void addWeightedInput(WeightedFileVector &WNI, const WeightedFile &WF) {
1770 StringRef Filename = WF.Filename;
1771 uint64_t Weight = WF.Weight;
1772
1773 // If it's STDIN just pass it on.
1774 if (Filename == "-") {
1775 WNI.push_back(Elt: {.Filename: std::string(Filename), .Weight: Weight});
1776 return;
1777 }
1778
1779 llvm::sys::fs::file_status Status;
1780 llvm::sys::fs::status(path: Filename, result&: Status);
1781 if (!llvm::sys::fs::exists(status: Status))
1782 exitWithErrorCode(EC: make_error_code(E: errc::no_such_file_or_directory),
1783 Whence: Filename);
1784 // If it's a source file, collect it.
1785 if (llvm::sys::fs::is_regular_file(status: Status)) {
1786 WNI.push_back(Elt: {.Filename: std::string(Filename), .Weight: Weight});
1787 return;
1788 }
1789
1790 if (llvm::sys::fs::is_directory(status: Status)) {
1791 std::error_code EC;
1792 for (llvm::sys::fs::recursive_directory_iterator F(Filename, EC), E;
1793 F != E && !EC; F.increment(ec&: EC)) {
1794 if (llvm::sys::fs::is_regular_file(Path: F->path())) {
1795 addWeightedInput(WNI, WF: {.Filename: F->path(), .Weight: Weight});
1796 }
1797 }
1798 if (EC)
1799 exitWithErrorCode(EC, Whence: Filename);
1800 }
1801}
1802
1803static void parseInputFilenamesFile(MemoryBuffer *Buffer,
1804 WeightedFileVector &WFV) {
1805 if (!Buffer)
1806 return;
1807
1808 SmallVector<StringRef, 8> Entries;
1809 StringRef Data = Buffer->getBuffer();
1810 Data.split(A&: Entries, Separator: '\n', /*MaxSplit=*/-1, /*KeepEmpty=*/false);
1811 for (const StringRef &FileWeightEntry : Entries) {
1812 StringRef SanitizedEntry = FileWeightEntry.trim(Chars: " \t\v\f\r");
1813 // Skip comments.
1814 if (SanitizedEntry.starts_with(Prefix: "#"))
1815 continue;
1816 // If there's no comma, it's an unweighted profile.
1817 else if (!SanitizedEntry.contains(C: ','))
1818 addWeightedInput(WNI&: WFV, WF: {.Filename: std::string(SanitizedEntry), .Weight: 1});
1819 else
1820 addWeightedInput(WNI&: WFV, WF: parseWeightedFile(WeightedFilename: SanitizedEntry));
1821 }
1822}
1823
1824static int merge_main(StringRef ProgName) {
1825 WeightedFileVector WeightedInputs;
1826 for (StringRef Filename : InputFilenames)
1827 addWeightedInput(WNI&: WeightedInputs, WF: {.Filename: std::string(Filename), .Weight: 1});
1828 for (StringRef WeightedFilename : WeightedInputFilenames)
1829 addWeightedInput(WNI&: WeightedInputs, WF: parseWeightedFile(WeightedFilename));
1830
1831 // Make sure that the file buffer stays alive for the duration of the
1832 // weighted input vector's lifetime.
1833 auto Buffer = getInputFileBuf(InputFile: InputFilenamesFile);
1834 parseInputFilenamesFile(Buffer: Buffer.get(), WFV&: WeightedInputs);
1835
1836 if (WeightedInputs.empty())
1837 exitWithError(Message: "no input files specified. See " + ProgName + " merge -help");
1838
1839 if (DumpInputFileList) {
1840 for (auto &WF : WeightedInputs)
1841 outs() << WF.Weight << "," << WF.Filename << "\n";
1842 return 0;
1843 }
1844
1845 std::unique_ptr<SymbolRemapper> Remapper;
1846 if (!RemappingFile.empty())
1847 Remapper = SymbolRemapper::create(InputFile: RemappingFile);
1848
1849 if (!SupplInstrWithSample.empty()) {
1850 if (ProfileKind != instr)
1851 exitWithError(
1852 Message: "-supplement-instr-with-sample can only work with -instr. ");
1853
1854 supplementInstrProfile(Inputs: WeightedInputs, SampleFilename: SupplInstrWithSample, OutputSparse,
1855 SupplMinSizeThreshold, ZeroCounterThreshold,
1856 InstrProfColdThreshold);
1857 return 0;
1858 }
1859
1860 if (ProfileKind == instr)
1861 mergeInstrProfile(Inputs: WeightedInputs, Remapper: Remapper.get(), MaxDbgCorrelationWarnings,
1862 ProfiledBinary);
1863 else
1864 mergeSampleProfile(Inputs: WeightedInputs, Remapper: Remapper.get(), ProfileSymbolListFile,
1865 OutputSizeLimit);
1866 return 0;
1867}
1868
1869/// Computer the overlap b/w profile BaseFilename and profile TestFilename.
1870static void overlapInstrProfile(const std::string &BaseFilename,
1871 const std::string &TestFilename,
1872 const OverlapFuncFilters &FuncFilter,
1873 raw_fd_ostream &OS, bool IsCS) {
1874 std::mutex ErrorLock;
1875 SmallSet<instrprof_error, 4> WriterErrorCodes;
1876 WriterContext Context(false, ErrorLock, WriterErrorCodes);
1877 WeightedFile WeightedInput{.Filename: BaseFilename, .Weight: 1};
1878 OverlapStats Overlap;
1879 Error E = Overlap.accumulateCounts(BaseFilename, TestFilename, IsCS);
1880 if (E)
1881 exitWithError(E: std::move(E), Whence: "error in getting profile count sums");
1882 if (Overlap.Base.CountSum < 1.0f) {
1883 OS << "Sum of edge counts for profile " << BaseFilename << " is 0.\n";
1884 exit(status: 0);
1885 }
1886 if (Overlap.Test.CountSum < 1.0f) {
1887 OS << "Sum of edge counts for profile " << TestFilename << " is 0.\n";
1888 exit(status: 0);
1889 }
1890 loadInput(Input: WeightedInput, Remapper: nullptr, Correlator: nullptr, /*ProfiledBinary=*/"", WC: &Context);
1891 overlapInput(BaseFilename, TestFilename, WC: &Context, Overlap, FuncFilter, OS,
1892 IsCS);
1893 Overlap.dump(OS);
1894}
1895
1896namespace {
1897struct SampleOverlapStats {
1898 SampleContext BaseName;
1899 SampleContext TestName;
1900 // Number of overlap units
1901 uint64_t OverlapCount = 0;
1902 // Total samples of overlap units
1903 uint64_t OverlapSample = 0;
1904 // Number of and total samples of units that only present in base or test
1905 // profile
1906 uint64_t BaseUniqueCount = 0;
1907 uint64_t BaseUniqueSample = 0;
1908 uint64_t TestUniqueCount = 0;
1909 uint64_t TestUniqueSample = 0;
1910 // Number of units and total samples in base or test profile
1911 uint64_t BaseCount = 0;
1912 uint64_t BaseSample = 0;
1913 uint64_t TestCount = 0;
1914 uint64_t TestSample = 0;
1915 // Number of and total samples of units that present in at least one profile
1916 uint64_t UnionCount = 0;
1917 uint64_t UnionSample = 0;
1918 // Weighted similarity
1919 double Similarity = 0.0;
1920 // For SampleOverlapStats instances representing functions, weights of the
1921 // function in base and test profiles
1922 double BaseWeight = 0.0;
1923 double TestWeight = 0.0;
1924
1925 SampleOverlapStats() = default;
1926};
1927} // end anonymous namespace
1928
1929namespace {
1930struct FuncSampleStats {
1931 uint64_t SampleSum = 0;
1932 uint64_t MaxSample = 0;
1933 uint64_t HotBlockCount = 0;
1934 FuncSampleStats() = default;
1935 FuncSampleStats(uint64_t SampleSum, uint64_t MaxSample,
1936 uint64_t HotBlockCount)
1937 : SampleSum(SampleSum), MaxSample(MaxSample),
1938 HotBlockCount(HotBlockCount) {}
1939};
1940} // end anonymous namespace
1941
1942namespace {
1943enum MatchStatus { MS_Match, MS_FirstUnique, MS_SecondUnique, MS_None };
1944
1945// Class for updating merging steps for two sorted maps. The class should be
1946// instantiated with a map iterator type.
1947template <class T> class MatchStep {
1948public:
1949 MatchStep() = delete;
1950
1951 MatchStep(T FirstIter, T FirstEnd, T SecondIter, T SecondEnd)
1952 : FirstIter(FirstIter), FirstEnd(FirstEnd), SecondIter(SecondIter),
1953 SecondEnd(SecondEnd), Status(MS_None) {}
1954
1955 bool areBothFinished() const {
1956 return (FirstIter == FirstEnd && SecondIter == SecondEnd);
1957 }
1958
1959 bool isFirstFinished() const { return FirstIter == FirstEnd; }
1960
1961 bool isSecondFinished() const { return SecondIter == SecondEnd; }
1962
1963 /// Advance one step based on the previous match status unless the previous
1964 /// status is MS_None. Then update Status based on the comparison between two
1965 /// container iterators at the current step. If the previous status is
1966 /// MS_None, it means two iterators are at the beginning and no comparison has
1967 /// been made, so we simply update Status without advancing the iterators.
1968 void updateOneStep();
1969
1970 T getFirstIter() const { return FirstIter; }
1971
1972 T getSecondIter() const { return SecondIter; }
1973
1974 MatchStatus getMatchStatus() const { return Status; }
1975
1976private:
1977 // Current iterator and end iterator of the first container.
1978 T FirstIter;
1979 T FirstEnd;
1980 // Current iterator and end iterator of the second container.
1981 T SecondIter;
1982 T SecondEnd;
1983 // Match status of the current step.
1984 MatchStatus Status;
1985};
1986} // end anonymous namespace
1987
1988template <class T> void MatchStep<T>::updateOneStep() {
1989 switch (Status) {
1990 case MS_Match:
1991 ++FirstIter;
1992 ++SecondIter;
1993 break;
1994 case MS_FirstUnique:
1995 ++FirstIter;
1996 break;
1997 case MS_SecondUnique:
1998 ++SecondIter;
1999 break;
2000 case MS_None:
2001 break;
2002 }
2003
2004 // Update Status according to iterators at the current step.
2005 if (areBothFinished())
2006 return;
2007 if (FirstIter != FirstEnd &&
2008 (SecondIter == SecondEnd || FirstIter->first < SecondIter->first))
2009 Status = MS_FirstUnique;
2010 else if (SecondIter != SecondEnd &&
2011 (FirstIter == FirstEnd || SecondIter->first < FirstIter->first))
2012 Status = MS_SecondUnique;
2013 else
2014 Status = MS_Match;
2015}
2016
2017// Return the sum of line/block samples, the max line/block sample, and the
2018// number of line/block samples above the given threshold in a function
2019// including its inlinees.
2020static void getFuncSampleStats(const sampleprof::FunctionSamples &Func,
2021 FuncSampleStats &FuncStats,
2022 uint64_t HotThreshold) {
2023 for (const auto &L : Func.getBodySamples()) {
2024 uint64_t Sample = L.second.getSamples();
2025 FuncStats.SampleSum += Sample;
2026 FuncStats.MaxSample = std::max(a: FuncStats.MaxSample, b: Sample);
2027 if (Sample >= HotThreshold)
2028 ++FuncStats.HotBlockCount;
2029 }
2030
2031 for (const auto &C : Func.getCallsiteSamples()) {
2032 for (const auto &F : C.second)
2033 getFuncSampleStats(Func: F.second, FuncStats, HotThreshold);
2034 }
2035}
2036
2037/// Predicate that determines if a function is hot with a given threshold. We
2038/// keep it separate from its callsites for possible extension in the future.
2039static bool isFunctionHot(const FuncSampleStats &FuncStats,
2040 uint64_t HotThreshold) {
2041 // We intentionally compare the maximum sample count in a function with the
2042 // HotThreshold to get an approximate determination on hot functions.
2043 return (FuncStats.MaxSample >= HotThreshold);
2044}
2045
2046namespace {
2047class SampleOverlapAggregator {
2048public:
2049 SampleOverlapAggregator(const std::string &BaseFilename,
2050 const std::string &TestFilename,
2051 double LowSimilarityThreshold, double Epsilon,
2052 const OverlapFuncFilters &FuncFilter)
2053 : BaseFilename(BaseFilename), TestFilename(TestFilename),
2054 LowSimilarityThreshold(LowSimilarityThreshold), Epsilon(Epsilon),
2055 FuncFilter(FuncFilter) {}
2056
2057 /// Detect 0-sample input profile and report to output stream. This interface
2058 /// should be called after loadProfiles().
2059 bool detectZeroSampleProfile(raw_fd_ostream &OS) const;
2060
2061 /// Write out function-level similarity statistics for functions specified by
2062 /// options --function, --value-cutoff, and --similarity-cutoff.
2063 void dumpFuncSimilarity(raw_fd_ostream &OS) const;
2064
2065 /// Write out program-level similarity and overlap statistics.
2066 void dumpProgramSummary(raw_fd_ostream &OS) const;
2067
2068 /// Write out hot-function and hot-block statistics for base_profile,
2069 /// test_profile, and their overlap. For both cases, the overlap HO is
2070 /// calculated as follows:
2071 /// Given the number of functions (or blocks) that are hot in both profiles
2072 /// HCommon and the number of functions (or blocks) that are hot in at
2073 /// least one profile HUnion, HO = HCommon / HUnion.
2074 void dumpHotFuncAndBlockOverlap(raw_fd_ostream &OS) const;
2075
2076 /// This function tries matching functions in base and test profiles. For each
2077 /// pair of matched functions, it aggregates the function-level
2078 /// similarity into a profile-level similarity. It also dump function-level
2079 /// similarity information of functions specified by --function,
2080 /// --value-cutoff, and --similarity-cutoff options. The program-level
2081 /// similarity PS is computed as follows:
2082 /// Given function-level similarity FS(A) for all function A, the
2083 /// weight of function A in base profile WB(A), and the weight of function
2084 /// A in test profile WT(A), compute PS(base_profile, test_profile) =
2085 /// sum_A(FS(A) * avg(WB(A), WT(A))) ranging in [0.0f to 1.0f] with 0.0
2086 /// meaning no-overlap.
2087 void computeSampleProfileOverlap(raw_fd_ostream &OS);
2088
2089 /// Initialize ProfOverlap with the sum of samples in base and test
2090 /// profiles. This function also computes and keeps the sum of samples and
2091 /// max sample counts of each function in BaseStats and TestStats for later
2092 /// use to avoid re-computations.
2093 void initializeSampleProfileOverlap();
2094
2095 /// Load profiles specified by BaseFilename and TestFilename.
2096 std::error_code loadProfiles();
2097
2098 using FuncSampleStatsMap = DenseMap<SampleContext, FuncSampleStats>;
2099
2100private:
2101 SampleOverlapStats ProfOverlap;
2102 SampleOverlapStats HotFuncOverlap;
2103 SampleOverlapStats HotBlockOverlap;
2104 std::string BaseFilename;
2105 std::string TestFilename;
2106 std::unique_ptr<sampleprof::SampleProfileReader> BaseReader;
2107 std::unique_ptr<sampleprof::SampleProfileReader> TestReader;
2108 // BaseStats and TestStats hold FuncSampleStats for each function, with
2109 // function name as the key.
2110 FuncSampleStatsMap BaseStats;
2111 FuncSampleStatsMap TestStats;
2112 // Low similarity threshold in floating point number
2113 double LowSimilarityThreshold;
2114 // Block samples above BaseHotThreshold or TestHotThreshold are considered hot
2115 // for tracking hot blocks.
2116 uint64_t BaseHotThreshold;
2117 uint64_t TestHotThreshold;
2118 // A small threshold used to round the results of floating point accumulations
2119 // to resolve imprecision.
2120 const double Epsilon;
2121 std::multimap<double, SampleOverlapStats, std::greater<double>>
2122 FuncSimilarityDump;
2123 // FuncFilter carries specifications in options --value-cutoff and
2124 // --function.
2125 OverlapFuncFilters FuncFilter;
2126 // Column offsets for printing the function-level details table.
2127 static const unsigned int TestWeightCol = 15;
2128 static const unsigned int SimilarityCol = 30;
2129 static const unsigned int OverlapCol = 43;
2130 static const unsigned int BaseUniqueCol = 53;
2131 static const unsigned int TestUniqueCol = 67;
2132 static const unsigned int BaseSampleCol = 81;
2133 static const unsigned int TestSampleCol = 96;
2134 static const unsigned int FuncNameCol = 111;
2135
2136 /// Return a similarity of two line/block sample counters in the same
2137 /// function in base and test profiles. The line/block-similarity BS(i) is
2138 /// computed as follows:
2139 /// For an offsets i, given the sample count at i in base profile BB(i),
2140 /// the sample count at i in test profile BT(i), the sum of sample counts
2141 /// in this function in base profile SB, and the sum of sample counts in
2142 /// this function in test profile ST, compute BS(i) = 1.0 - fabs(BB(i)/SB -
2143 /// BT(i)/ST), ranging in [0.0f to 1.0f] with 0.0 meaning no-overlap.
2144 double computeBlockSimilarity(uint64_t BaseSample, uint64_t TestSample,
2145 const SampleOverlapStats &FuncOverlap) const;
2146
2147 void updateHotBlockOverlap(uint64_t BaseSample, uint64_t TestSample,
2148 uint64_t HotBlockCount);
2149
2150 void getHotFunctions(const FuncSampleStatsMap &ProfStats,
2151 FuncSampleStatsMap &HotFunc,
2152 uint64_t HotThreshold) const;
2153
2154 void computeHotFuncOverlap();
2155
2156 /// This function updates statistics in FuncOverlap, HotBlockOverlap, and
2157 /// Difference for two sample units in a matched function according to the
2158 /// given match status.
2159 void updateOverlapStatsForFunction(uint64_t BaseSample, uint64_t TestSample,
2160 uint64_t HotBlockCount,
2161 SampleOverlapStats &FuncOverlap,
2162 double &Difference, MatchStatus Status);
2163
2164 /// This function updates statistics in FuncOverlap, HotBlockOverlap, and
2165 /// Difference for unmatched callees that only present in one profile in a
2166 /// matched caller function.
2167 void updateForUnmatchedCallee(const sampleprof::FunctionSamples &Func,
2168 SampleOverlapStats &FuncOverlap,
2169 double &Difference, MatchStatus Status);
2170
2171 /// This function updates sample overlap statistics of an overlap function in
2172 /// base and test profile. It also calculates a function-internal similarity
2173 /// FIS as follows:
2174 /// For offsets i that have samples in at least one profile in this
2175 /// function A, given BS(i) returned by computeBlockSimilarity(), compute
2176 /// FIS(A) = (2.0 - sum_i(1.0 - BS(i))) / 2, ranging in [0.0f to 1.0f] with
2177 /// 0.0 meaning no overlap.
2178 double computeSampleFunctionInternalOverlap(
2179 const sampleprof::FunctionSamples &BaseFunc,
2180 const sampleprof::FunctionSamples &TestFunc,
2181 SampleOverlapStats &FuncOverlap);
2182
2183 /// Function-level similarity (FS) is a weighted value over function internal
2184 /// similarity (FIS). This function computes a function's FS from its FIS by
2185 /// applying the weight.
2186 double weightForFuncSimilarity(double FuncSimilarity, uint64_t BaseFuncSample,
2187 uint64_t TestFuncSample) const;
2188
2189 /// The function-level similarity FS(A) for a function A is computed as
2190 /// follows:
2191 /// Compute a function-internal similarity FIS(A) by
2192 /// computeSampleFunctionInternalOverlap(). Then, with the weight of
2193 /// function A in base profile WB(A), and the weight of function A in test
2194 /// profile WT(A), compute FS(A) = FIS(A) * (1.0 - fabs(WB(A) - WT(A)))
2195 /// ranging in [0.0f to 1.0f] with 0.0 meaning no overlap.
2196 double
2197 computeSampleFunctionOverlap(const sampleprof::FunctionSamples *BaseFunc,
2198 const sampleprof::FunctionSamples *TestFunc,
2199 SampleOverlapStats *FuncOverlap,
2200 uint64_t BaseFuncSample,
2201 uint64_t TestFuncSample);
2202
2203 /// Profile-level similarity (PS) is a weighted aggregate over function-level
2204 /// similarities (FS). This method weights the FS value by the function
2205 /// weights in the base and test profiles for the aggregation.
2206 double weightByImportance(double FuncSimilarity, uint64_t BaseFuncSample,
2207 uint64_t TestFuncSample) const;
2208};
2209} // end anonymous namespace
2210
2211bool SampleOverlapAggregator::detectZeroSampleProfile(
2212 raw_fd_ostream &OS) const {
2213 bool HaveZeroSample = false;
2214 if (ProfOverlap.BaseSample == 0) {
2215 OS << "Sum of sample counts for profile " << BaseFilename << " is 0.\n";
2216 HaveZeroSample = true;
2217 }
2218 if (ProfOverlap.TestSample == 0) {
2219 OS << "Sum of sample counts for profile " << TestFilename << " is 0.\n";
2220 HaveZeroSample = true;
2221 }
2222 return HaveZeroSample;
2223}
2224
2225double SampleOverlapAggregator::computeBlockSimilarity(
2226 uint64_t BaseSample, uint64_t TestSample,
2227 const SampleOverlapStats &FuncOverlap) const {
2228 double BaseFrac = 0.0;
2229 double TestFrac = 0.0;
2230 if (FuncOverlap.BaseSample > 0)
2231 BaseFrac = static_cast<double>(BaseSample) / FuncOverlap.BaseSample;
2232 if (FuncOverlap.TestSample > 0)
2233 TestFrac = static_cast<double>(TestSample) / FuncOverlap.TestSample;
2234 return 1.0 - std::fabs(x: BaseFrac - TestFrac);
2235}
2236
2237void SampleOverlapAggregator::updateHotBlockOverlap(uint64_t BaseSample,
2238 uint64_t TestSample,
2239 uint64_t HotBlockCount) {
2240 bool IsBaseHot = (BaseSample >= BaseHotThreshold);
2241 bool IsTestHot = (TestSample >= TestHotThreshold);
2242 if (!IsBaseHot && !IsTestHot)
2243 return;
2244
2245 HotBlockOverlap.UnionCount += HotBlockCount;
2246 if (IsBaseHot)
2247 HotBlockOverlap.BaseCount += HotBlockCount;
2248 if (IsTestHot)
2249 HotBlockOverlap.TestCount += HotBlockCount;
2250 if (IsBaseHot && IsTestHot)
2251 HotBlockOverlap.OverlapCount += HotBlockCount;
2252}
2253
2254void SampleOverlapAggregator::getHotFunctions(
2255 const FuncSampleStatsMap &ProfStats, FuncSampleStatsMap &HotFunc,
2256 uint64_t HotThreshold) const {
2257 for (const auto &F : ProfStats) {
2258 if (isFunctionHot(FuncStats: F.second, HotThreshold))
2259 HotFunc.try_emplace(Key: F.first, Args: F.second);
2260 }
2261}
2262
2263void SampleOverlapAggregator::computeHotFuncOverlap() {
2264 FuncSampleStatsMap BaseHotFunc;
2265 getHotFunctions(ProfStats: BaseStats, HotFunc&: BaseHotFunc, HotThreshold: BaseHotThreshold);
2266 HotFuncOverlap.BaseCount = BaseHotFunc.size();
2267
2268 FuncSampleStatsMap TestHotFunc;
2269 getHotFunctions(ProfStats: TestStats, HotFunc&: TestHotFunc, HotThreshold: TestHotThreshold);
2270 HotFuncOverlap.TestCount = TestHotFunc.size();
2271 HotFuncOverlap.UnionCount = HotFuncOverlap.TestCount;
2272
2273 for (const auto &F : BaseHotFunc) {
2274 if (TestHotFunc.count(Val: F.first))
2275 ++HotFuncOverlap.OverlapCount;
2276 else
2277 ++HotFuncOverlap.UnionCount;
2278 }
2279}
2280
2281void SampleOverlapAggregator::updateOverlapStatsForFunction(
2282 uint64_t BaseSample, uint64_t TestSample, uint64_t HotBlockCount,
2283 SampleOverlapStats &FuncOverlap, double &Difference, MatchStatus Status) {
2284 assert(Status != MS_None &&
2285 "Match status should be updated before updating overlap statistics");
2286 if (Status == MS_FirstUnique) {
2287 TestSample = 0;
2288 FuncOverlap.BaseUniqueSample += BaseSample;
2289 } else if (Status == MS_SecondUnique) {
2290 BaseSample = 0;
2291 FuncOverlap.TestUniqueSample += TestSample;
2292 } else {
2293 ++FuncOverlap.OverlapCount;
2294 }
2295
2296 FuncOverlap.UnionSample += std::max(a: BaseSample, b: TestSample);
2297 FuncOverlap.OverlapSample += std::min(a: BaseSample, b: TestSample);
2298 Difference +=
2299 1.0 - computeBlockSimilarity(BaseSample, TestSample, FuncOverlap);
2300 updateHotBlockOverlap(BaseSample, TestSample, HotBlockCount);
2301}
2302
2303void SampleOverlapAggregator::updateForUnmatchedCallee(
2304 const sampleprof::FunctionSamples &Func, SampleOverlapStats &FuncOverlap,
2305 double &Difference, MatchStatus Status) {
2306 assert((Status == MS_FirstUnique || Status == MS_SecondUnique) &&
2307 "Status must be either of the two unmatched cases");
2308 FuncSampleStats FuncStats;
2309 if (Status == MS_FirstUnique) {
2310 getFuncSampleStats(Func, FuncStats, HotThreshold: BaseHotThreshold);
2311 updateOverlapStatsForFunction(BaseSample: FuncStats.SampleSum, TestSample: 0,
2312 HotBlockCount: FuncStats.HotBlockCount, FuncOverlap,
2313 Difference, Status);
2314 } else {
2315 getFuncSampleStats(Func, FuncStats, HotThreshold: TestHotThreshold);
2316 updateOverlapStatsForFunction(BaseSample: 0, TestSample: FuncStats.SampleSum,
2317 HotBlockCount: FuncStats.HotBlockCount, FuncOverlap,
2318 Difference, Status);
2319 }
2320}
2321
2322double SampleOverlapAggregator::computeSampleFunctionInternalOverlap(
2323 const sampleprof::FunctionSamples &BaseFunc,
2324 const sampleprof::FunctionSamples &TestFunc,
2325 SampleOverlapStats &FuncOverlap) {
2326
2327 using namespace sampleprof;
2328
2329 double Difference = 0;
2330
2331 // Accumulate Difference for regular line/block samples in the function.
2332 // We match them through sort-merge join algorithm because
2333 // FunctionSamples::getBodySamples() returns a map of sample counters ordered
2334 // by their offsets.
2335 MatchStep<BodySampleMap::const_iterator> BlockIterStep(
2336 BaseFunc.getBodySamples().cbegin(), BaseFunc.getBodySamples().cend(),
2337 TestFunc.getBodySamples().cbegin(), TestFunc.getBodySamples().cend());
2338 BlockIterStep.updateOneStep();
2339 while (!BlockIterStep.areBothFinished()) {
2340 uint64_t BaseSample =
2341 BlockIterStep.isFirstFinished()
2342 ? 0
2343 : BlockIterStep.getFirstIter()->second.getSamples();
2344 uint64_t TestSample =
2345 BlockIterStep.isSecondFinished()
2346 ? 0
2347 : BlockIterStep.getSecondIter()->second.getSamples();
2348 updateOverlapStatsForFunction(BaseSample, TestSample, HotBlockCount: 1, FuncOverlap,
2349 Difference, Status: BlockIterStep.getMatchStatus());
2350
2351 BlockIterStep.updateOneStep();
2352 }
2353
2354 // Accumulate Difference for callsite lines in the function. We match
2355 // them through sort-merge algorithm because
2356 // FunctionSamples::getCallsiteSamples() returns a map of callsite records
2357 // ordered by their offsets.
2358 MatchStep<CallsiteSampleMap::const_iterator> CallsiteIterStep(
2359 BaseFunc.getCallsiteSamples().cbegin(),
2360 BaseFunc.getCallsiteSamples().cend(),
2361 TestFunc.getCallsiteSamples().cbegin(),
2362 TestFunc.getCallsiteSamples().cend());
2363 CallsiteIterStep.updateOneStep();
2364 while (!CallsiteIterStep.areBothFinished()) {
2365 MatchStatus CallsiteStepStatus = CallsiteIterStep.getMatchStatus();
2366 assert(CallsiteStepStatus != MS_None &&
2367 "Match status should be updated before entering loop body");
2368
2369 if (CallsiteStepStatus != MS_Match) {
2370 auto Callsite = (CallsiteStepStatus == MS_FirstUnique)
2371 ? CallsiteIterStep.getFirstIter()
2372 : CallsiteIterStep.getSecondIter();
2373 for (const auto &F : Callsite->second)
2374 updateForUnmatchedCallee(Func: F.second, FuncOverlap, Difference,
2375 Status: CallsiteStepStatus);
2376 } else {
2377 // There may be multiple inlinees at the same offset, so we need to try
2378 // matching all of them. This match is implemented through sort-merge
2379 // algorithm because callsite records at the same offset are ordered by
2380 // function names.
2381 MatchStep<FunctionSamplesMap::const_iterator> CalleeIterStep(
2382 CallsiteIterStep.getFirstIter()->second.cbegin(),
2383 CallsiteIterStep.getFirstIter()->second.cend(),
2384 CallsiteIterStep.getSecondIter()->second.cbegin(),
2385 CallsiteIterStep.getSecondIter()->second.cend());
2386 CalleeIterStep.updateOneStep();
2387 while (!CalleeIterStep.areBothFinished()) {
2388 MatchStatus CalleeStepStatus = CalleeIterStep.getMatchStatus();
2389 if (CalleeStepStatus != MS_Match) {
2390 auto Callee = (CalleeStepStatus == MS_FirstUnique)
2391 ? CalleeIterStep.getFirstIter()
2392 : CalleeIterStep.getSecondIter();
2393 updateForUnmatchedCallee(Func: Callee->second, FuncOverlap, Difference,
2394 Status: CalleeStepStatus);
2395 } else {
2396 // An inlined function can contain other inlinees inside, so compute
2397 // the Difference recursively.
2398 Difference += 2.0 - 2 * computeSampleFunctionInternalOverlap(
2399 BaseFunc: CalleeIterStep.getFirstIter()->second,
2400 TestFunc: CalleeIterStep.getSecondIter()->second,
2401 FuncOverlap);
2402 }
2403 CalleeIterStep.updateOneStep();
2404 }
2405 }
2406 CallsiteIterStep.updateOneStep();
2407 }
2408
2409 // Difference reflects the total differences of line/block samples in this
2410 // function and ranges in [0.0f to 2.0f]. Take (2.0 - Difference) / 2 to
2411 // reflect the similarity between function profiles in [0.0f to 1.0f].
2412 return (2.0 - Difference) / 2;
2413}
2414
2415double SampleOverlapAggregator::weightForFuncSimilarity(
2416 double FuncInternalSimilarity, uint64_t BaseFuncSample,
2417 uint64_t TestFuncSample) const {
2418 // Compute the weight as the distance between the function weights in two
2419 // profiles.
2420 double BaseFrac = 0.0;
2421 double TestFrac = 0.0;
2422 assert(ProfOverlap.BaseSample > 0 &&
2423 "Total samples in base profile should be greater than 0");
2424 BaseFrac = static_cast<double>(BaseFuncSample) / ProfOverlap.BaseSample;
2425 assert(ProfOverlap.TestSample > 0 &&
2426 "Total samples in test profile should be greater than 0");
2427 TestFrac = static_cast<double>(TestFuncSample) / ProfOverlap.TestSample;
2428 double WeightDistance = std::fabs(x: BaseFrac - TestFrac);
2429
2430 // Take WeightDistance into the similarity.
2431 return FuncInternalSimilarity * (1 - WeightDistance);
2432}
2433
2434double
2435SampleOverlapAggregator::weightByImportance(double FuncSimilarity,
2436 uint64_t BaseFuncSample,
2437 uint64_t TestFuncSample) const {
2438
2439 double BaseFrac = 0.0;
2440 double TestFrac = 0.0;
2441 assert(ProfOverlap.BaseSample > 0 &&
2442 "Total samples in base profile should be greater than 0");
2443 BaseFrac = static_cast<double>(BaseFuncSample) / ProfOverlap.BaseSample / 2.0;
2444 assert(ProfOverlap.TestSample > 0 &&
2445 "Total samples in test profile should be greater than 0");
2446 TestFrac = static_cast<double>(TestFuncSample) / ProfOverlap.TestSample / 2.0;
2447 return FuncSimilarity * (BaseFrac + TestFrac);
2448}
2449
2450double SampleOverlapAggregator::computeSampleFunctionOverlap(
2451 const sampleprof::FunctionSamples *BaseFunc,
2452 const sampleprof::FunctionSamples *TestFunc,
2453 SampleOverlapStats *FuncOverlap, uint64_t BaseFuncSample,
2454 uint64_t TestFuncSample) {
2455 // Default function internal similarity before weighted, meaning two functions
2456 // has no overlap.
2457 const double DefaultFuncInternalSimilarity = 0;
2458 double FuncSimilarity;
2459 double FuncInternalSimilarity;
2460
2461 // If BaseFunc or TestFunc is nullptr, it means the functions do not overlap.
2462 // In this case, we use DefaultFuncInternalSimilarity as the function internal
2463 // similarity.
2464 if (!BaseFunc || !TestFunc) {
2465 FuncInternalSimilarity = DefaultFuncInternalSimilarity;
2466 } else {
2467 assert(FuncOverlap != nullptr &&
2468 "FuncOverlap should be provided in this case");
2469 FuncInternalSimilarity = computeSampleFunctionInternalOverlap(
2470 BaseFunc: *BaseFunc, TestFunc: *TestFunc, FuncOverlap&: *FuncOverlap);
2471 // Now, FuncInternalSimilarity may be a little less than 0 due to
2472 // imprecision of floating point accumulations. Make it zero if the
2473 // difference is below Epsilon.
2474 FuncInternalSimilarity = (std::fabs(x: FuncInternalSimilarity - 0) < Epsilon)
2475 ? 0
2476 : FuncInternalSimilarity;
2477 }
2478 FuncSimilarity = weightForFuncSimilarity(FuncInternalSimilarity,
2479 BaseFuncSample, TestFuncSample);
2480 return FuncSimilarity;
2481}
2482
2483void SampleOverlapAggregator::computeSampleProfileOverlap(raw_fd_ostream &OS) {
2484 using namespace sampleprof;
2485
2486 DenseMap<SampleContext, const FunctionSamples *> BaseFuncProf;
2487 const auto &BaseProfiles = BaseReader->getProfiles();
2488 for (const auto &BaseFunc : BaseProfiles) {
2489 BaseFuncProf.try_emplace(Key: BaseFunc.second.getContext(), Args: &(BaseFunc.second));
2490 }
2491 ProfOverlap.UnionCount = BaseFuncProf.size();
2492
2493 const auto &TestProfiles = TestReader->getProfiles();
2494 for (const auto &TestFunc : TestProfiles) {
2495 SampleOverlapStats FuncOverlap;
2496 FuncOverlap.TestName = TestFunc.second.getContext();
2497 assert(TestStats.count(FuncOverlap.TestName) &&
2498 "TestStats should have records for all functions in test profile "
2499 "except inlinees");
2500 FuncOverlap.TestSample = TestStats[FuncOverlap.TestName].SampleSum;
2501
2502 bool Matched = false;
2503 const auto Match = BaseFuncProf.find(Val: FuncOverlap.TestName);
2504 if (Match == BaseFuncProf.end()) {
2505 const FuncSampleStats &FuncStats = TestStats[FuncOverlap.TestName];
2506 ++ProfOverlap.TestUniqueCount;
2507 ProfOverlap.TestUniqueSample += FuncStats.SampleSum;
2508 FuncOverlap.TestUniqueSample = FuncStats.SampleSum;
2509
2510 updateHotBlockOverlap(BaseSample: 0, TestSample: FuncStats.SampleSum, HotBlockCount: FuncStats.HotBlockCount);
2511
2512 double FuncSimilarity = computeSampleFunctionOverlap(
2513 BaseFunc: nullptr, TestFunc: nullptr, FuncOverlap: nullptr, BaseFuncSample: 0, TestFuncSample: FuncStats.SampleSum);
2514 ProfOverlap.Similarity +=
2515 weightByImportance(FuncSimilarity, BaseFuncSample: 0, TestFuncSample: FuncStats.SampleSum);
2516
2517 ++ProfOverlap.UnionCount;
2518 ProfOverlap.UnionSample += FuncStats.SampleSum;
2519 } else {
2520 ++ProfOverlap.OverlapCount;
2521
2522 // Two functions match with each other. Compute function-level overlap and
2523 // aggregate them into profile-level overlap.
2524 FuncOverlap.BaseName = Match->second->getContext();
2525 assert(BaseStats.count(FuncOverlap.BaseName) &&
2526 "BaseStats should have records for all functions in base profile "
2527 "except inlinees");
2528 FuncOverlap.BaseSample = BaseStats[FuncOverlap.BaseName].SampleSum;
2529
2530 FuncOverlap.Similarity = computeSampleFunctionOverlap(
2531 BaseFunc: Match->second, TestFunc: &TestFunc.second, FuncOverlap: &FuncOverlap, BaseFuncSample: FuncOverlap.BaseSample,
2532 TestFuncSample: FuncOverlap.TestSample);
2533 ProfOverlap.Similarity +=
2534 weightByImportance(FuncSimilarity: FuncOverlap.Similarity, BaseFuncSample: FuncOverlap.BaseSample,
2535 TestFuncSample: FuncOverlap.TestSample);
2536 ProfOverlap.OverlapSample += FuncOverlap.OverlapSample;
2537 ProfOverlap.UnionSample += FuncOverlap.UnionSample;
2538
2539 // Accumulate the percentage of base unique and test unique samples into
2540 // ProfOverlap.
2541 ProfOverlap.BaseUniqueSample += FuncOverlap.BaseUniqueSample;
2542 ProfOverlap.TestUniqueSample += FuncOverlap.TestUniqueSample;
2543
2544 // Remove matched base functions for later reporting functions not found
2545 // in test profile.
2546 BaseFuncProf.erase(I: Match);
2547 Matched = true;
2548 }
2549
2550 // Print function-level similarity information if specified by options.
2551 assert(TestStats.count(FuncOverlap.TestName) &&
2552 "TestStats should have records for all functions in test profile "
2553 "except inlinees");
2554 if (TestStats[FuncOverlap.TestName].MaxSample >= FuncFilter.ValueCutoff ||
2555 (Matched && FuncOverlap.Similarity < LowSimilarityThreshold) ||
2556 (Matched && !FuncFilter.NameFilter.empty() &&
2557 FuncOverlap.BaseName.toString().find(str: FuncFilter.NameFilter) !=
2558 std::string::npos)) {
2559 assert(ProfOverlap.BaseSample > 0 &&
2560 "Total samples in base profile should be greater than 0");
2561 FuncOverlap.BaseWeight =
2562 static_cast<double>(FuncOverlap.BaseSample) / ProfOverlap.BaseSample;
2563 assert(ProfOverlap.TestSample > 0 &&
2564 "Total samples in test profile should be greater than 0");
2565 FuncOverlap.TestWeight =
2566 static_cast<double>(FuncOverlap.TestSample) / ProfOverlap.TestSample;
2567 FuncSimilarityDump.emplace(args&: FuncOverlap.BaseWeight, args&: FuncOverlap);
2568 }
2569 }
2570
2571 // Traverse through functions in base profile but not in test profile.
2572 for (const auto &F : BaseFuncProf) {
2573 assert(BaseStats.count(F.second->getContext()) &&
2574 "BaseStats should have records for all functions in base profile "
2575 "except inlinees");
2576 const FuncSampleStats &FuncStats = BaseStats[F.second->getContext()];
2577 ++ProfOverlap.BaseUniqueCount;
2578 ProfOverlap.BaseUniqueSample += FuncStats.SampleSum;
2579
2580 updateHotBlockOverlap(BaseSample: FuncStats.SampleSum, TestSample: 0, HotBlockCount: FuncStats.HotBlockCount);
2581
2582 double FuncSimilarity = computeSampleFunctionOverlap(
2583 BaseFunc: nullptr, TestFunc: nullptr, FuncOverlap: nullptr, BaseFuncSample: FuncStats.SampleSum, TestFuncSample: 0);
2584 ProfOverlap.Similarity +=
2585 weightByImportance(FuncSimilarity, BaseFuncSample: FuncStats.SampleSum, TestFuncSample: 0);
2586
2587 ProfOverlap.UnionSample += FuncStats.SampleSum;
2588 }
2589
2590 // Now, ProfSimilarity may be a little greater than 1 due to imprecision
2591 // of floating point accumulations. Make it 1.0 if the difference is below
2592 // Epsilon.
2593 ProfOverlap.Similarity = (std::fabs(x: ProfOverlap.Similarity - 1) < Epsilon)
2594 ? 1
2595 : ProfOverlap.Similarity;
2596
2597 computeHotFuncOverlap();
2598}
2599
2600void SampleOverlapAggregator::initializeSampleProfileOverlap() {
2601 const auto &BaseProf = BaseReader->getProfiles();
2602 for (const auto &I : BaseProf) {
2603 ++ProfOverlap.BaseCount;
2604 FuncSampleStats FuncStats;
2605 getFuncSampleStats(Func: I.second, FuncStats, HotThreshold: BaseHotThreshold);
2606 ProfOverlap.BaseSample += FuncStats.SampleSum;
2607 BaseStats.try_emplace(Key: I.second.getContext(), Args&: FuncStats);
2608 }
2609
2610 const auto &TestProf = TestReader->getProfiles();
2611 for (const auto &I : TestProf) {
2612 ++ProfOverlap.TestCount;
2613 FuncSampleStats FuncStats;
2614 getFuncSampleStats(Func: I.second, FuncStats, HotThreshold: TestHotThreshold);
2615 ProfOverlap.TestSample += FuncStats.SampleSum;
2616 TestStats.try_emplace(Key: I.second.getContext(), Args&: FuncStats);
2617 }
2618
2619 ProfOverlap.BaseName = StringRef(BaseFilename);
2620 ProfOverlap.TestName = StringRef(TestFilename);
2621}
2622
2623void SampleOverlapAggregator::dumpFuncSimilarity(raw_fd_ostream &OS) const {
2624 using namespace sampleprof;
2625
2626 if (FuncSimilarityDump.empty())
2627 return;
2628
2629 formatted_raw_ostream FOS(OS);
2630 FOS << "Function-level details:\n";
2631 FOS << "Base weight";
2632 FOS.PadToColumn(NewCol: TestWeightCol);
2633 FOS << "Test weight";
2634 FOS.PadToColumn(NewCol: SimilarityCol);
2635 FOS << "Similarity";
2636 FOS.PadToColumn(NewCol: OverlapCol);
2637 FOS << "Overlap";
2638 FOS.PadToColumn(NewCol: BaseUniqueCol);
2639 FOS << "Base unique";
2640 FOS.PadToColumn(NewCol: TestUniqueCol);
2641 FOS << "Test unique";
2642 FOS.PadToColumn(NewCol: BaseSampleCol);
2643 FOS << "Base samples";
2644 FOS.PadToColumn(NewCol: TestSampleCol);
2645 FOS << "Test samples";
2646 FOS.PadToColumn(NewCol: FuncNameCol);
2647 FOS << "Function name\n";
2648 for (const auto &F : FuncSimilarityDump) {
2649 double OverlapPercent =
2650 F.second.UnionSample > 0
2651 ? static_cast<double>(F.second.OverlapSample) / F.second.UnionSample
2652 : 0;
2653 double BaseUniquePercent =
2654 F.second.BaseSample > 0
2655 ? static_cast<double>(F.second.BaseUniqueSample) /
2656 F.second.BaseSample
2657 : 0;
2658 double TestUniquePercent =
2659 F.second.TestSample > 0
2660 ? static_cast<double>(F.second.TestUniqueSample) /
2661 F.second.TestSample
2662 : 0;
2663
2664 FOS << format(Fmt: "%.2f%%", Vals: F.second.BaseWeight * 100);
2665 FOS.PadToColumn(NewCol: TestWeightCol);
2666 FOS << format(Fmt: "%.2f%%", Vals: F.second.TestWeight * 100);
2667 FOS.PadToColumn(NewCol: SimilarityCol);
2668 FOS << format(Fmt: "%.2f%%", Vals: F.second.Similarity * 100);
2669 FOS.PadToColumn(NewCol: OverlapCol);
2670 FOS << format(Fmt: "%.2f%%", Vals: OverlapPercent * 100);
2671 FOS.PadToColumn(NewCol: BaseUniqueCol);
2672 FOS << format(Fmt: "%.2f%%", Vals: BaseUniquePercent * 100);
2673 FOS.PadToColumn(NewCol: TestUniqueCol);
2674 FOS << format(Fmt: "%.2f%%", Vals: TestUniquePercent * 100);
2675 FOS.PadToColumn(NewCol: BaseSampleCol);
2676 FOS << F.second.BaseSample;
2677 FOS.PadToColumn(NewCol: TestSampleCol);
2678 FOS << F.second.TestSample;
2679 FOS.PadToColumn(NewCol: FuncNameCol);
2680 FOS << F.second.TestName.toString() << "\n";
2681 }
2682}
2683
2684void SampleOverlapAggregator::dumpProgramSummary(raw_fd_ostream &OS) const {
2685 OS << "Profile overlap information for base_profile: "
2686 << ProfOverlap.BaseName.toString()
2687 << " and test_profile: " << ProfOverlap.TestName.toString()
2688 << "\nProgram level:\n";
2689
2690 OS << " Whole program profile similarity: "
2691 << format(Fmt: "%.3f%%", Vals: ProfOverlap.Similarity * 100) << "\n";
2692
2693 assert(ProfOverlap.UnionSample > 0 &&
2694 "Total samples in two profile should be greater than 0");
2695 double OverlapPercent =
2696 static_cast<double>(ProfOverlap.OverlapSample) / ProfOverlap.UnionSample;
2697 assert(ProfOverlap.BaseSample > 0 &&
2698 "Total samples in base profile should be greater than 0");
2699 double BaseUniquePercent = static_cast<double>(ProfOverlap.BaseUniqueSample) /
2700 ProfOverlap.BaseSample;
2701 assert(ProfOverlap.TestSample > 0 &&
2702 "Total samples in test profile should be greater than 0");
2703 double TestUniquePercent = static_cast<double>(ProfOverlap.TestUniqueSample) /
2704 ProfOverlap.TestSample;
2705
2706 OS << " Whole program sample overlap: "
2707 << format(Fmt: "%.3f%%", Vals: OverlapPercent * 100) << "\n";
2708 OS << " percentage of samples unique in base profile: "
2709 << format(Fmt: "%.3f%%", Vals: BaseUniquePercent * 100) << "\n";
2710 OS << " percentage of samples unique in test profile: "
2711 << format(Fmt: "%.3f%%", Vals: TestUniquePercent * 100) << "\n";
2712 OS << " total samples in base profile: " << ProfOverlap.BaseSample << "\n"
2713 << " total samples in test profile: " << ProfOverlap.TestSample << "\n";
2714
2715 assert(ProfOverlap.UnionCount > 0 &&
2716 "There should be at least one function in two input profiles");
2717 double FuncOverlapPercent =
2718 static_cast<double>(ProfOverlap.OverlapCount) / ProfOverlap.UnionCount;
2719 OS << " Function overlap: " << format(Fmt: "%.3f%%", Vals: FuncOverlapPercent * 100)
2720 << "\n";
2721 OS << " overlap functions: " << ProfOverlap.OverlapCount << "\n";
2722 OS << " functions unique in base profile: " << ProfOverlap.BaseUniqueCount
2723 << "\n";
2724 OS << " functions unique in test profile: " << ProfOverlap.TestUniqueCount
2725 << "\n";
2726}
2727
2728void SampleOverlapAggregator::dumpHotFuncAndBlockOverlap(
2729 raw_fd_ostream &OS) const {
2730 assert(HotFuncOverlap.UnionCount > 0 &&
2731 "There should be at least one hot function in two input profiles");
2732 OS << " Hot-function overlap: "
2733 << format(Fmt: "%.3f%%", Vals: static_cast<double>(HotFuncOverlap.OverlapCount) /
2734 HotFuncOverlap.UnionCount * 100)
2735 << "\n";
2736 OS << " overlap hot functions: " << HotFuncOverlap.OverlapCount << "\n";
2737 OS << " hot functions unique in base profile: "
2738 << HotFuncOverlap.BaseCount - HotFuncOverlap.OverlapCount << "\n";
2739 OS << " hot functions unique in test profile: "
2740 << HotFuncOverlap.TestCount - HotFuncOverlap.OverlapCount << "\n";
2741
2742 assert(HotBlockOverlap.UnionCount > 0 &&
2743 "There should be at least one hot block in two input profiles");
2744 OS << " Hot-block overlap: "
2745 << format(Fmt: "%.3f%%", Vals: static_cast<double>(HotBlockOverlap.OverlapCount) /
2746 HotBlockOverlap.UnionCount * 100)
2747 << "\n";
2748 OS << " overlap hot blocks: " << HotBlockOverlap.OverlapCount << "\n";
2749 OS << " hot blocks unique in base profile: "
2750 << HotBlockOverlap.BaseCount - HotBlockOverlap.OverlapCount << "\n";
2751 OS << " hot blocks unique in test profile: "
2752 << HotBlockOverlap.TestCount - HotBlockOverlap.OverlapCount << "\n";
2753}
2754
2755std::error_code SampleOverlapAggregator::loadProfiles() {
2756 using namespace sampleprof;
2757
2758 LLVMContext Context;
2759 auto FS = vfs::getRealFileSystem();
2760 auto BaseReaderOrErr = SampleProfileReader::create(Filename: BaseFilename, C&: Context, FS&: *FS,
2761 P: FSDiscriminatorPassOption);
2762 if (std::error_code EC = BaseReaderOrErr.getError())
2763 exitWithErrorCode(EC, Whence: BaseFilename);
2764
2765 auto TestReaderOrErr = SampleProfileReader::create(Filename: TestFilename, C&: Context, FS&: *FS,
2766 P: FSDiscriminatorPassOption);
2767 if (std::error_code EC = TestReaderOrErr.getError())
2768 exitWithErrorCode(EC, Whence: TestFilename);
2769
2770 BaseReader = std::move(BaseReaderOrErr.get());
2771 TestReader = std::move(TestReaderOrErr.get());
2772
2773 if (std::error_code EC = BaseReader->read())
2774 exitWithErrorCode(EC, Whence: BaseFilename);
2775 if (std::error_code EC = TestReader->read())
2776 exitWithErrorCode(EC, Whence: TestFilename);
2777 if (BaseReader->profileIsProbeBased() != TestReader->profileIsProbeBased())
2778 exitWithError(
2779 Message: "cannot compare probe-based profile with non-probe-based profile");
2780 if (BaseReader->profileIsCS() != TestReader->profileIsCS())
2781 exitWithError(Message: "cannot compare CS profile with non-CS profile");
2782
2783 // Load BaseHotThreshold and TestHotThreshold as 99-percentile threshold in
2784 // profile summary.
2785 ProfileSummary &BasePS = BaseReader->getSummary();
2786 ProfileSummary &TestPS = TestReader->getSummary();
2787 BaseHotThreshold =
2788 ProfileSummaryBuilder::getHotCountThreshold(DS: BasePS.getDetailedSummary());
2789 TestHotThreshold =
2790 ProfileSummaryBuilder::getHotCountThreshold(DS: TestPS.getDetailedSummary());
2791
2792 return std::error_code();
2793}
2794
2795void overlapSampleProfile(const std::string &BaseFilename,
2796 const std::string &TestFilename,
2797 const OverlapFuncFilters &FuncFilter,
2798 uint64_t SimilarityCutoff, raw_fd_ostream &OS) {
2799 using namespace sampleprof;
2800
2801 // We use 0.000005 to initialize OverlapAggr.Epsilon because the final metrics
2802 // report 2--3 places after decimal point in percentage numbers.
2803 SampleOverlapAggregator OverlapAggr(
2804 BaseFilename, TestFilename,
2805 static_cast<double>(SimilarityCutoff) / 1000000, 0.000005, FuncFilter);
2806 if (std::error_code EC = OverlapAggr.loadProfiles())
2807 exitWithErrorCode(EC);
2808
2809 OverlapAggr.initializeSampleProfileOverlap();
2810 if (OverlapAggr.detectZeroSampleProfile(OS))
2811 return;
2812
2813 OverlapAggr.computeSampleProfileOverlap(OS);
2814
2815 OverlapAggr.dumpProgramSummary(OS);
2816 OverlapAggr.dumpHotFuncAndBlockOverlap(OS);
2817 OverlapAggr.dumpFuncSimilarity(OS);
2818}
2819
2820static int overlap_main() {
2821 std::error_code EC;
2822 raw_fd_ostream OS(OutputFilename.data(), EC, sys::fs::OF_TextWithCRLF);
2823 if (EC)
2824 exitWithErrorCode(EC, Whence: OutputFilename);
2825
2826 if (ProfileKind == instr)
2827 overlapInstrProfile(BaseFilename, TestFilename,
2828 FuncFilter: OverlapFuncFilters{.ValueCutoff: OverlapValueCutoff, .NameFilter: FuncNameFilter},
2829 OS, IsCS);
2830 else
2831 overlapSampleProfile(BaseFilename, TestFilename,
2832 FuncFilter: OverlapFuncFilters{.ValueCutoff: OverlapValueCutoff, .NameFilter: FuncNameFilter},
2833 SimilarityCutoff, OS);
2834
2835 return 0;
2836}
2837
2838namespace {
2839struct ValueSitesStats {
2840 ValueSitesStats() = default;
2841 uint64_t TotalNumValueSites = 0;
2842 uint64_t TotalNumValueSitesWithValueProfile = 0;
2843 uint64_t TotalNumValues = 0;
2844 std::vector<unsigned> ValueSitesHistogram;
2845};
2846} // namespace
2847
2848static void traverseAllValueSites(const InstrProfRecord &Func, uint32_t VK,
2849 ValueSitesStats &Stats, raw_fd_ostream &OS,
2850 InstrProfSymtab *Symtab) {
2851 uint32_t NS = Func.getNumValueSites(ValueKind: VK);
2852 Stats.TotalNumValueSites += NS;
2853 for (size_t I = 0; I < NS; ++I) {
2854 auto VD = Func.getValueArrayForSite(ValueKind: VK, Site: I);
2855 uint32_t NV = VD.size();
2856 if (NV == 0)
2857 continue;
2858 Stats.TotalNumValues += NV;
2859 Stats.TotalNumValueSitesWithValueProfile++;
2860 if (NV > Stats.ValueSitesHistogram.size())
2861 Stats.ValueSitesHistogram.resize(new_size: NV, x: 0);
2862 Stats.ValueSitesHistogram[NV - 1]++;
2863
2864 uint64_t SiteSum = 0;
2865 for (const auto &V : VD)
2866 SiteSum += V.Count;
2867 if (SiteSum == 0)
2868 SiteSum = 1;
2869
2870 for (const auto &V : VD) {
2871 OS << "\t[ " << format(Fmt: "%2u", Vals: I) << ", ";
2872 if (Symtab == nullptr)
2873 OS << format(Fmt: "%4" PRIu64, Vals: V.Value);
2874 else
2875 OS << Symtab->getFuncOrVarName(MD5Hash: V.Value);
2876 OS << ", " << format(Fmt: "%10" PRId64, Vals: V.Count) << " ] ("
2877 << format(Fmt: "%.2f%%", Vals: (V.Count * 100.0 / SiteSum)) << ")\n";
2878 }
2879 }
2880}
2881
2882static void showValueSitesStats(raw_fd_ostream &OS, uint32_t VK,
2883 ValueSitesStats &Stats) {
2884 OS << " Total number of sites: " << Stats.TotalNumValueSites << "\n";
2885 OS << " Total number of sites with values: "
2886 << Stats.TotalNumValueSitesWithValueProfile << "\n";
2887 OS << " Total number of profiled values: " << Stats.TotalNumValues << "\n";
2888
2889 OS << " Value sites histogram:\n\tNumTargets, SiteCount\n";
2890 for (unsigned I = 0; I < Stats.ValueSitesHistogram.size(); I++) {
2891 if (Stats.ValueSitesHistogram[I] > 0)
2892 OS << "\t" << I + 1 << ", " << Stats.ValueSitesHistogram[I] << "\n";
2893 }
2894}
2895
2896static int showInstrProfile(ShowFormat SFormat, raw_fd_ostream &OS) {
2897 if (SFormat == ShowFormat::Json)
2898 exitWithError(Message: "JSON output is not supported for instr profiles");
2899 if (SFormat == ShowFormat::Yaml)
2900 exitWithError(Message: "YAML output is not supported for instr profiles");
2901 auto FS = vfs::getRealFileSystem();
2902 auto ReaderOrErr = InstrProfReader::create(Path: Filename, FS&: *FS);
2903 std::vector<uint32_t> Cutoffs = std::move(DetailedSummaryCutoffs);
2904 if (Cutoffs.empty() && (ShowDetailedSummary || ShowHotFuncList))
2905 Cutoffs = ProfileSummaryBuilder::DefaultCutoffs;
2906 InstrProfSummaryBuilder Builder(std::move(Cutoffs));
2907 if (Error E = ReaderOrErr.takeError())
2908 exitWithError(E: std::move(E), Whence: Filename);
2909
2910 auto Reader = std::move(ReaderOrErr.get());
2911 bool IsIRInstr = Reader->isIRLevelProfile();
2912 size_t ShownFunctions = 0;
2913 size_t BelowCutoffFunctions = 0;
2914 int NumVPKind = IPVK_Last - IPVK_First + 1;
2915 std::vector<ValueSitesStats> VPStats(NumVPKind);
2916
2917 std::vector<std::pair<StringRef, uint64_t>> NameAndMaxCount;
2918
2919 if (!TextFormat && OnlyListBelow) {
2920 OS << "The list of functions with the maximum counter less than "
2921 << ShowValueCutoff << ":\n";
2922 }
2923
2924 // Add marker so that IR-level instrumentation round-trips properly.
2925 if (TextFormat && IsIRInstr)
2926 OS << ":ir\n";
2927
2928 for (const auto &Func : *Reader) {
2929 if (Reader->isIRLevelProfile()) {
2930 bool FuncIsCS = NamedInstrProfRecord::hasCSFlagInHash(FuncHash: Func.Hash);
2931 if (FuncIsCS != ShowCS)
2932 continue;
2933 }
2934 bool Show = ShowAllFunctions ||
2935 (!FuncNameFilter.empty() && Func.Name.contains(Other: FuncNameFilter));
2936
2937 bool doTextFormatDump = (Show && TextFormat);
2938
2939 if (doTextFormatDump) {
2940 InstrProfSymtab &Symtab = Reader->getSymtab();
2941 InstrProfWriter::writeRecordInText(Name: Func.Name, Hash: Func.Hash, Counters: Func, Symtab,
2942 OS);
2943 continue;
2944 }
2945
2946 assert(Func.Counts.size() > 0 && "function missing entry counter");
2947 Builder.addRecord(Func);
2948
2949 if (ShowCovered) {
2950 if (llvm::any_of(Range: Func.Counts, P: [](uint64_t C) { return C; }))
2951 OS << Func.Name << "\n";
2952 continue;
2953 }
2954
2955 uint64_t FuncMax = 0;
2956 uint64_t FuncSum = 0;
2957
2958 auto PseudoKind = Func.getCountPseudoKind();
2959 if (PseudoKind != InstrProfRecord::NotPseudo) {
2960 if (Show) {
2961 if (!ShownFunctions)
2962 OS << "Counters:\n";
2963 ++ShownFunctions;
2964 OS << " " << Func.Name << ":\n"
2965 << " Hash: " << format(Fmt: "0x%016" PRIx64, Vals: Func.Hash) << "\n"
2966 << " Counters: " << Func.Counts.size();
2967 if (PseudoKind == InstrProfRecord::PseudoHot)
2968 OS << " <PseudoHot>\n";
2969 else if (PseudoKind == InstrProfRecord::PseudoWarm)
2970 OS << " <PseudoWarm>\n";
2971 else
2972 llvm_unreachable("Unknown PseudoKind");
2973 }
2974 continue;
2975 }
2976
2977 for (uint64_t Count : Func.Counts) {
2978 FuncMax = std::max(a: FuncMax, b: Count);
2979 FuncSum += Count;
2980 }
2981
2982 if (FuncMax < ShowValueCutoff) {
2983 ++BelowCutoffFunctions;
2984 if (OnlyListBelow) {
2985 OS << " " << Func.Name << ": (Max = " << FuncMax
2986 << " Sum = " << FuncSum << ")\n";
2987 }
2988 continue;
2989 } else if (OnlyListBelow)
2990 continue;
2991
2992 if (TopNFunctions || ShowHotFuncList)
2993 NameAndMaxCount.emplace_back(args: Func.Name, args&: FuncMax);
2994
2995 if (Show) {
2996 if (!ShownFunctions)
2997 OS << "Counters:\n";
2998
2999 ++ShownFunctions;
3000
3001 OS << " " << Func.Name << ":\n"
3002 << " Hash: " << format(Fmt: "0x%016" PRIx64, Vals: Func.Hash) << "\n"
3003 << " Counters: " << Func.Counts.size() << "\n";
3004 if (!IsIRInstr)
3005 OS << " Function count: " << Func.Counts[0] << "\n";
3006
3007 if (ShowIndirectCallTargets)
3008 OS << " Indirect Call Site Count: "
3009 << Func.getNumValueSites(ValueKind: IPVK_IndirectCallTarget) << "\n";
3010
3011 if (ShowVTables)
3012 OS << " Number of instrumented vtables: "
3013 << Func.getNumValueSites(ValueKind: IPVK_VTableTarget) << "\n";
3014
3015 uint32_t NumMemOPCalls = Func.getNumValueSites(ValueKind: IPVK_MemOPSize);
3016 if (ShowMemOPSizes && NumMemOPCalls > 0)
3017 OS << " Number of Memory Intrinsics Calls: " << NumMemOPCalls
3018 << "\n";
3019
3020 if (ShowCounts) {
3021 OS << " Block counts: [";
3022 size_t Start = (IsIRInstr ? 0 : 1);
3023 for (size_t I = Start, E = Func.Counts.size(); I < E; ++I) {
3024 OS << (I == Start ? "" : ", ") << Func.Counts[I];
3025 }
3026 OS << "]\n";
3027
3028 // Show uniformity bits if present
3029 if (!Func.UniformityBits.empty()) {
3030 OS << " Block uniformity: [";
3031 for (size_t I = Start, E = Func.Counts.size(); I < E; ++I) {
3032 bool IsUniform = Func.isBlockUniform(BlockIdx: I);
3033 OS << (I == Start ? "" : ", ") << (IsUniform ? "U" : "D");
3034 }
3035 OS << "]\n";
3036 }
3037 }
3038
3039 if (ShowIndirectCallTargets) {
3040 OS << " Indirect Target Results:\n";
3041 traverseAllValueSites(Func, VK: IPVK_IndirectCallTarget,
3042 Stats&: VPStats[IPVK_IndirectCallTarget], OS,
3043 Symtab: &(Reader->getSymtab()));
3044 }
3045
3046 if (ShowVTables) {
3047 OS << " VTable Results:\n";
3048 traverseAllValueSites(Func, VK: IPVK_VTableTarget,
3049 Stats&: VPStats[IPVK_VTableTarget], OS,
3050 Symtab: &(Reader->getSymtab()));
3051 }
3052
3053 if (ShowMemOPSizes && NumMemOPCalls > 0) {
3054 OS << " Memory Intrinsic Size Results:\n";
3055 traverseAllValueSites(Func, VK: IPVK_MemOPSize, Stats&: VPStats[IPVK_MemOPSize], OS,
3056 Symtab: nullptr);
3057 }
3058 }
3059 }
3060 if (Reader->hasError())
3061 exitWithError(E: Reader->getError(), Whence: Filename);
3062
3063 if (TextFormat || ShowCovered)
3064 return 0;
3065 std::unique_ptr<ProfileSummary> PS(Builder.getSummary());
3066 bool IsIR = Reader->isIRLevelProfile();
3067 OS << "Instrumentation level: " << (IsIR ? "IR" : "Front-end");
3068 if (IsIR) {
3069 OS << " entry_first = " << Reader->instrEntryBBEnabled();
3070 OS << " instrument_loop_entries = " << Reader->instrLoopEntriesEnabled();
3071 }
3072 OS << "\n";
3073 if (ShowAllFunctions || !FuncNameFilter.empty())
3074 OS << "Functions shown: " << ShownFunctions << "\n";
3075 PS->printSummary(OS);
3076 if (ShowValueCutoff > 0) {
3077 OS << "Number of functions with maximum count (< " << ShowValueCutoff
3078 << "): " << BelowCutoffFunctions << "\n";
3079 OS << "Number of functions with maximum count (>= " << ShowValueCutoff
3080 << "): " << PS->getNumFunctions() - BelowCutoffFunctions << "\n";
3081 }
3082
3083 // Sort by MaxCount in decreasing order
3084 llvm::stable_sort(Range&: NameAndMaxCount, C: [](const auto &L, const auto &R) {
3085 return L.second > R.second;
3086 });
3087 if (TopNFunctions) {
3088 OS << "Top " << TopNFunctions
3089 << " functions with the largest internal block counts: \n";
3090 auto TopFuncs = ArrayRef(NameAndMaxCount).take_front(N: TopNFunctions);
3091 for (auto [Name, MaxCount] : TopFuncs)
3092 OS << " " << Name << ", max count = " << MaxCount << "\n";
3093 }
3094
3095 if (ShowHotFuncList) {
3096 auto HotCountThreshold =
3097 ProfileSummaryBuilder::getHotCountThreshold(DS: PS->getDetailedSummary());
3098 OS << "# Hot count threshold: " << HotCountThreshold << "\n";
3099 for (auto [Name, MaxCount] : NameAndMaxCount) {
3100 if (MaxCount < HotCountThreshold)
3101 break;
3102 OS << Name << "\n";
3103 }
3104 }
3105
3106 if (ShownFunctions && ShowIndirectCallTargets) {
3107 OS << "Statistics for indirect call sites profile:\n";
3108 showValueSitesStats(OS, VK: IPVK_IndirectCallTarget,
3109 Stats&: VPStats[IPVK_IndirectCallTarget]);
3110 }
3111
3112 if (ShownFunctions && ShowVTables) {
3113 OS << "Statistics for vtable profile:\n";
3114 showValueSitesStats(OS, VK: IPVK_VTableTarget, Stats&: VPStats[IPVK_VTableTarget]);
3115 }
3116
3117 if (ShownFunctions && ShowMemOPSizes) {
3118 OS << "Statistics for memory intrinsic calls sizes profile:\n";
3119 showValueSitesStats(OS, VK: IPVK_MemOPSize, Stats&: VPStats[IPVK_MemOPSize]);
3120 }
3121
3122 if (ShowDetailedSummary)
3123 PS->printDetailedSummary(OS);
3124
3125 if (ShowBinaryIds)
3126 if (Error E = Reader->printBinaryIds(OS))
3127 exitWithError(E: std::move(E), Whence: Filename);
3128
3129 if (ShowProfileVersion)
3130 OS << "Profile version: " << Reader->getVersion() << "\n";
3131
3132 if (ShowTemporalProfTraces) {
3133 auto &Traces = Reader->getTemporalProfTraces();
3134 OS << "Temporal Profile Traces (samples=" << Traces.size()
3135 << " seen=" << Reader->getTemporalProfTraceStreamSize() << "):\n";
3136 for (unsigned i = 0; i < Traces.size(); i++) {
3137 OS << " Temporal Profile Trace " << i << " (weight=" << Traces[i].Weight
3138 << " count=" << Traces[i].FunctionNameRefs.size() << "):\n";
3139 for (auto &NameRef : Traces[i].FunctionNameRefs)
3140 OS << " " << Reader->getSymtab().getFuncOrVarName(MD5Hash: NameRef) << "\n";
3141 }
3142 }
3143
3144 return 0;
3145}
3146
3147static void showSectionInfo(sampleprof::SampleProfileReader *Reader,
3148 raw_fd_ostream &OS) {
3149 if (!Reader->dumpSectionInfo(OS)) {
3150 WithColor::warning() << "-show-sec-info-only is only supported for "
3151 << "sample profile in extbinary format and is "
3152 << "ignored for other formats.\n";
3153 return;
3154 }
3155}
3156
3157namespace {
3158struct HotFuncInfo {
3159 std::string FuncName;
3160 uint64_t TotalCount = 0;
3161 double TotalCountPercent = 0.0f;
3162 uint64_t MaxCount = 0;
3163 uint64_t EntryCount = 0;
3164
3165 HotFuncInfo() = default;
3166
3167 HotFuncInfo(StringRef FN, uint64_t TS, double TSP, uint64_t MS, uint64_t ES)
3168 : FuncName(FN.begin(), FN.end()), TotalCount(TS), TotalCountPercent(TSP),
3169 MaxCount(MS), EntryCount(ES) {}
3170};
3171} // namespace
3172
3173// Print out detailed information about hot functions in PrintValues vector.
3174// Users specify titles and offset of every columns through ColumnTitle and
3175// ColumnOffset. The size of ColumnTitle and ColumnOffset need to be the same
3176// and at least 4. Besides, users can optionally give a HotFuncMetric string to
3177// print out or let it be an empty string.
3178static void dumpHotFunctionList(const std::vector<std::string> &ColumnTitle,
3179 const std::vector<int> &ColumnOffset,
3180 const std::vector<HotFuncInfo> &PrintValues,
3181 uint64_t HotFuncCount, uint64_t TotalFuncCount,
3182 uint64_t HotProfCount, uint64_t TotalProfCount,
3183 const std::string &HotFuncMetric,
3184 uint32_t TopNFunctions, raw_fd_ostream &OS) {
3185 assert(ColumnOffset.size() == ColumnTitle.size() &&
3186 "ColumnOffset and ColumnTitle should have the same size");
3187 assert(ColumnTitle.size() >= 4 &&
3188 "ColumnTitle should have at least 4 elements");
3189 assert(TotalFuncCount > 0 &&
3190 "There should be at least one function in the profile");
3191 double TotalProfPercent = 0;
3192 if (TotalProfCount > 0)
3193 TotalProfPercent = static_cast<double>(HotProfCount) / TotalProfCount * 100;
3194
3195 formatted_raw_ostream FOS(OS);
3196 FOS << HotFuncCount << " out of " << TotalFuncCount
3197 << " functions with profile ("
3198 << format(Fmt: "%.2f%%",
3199 Vals: (static_cast<double>(HotFuncCount) / TotalFuncCount * 100))
3200 << ") are considered hot functions";
3201 if (!HotFuncMetric.empty())
3202 FOS << " (" << HotFuncMetric << ")";
3203 FOS << ".\n";
3204 FOS << HotProfCount << " out of " << TotalProfCount << " profile counts ("
3205 << format(Fmt: "%.2f%%", Vals: TotalProfPercent) << ") are from hot functions.\n";
3206
3207 for (size_t I = 0; I < ColumnTitle.size(); ++I) {
3208 FOS.PadToColumn(NewCol: ColumnOffset[I]);
3209 FOS << ColumnTitle[I];
3210 }
3211 FOS << "\n";
3212
3213 uint32_t Count = 0;
3214 for (const auto &R : PrintValues) {
3215 if (TopNFunctions && (Count++ == TopNFunctions))
3216 break;
3217 FOS.PadToColumn(NewCol: ColumnOffset[0]);
3218 FOS << R.TotalCount << " (" << format(Fmt: "%.2f%%", Vals: R.TotalCountPercent) << ")";
3219 FOS.PadToColumn(NewCol: ColumnOffset[1]);
3220 FOS << R.MaxCount;
3221 FOS.PadToColumn(NewCol: ColumnOffset[2]);
3222 FOS << R.EntryCount;
3223 FOS.PadToColumn(NewCol: ColumnOffset[3]);
3224 FOS << R.FuncName << "\n";
3225 }
3226}
3227
3228static int showHotFunctionList(const sampleprof::SampleProfileMap &Profiles,
3229 ProfileSummary &PS, uint32_t TopN,
3230 raw_fd_ostream &OS) {
3231 using namespace sampleprof;
3232
3233 const uint32_t HotFuncCutoff = 990000;
3234 auto &SummaryVector = PS.getDetailedSummary();
3235 uint64_t MinCountThreshold = 0;
3236 for (const ProfileSummaryEntry &SummaryEntry : SummaryVector) {
3237 if (SummaryEntry.Cutoff == HotFuncCutoff) {
3238 MinCountThreshold = SummaryEntry.MinCount;
3239 break;
3240 }
3241 }
3242
3243 // Traverse all functions in the profile and keep only hot functions.
3244 // The following loop also calculates the sum of total samples of all
3245 // functions.
3246 std::multimap<uint64_t, std::pair<const FunctionSamples *, const uint64_t>,
3247 std::greater<uint64_t>>
3248 HotFunc;
3249 uint64_t ProfileTotalSample = 0;
3250 uint64_t HotFuncSample = 0;
3251 uint64_t HotFuncCount = 0;
3252
3253 for (const auto &I : Profiles) {
3254 FuncSampleStats FuncStats;
3255 const FunctionSamples &FuncProf = I.second;
3256 ProfileTotalSample += FuncProf.getTotalSamples();
3257 getFuncSampleStats(Func: FuncProf, FuncStats, HotThreshold: MinCountThreshold);
3258
3259 if (isFunctionHot(FuncStats, HotThreshold: MinCountThreshold)) {
3260 HotFunc.emplace(args: FuncProf.getTotalSamples(),
3261 args: std::make_pair(x: &(I.second), y&: FuncStats.MaxSample));
3262 HotFuncSample += FuncProf.getTotalSamples();
3263 ++HotFuncCount;
3264 }
3265 }
3266
3267 std::vector<std::string> ColumnTitle{"Total sample (%)", "Max sample",
3268 "Entry sample", "Function name"};
3269 std::vector<int> ColumnOffset{0, 24, 42, 58};
3270 std::string Metric =
3271 std::string("max sample >= ") + std::to_string(val: MinCountThreshold);
3272 std::vector<HotFuncInfo> PrintValues;
3273 for (const auto &FuncPair : HotFunc) {
3274 const FunctionSamples &Func = *FuncPair.second.first;
3275 double TotalSamplePercent =
3276 (ProfileTotalSample > 0)
3277 ? (Func.getTotalSamples() * 100.0) / ProfileTotalSample
3278 : 0;
3279 PrintValues.emplace_back(
3280 args: HotFuncInfo(Func.getContext().toString(), Func.getTotalSamples(),
3281 TotalSamplePercent, FuncPair.second.second,
3282 Func.getHeadSamplesEstimate()));
3283 }
3284 dumpHotFunctionList(ColumnTitle, ColumnOffset, PrintValues, HotFuncCount,
3285 TotalFuncCount: Profiles.size(), HotProfCount: HotFuncSample, TotalProfCount: ProfileTotalSample,
3286 HotFuncMetric: Metric, TopNFunctions: TopN, OS);
3287
3288 return 0;
3289}
3290
3291static int showSampleProfile(ShowFormat SFormat, raw_fd_ostream &OS) {
3292 if (SFormat == ShowFormat::Yaml)
3293 exitWithError(Message: "YAML output is not supported for sample profiles");
3294 if (ShowSectionInfoOnly && ShowCompositeInfoOnly)
3295 exitWithError(Message: "-show-sec-info-only and "
3296 "-show-composite-info-only cannot be used together");
3297
3298 using namespace sampleprof;
3299 LLVMContext Context;
3300 auto FS = vfs::getRealFileSystem();
3301 auto ReaderOrErr = SampleProfileReader::create(Filename, C&: Context, FS&: *FS,
3302 P: FSDiscriminatorPassOption);
3303 if (std::error_code EC = ReaderOrErr.getError())
3304 exitWithErrorCode(EC, Whence: Filename);
3305
3306 auto Reader = std::move(ReaderOrErr.get());
3307 if (ShowSectionInfoOnly) {
3308 showSectionInfo(Reader: Reader.get(), OS);
3309 return 0;
3310 }
3311
3312 if (ShowCompositeInfoOnly) {
3313 if (!Reader->hasCompositeProfileSection()) {
3314 WithColor::warning() << "no composite profile section; nothing to show\n";
3315 return 0;
3316 }
3317 if (std::error_code EC = Reader->dumpProfileTypeInfo(OS)) {
3318 OS.flush();
3319 exitWithErrorCode(EC, Whence: Filename);
3320 }
3321 return 0;
3322 }
3323
3324 if (std::error_code EC = Reader->read())
3325 exitWithErrorCode(EC, Whence: Filename);
3326
3327 if (ShowAllFunctions || FuncNameFilter.empty()) {
3328 if (SFormat == ShowFormat::Json)
3329 Reader->dumpJson(OS);
3330 else
3331 Reader->dump(OS);
3332 } else {
3333 if (SFormat == ShowFormat::Json)
3334 exitWithError(
3335 Message: "the JSON format is supported only when all functions are to "
3336 "be printed");
3337
3338 // TODO: parse context string to support filtering by contexts.
3339 FunctionSamples *FS = Reader->getSamplesFor(Fname: StringRef(FuncNameFilter));
3340 Reader->dumpFunctionProfile(FS: FS ? *FS : FunctionSamples(), OS);
3341 }
3342
3343 if (ShowProfileSymbolList) {
3344 std::unique_ptr<sampleprof::ProfileSymbolList> ReaderList =
3345 Reader->getProfileSymbolList();
3346 ReaderList->dump(OS);
3347 }
3348
3349 if (ShowDetailedSummary) {
3350 auto &PS = Reader->getSummary();
3351 PS.printSummary(OS);
3352 PS.printDetailedSummary(OS);
3353 }
3354
3355 if (ShowHotFuncList || TopNFunctions)
3356 showHotFunctionList(Profiles: Reader->getProfiles(), PS&: Reader->getSummary(),
3357 TopN: TopNFunctions, OS);
3358
3359 return 0;
3360}
3361
3362static int showMemProfProfile(ShowFormat SFormat, raw_fd_ostream &OS) {
3363 if (SFormat == ShowFormat::Json)
3364 exitWithError(Message: "JSON output is not supported for MemProf");
3365
3366 // Show the raw profile in YAML.
3367 if (memprof::RawMemProfReader::hasFormat(Path: Filename)) {
3368 auto ReaderOr = llvm::memprof::RawMemProfReader::create(
3369 Path: Filename, ProfiledBinary, /*KeepNames=*/KeepName: true);
3370 if (Error E = ReaderOr.takeError()) {
3371 // Since the error can be related to the profile or the binary we do not
3372 // pass whence. Instead additional context is provided where necessary in
3373 // the error message.
3374 exitWithError(E: std::move(E), /*Whence*/ "");
3375 }
3376
3377 std::unique_ptr<llvm::memprof::RawMemProfReader> Reader(
3378 ReaderOr.get().release());
3379
3380 Reader->printYAML(OS);
3381 return 0;
3382 }
3383
3384 // Show the indexed MemProf profile in YAML.
3385 auto FS = vfs::getRealFileSystem();
3386 auto ReaderOrErr = IndexedInstrProfReader::create(Path: Filename, FS&: *FS);
3387 if (Error E = ReaderOrErr.takeError())
3388 exitWithError(E: std::move(E), Whence: Filename);
3389
3390 auto Reader = std::move(ReaderOrErr.get());
3391 memprof::AllMemProfData Data = Reader->getAllMemProfData();
3392
3393 // For v4 and above the summary is serialized in the indexed profile, and can
3394 // be accessed from the reader. Earlier versions build the summary below.
3395 // The summary is emitted as YAML comments at the start of the output.
3396 if (auto *MemProfSum = Reader->getMemProfSummary()) {
3397 MemProfSum->printSummaryYaml(OS);
3398 } else {
3399 memprof::MemProfSummaryBuilder MemProfSumBuilder;
3400 for (auto &Pair : Data.HeapProfileRecords)
3401 MemProfSumBuilder.addRecord(Pair.Record);
3402 MemProfSumBuilder.getSummary()->printSummaryYaml(OS);
3403 }
3404 // Construct yaml::Output with the maximum column width of 80 so that each
3405 // Frame fits in one line.
3406 yaml::Output Yout(OS, nullptr, 80);
3407 Yout << Data;
3408
3409 return 0;
3410}
3411
3412static int showDebugInfoCorrelation(const std::string &Filename,
3413 ShowFormat SFormat, raw_fd_ostream &OS) {
3414 if (SFormat == ShowFormat::Json)
3415 exitWithError(Message: "JSON output is not supported for debug info correlation");
3416 std::unique_ptr<InstrProfCorrelator> Correlator;
3417 if (auto Err =
3418 InstrProfCorrelator::get(Filename, FileKind: InstrProfCorrelator::DEBUG_INFO)
3419 .moveInto(Value&: Correlator))
3420 exitWithError(E: std::move(Err), Whence: Filename);
3421 if (SFormat == ShowFormat::Yaml) {
3422 if (auto Err = Correlator->dumpYaml(MaxWarnings: MaxDbgCorrelationWarnings, OS))
3423 exitWithError(E: std::move(Err), Whence: Filename);
3424 return 0;
3425 }
3426
3427 if (auto Err = Correlator->correlateProfileData(MaxWarnings: MaxDbgCorrelationWarnings))
3428 exitWithError(E: std::move(Err), Whence: Filename);
3429
3430 InstrProfSymtab Symtab;
3431 if (auto Err = Symtab.create(
3432 NameStrings: StringRef(Correlator->getNamesPointer(), Correlator->getNamesSize())))
3433 exitWithError(E: std::move(Err), Whence: Filename);
3434
3435 if (ShowProfileSymbolList)
3436 Symtab.dumpNames(OS);
3437 // TODO: Read "Profile Data Type" from debug info to compute and show how many
3438 // counters the section holds.
3439 if (ShowDetailedSummary)
3440 OS << "Counters section size: 0x"
3441 << Twine::utohexstr(Val: Correlator->getCountersSectionSize()) << " bytes\n";
3442 OS << "Found " << Correlator->getDataSize() << " functions\n";
3443
3444 return 0;
3445}
3446
3447static int show_main(StringRef ProgName) {
3448 if (Filename.empty() && DebugInfoFilename.empty())
3449 exitWithError(
3450 Message: "the positional argument '<profdata-file>' is required unless '--" +
3451 DebugInfoFilename.ArgStr + "' is provided");
3452
3453 if (Filename == OutputFilename) {
3454 errs() << ProgName
3455 << " show: Input file name cannot be the same as the output file "
3456 "name!\n";
3457 return 1;
3458 }
3459 if (JsonFormat)
3460 SFormat = ShowFormat::Json;
3461
3462 std::error_code EC;
3463 raw_fd_ostream OS(OutputFilename.data(), EC, sys::fs::OF_TextWithCRLF);
3464 if (EC)
3465 exitWithErrorCode(EC, Whence: OutputFilename);
3466
3467 if (ShowAllFunctions && !FuncNameFilter.empty())
3468 WithColor::warning() << "-function argument ignored: showing all functions\n";
3469
3470 if (!DebugInfoFilename.empty())
3471 return showDebugInfoCorrelation(Filename: DebugInfoFilename, SFormat, OS);
3472
3473 if (ShowProfileKind == instr)
3474 return showInstrProfile(SFormat, OS);
3475 if (ShowProfileKind == sample)
3476 return showSampleProfile(SFormat, OS);
3477 return showMemProfProfile(SFormat, OS);
3478}
3479
3480static int order_main() {
3481 std::error_code EC;
3482 raw_fd_ostream OS(OutputFilename.data(), EC, sys::fs::OF_TextWithCRLF);
3483 if (EC)
3484 exitWithErrorCode(EC, Whence: OutputFilename);
3485 auto FS = vfs::getRealFileSystem();
3486 auto ReaderOrErr = InstrProfReader::create(Path: Filename, FS&: *FS);
3487 if (Error E = ReaderOrErr.takeError())
3488 exitWithError(E: std::move(E), Whence: Filename);
3489
3490 auto Reader = std::move(ReaderOrErr.get());
3491 for (auto &I : *Reader) {
3492 // Read all entries
3493 (void)I;
3494 }
3495 ArrayRef Traces = Reader->getTemporalProfTraces();
3496 if (NumTestTraces && NumTestTraces >= Traces.size())
3497 exitWithError(
3498 Message: "--" + NumTestTraces.ArgStr +
3499 " must be smaller than the total number of traces: expected: < " +
3500 Twine(Traces.size()) + ", actual: " + Twine(NumTestTraces));
3501 ArrayRef TestTraces = Traces.take_back(N: NumTestTraces);
3502 Traces = Traces.drop_back(N: NumTestTraces);
3503
3504 std::vector<BPFunctionNode> Nodes;
3505 TemporalProfTraceTy::createBPFunctionNodes(Traces, Nodes);
3506 BalancedPartitioningConfig Config;
3507 BalancedPartitioning BP(Config);
3508 BP.run(Nodes);
3509
3510 OS << "# Ordered " << Nodes.size() << " functions\n";
3511 if (!TestTraces.empty()) {
3512 // Since we don't know the symbol sizes, we assume 32 functions per page.
3513 DenseMap<BPFunctionNode::IDT, unsigned> IdToPageNumber;
3514 for (auto &Node : Nodes)
3515 IdToPageNumber[Node.Id] = IdToPageNumber.size() / 32;
3516
3517 SmallSet<unsigned, 0> TouchedPages;
3518 unsigned Area = 0;
3519 for (auto &Trace : TestTraces) {
3520 for (auto Id : Trace.FunctionNameRefs) {
3521 auto It = IdToPageNumber.find(Val: Id);
3522 if (It == IdToPageNumber.end())
3523 continue;
3524 TouchedPages.insert(V: It->getSecond());
3525 Area += TouchedPages.size();
3526 }
3527 TouchedPages.clear();
3528 }
3529 OS << "# Total area under the page fault curve: " << (float)Area << "\n";
3530 }
3531 OS << "# Warning: Mach-O may prefix symbols with \"_\" depending on the "
3532 "linkage and this output does not take that into account. Some "
3533 "post-processing may be required before passing to the linker via "
3534 "-order_file.\n";
3535 for (auto &N : Nodes) {
3536 auto [Filename, ParsedFuncName] =
3537 getParsedIRPGOName(IRPGOName: Reader->getSymtab().getFuncOrVarName(MD5Hash: N.Id));
3538 if (!Filename.empty())
3539 OS << "# " << Filename << "\n";
3540 OS << ParsedFuncName << "\n";
3541 }
3542 return 0;
3543}
3544
3545int main(int argc, const char *argv[]) {
3546 InitLLVM X(argc, argv);
3547 StringRef ProgName(sys::path::filename(path: argv[0]));
3548
3549 if (argc < 2) {
3550 errs()
3551 << ProgName
3552 << ": No subcommand specified! Run llvm-profdata --help for usage.\n";
3553 return 1;
3554 }
3555
3556 cl::ParseCommandLineOptions(argc, argv, Overview: "LLVM profile data\n");
3557
3558 if (ShowSubcommand)
3559 return show_main(ProgName);
3560
3561 if (OrderSubcommand)
3562 return order_main();
3563
3564 if (OverlapSubcommand)
3565 return overlap_main();
3566
3567 if (MergeSubcommand)
3568 return merge_main(ProgName);
3569
3570 errs() << ProgName
3571 << ": Unknown command. Run llvm-profdata --help for usage.\n";
3572 return 1;
3573}
3574