1//===- SampleProfReader.cpp - Read LLVM sample profile data ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the class that reads LLVM sample profiles. It
10// supports three file formats: text, binary and gcov.
11//
12// The textual representation is useful for debugging and testing purposes. The
13// binary representation is more compact, resulting in smaller file sizes.
14//
15// The gcov encoding is the one generated by GCC's AutoFDO profile creation
16// tool (https://github.com/google/autofdo)
17//
18// All three encodings can be used interchangeably as an input sample profile.
19//
20//===----------------------------------------------------------------------===//
21
22#include "llvm/ProfileData/SampleProfReader.h"
23#include "llvm/ADT/DenseMap.h"
24#include "llvm/ADT/STLExtras.h"
25#include "llvm/ADT/StringRef.h"
26#include "llvm/IR/Module.h"
27#include "llvm/IR/ProfileSummary.h"
28#include "llvm/ProfileData/ProfileCommon.h"
29#include "llvm/ProfileData/SampleProf.h"
30#include "llvm/Support/CommandLine.h"
31#include "llvm/Support/Compression.h"
32#include "llvm/Support/ErrorOr.h"
33#include "llvm/Support/JSON.h"
34#include "llvm/Support/LEB128.h"
35#include "llvm/Support/LineIterator.h"
36#include "llvm/Support/MD5.h"
37#include "llvm/Support/MemoryBuffer.h"
38#include "llvm/Support/VirtualFileSystem.h"
39#include "llvm/Support/raw_ostream.h"
40#include <algorithm>
41#include <cstddef>
42#include <cstdint>
43#include <limits>
44#include <memory>
45#include <system_error>
46#include <vector>
47
48using namespace llvm;
49using namespace sampleprof;
50
51#define DEBUG_TYPE "samplepgo-reader"
52
53// This internal option specifies if the profile uses FS discriminators.
54// It only applies to text, and binary format profiles.
55// For ext-binary format profiles, the flag is set in the summary.
56static cl::opt<bool> ProfileIsFSDisciminator(
57 "profile-isfs", cl::Hidden, cl::init(Val: false),
58 cl::desc("Profile uses flow sensitive discriminators"));
59
60/// Dump the function profile for \p FName.
61///
62/// \param FContext Name + context of the function to print.
63/// \param OS Stream to emit the output to.
64void SampleProfileReader::dumpFunctionProfile(const FunctionSamples &FS,
65 raw_ostream &OS) {
66 OS << "Function: " << FS.getContext().toString() << ": " << FS;
67}
68
69/// Dump all the function profiles found on stream \p OS.
70void SampleProfileReader::dump(raw_ostream &OS) {
71 std::vector<NameFunctionSamples> V;
72 sortFuncProfiles(ProfileMap: Profiles, SortedProfiles&: V);
73 for (const auto &I : V)
74 dumpFunctionProfile(FS: *I.second, OS);
75}
76
77static void dumpFunctionProfileJson(const FunctionSamples &S,
78 json::OStream &JOS, bool TopLevel = false) {
79 auto DumpBody = [&](const BodySampleMap &BodySamples) {
80 for (const auto &I : BodySamples) {
81 const LineLocation &Loc = I.first;
82 const SampleRecord &Sample = I.second;
83 JOS.object(Contents: [&] {
84 JOS.attribute(Key: "line", Contents: Loc.LineOffset);
85 if (Loc.Discriminator)
86 JOS.attribute(Key: "discriminator", Contents: Loc.Discriminator);
87 JOS.attribute(Key: "samples", Contents: Sample.getSamples());
88
89 auto CallTargets = Sample.getSortedCallTargets();
90 if (!CallTargets.empty()) {
91 JOS.attributeArray(Key: "calls", Contents: [&] {
92 for (const auto &J : CallTargets) {
93 JOS.object(Contents: [&] {
94 JOS.attribute(Key: "function", Contents: J.first.str());
95 JOS.attribute(Key: "samples", Contents: J.second);
96 });
97 }
98 });
99 }
100 });
101 }
102 };
103
104 auto DumpCallsiteSamples = [&](const CallsiteSampleMap &CallsiteSamples) {
105 for (const auto &I : CallsiteSamples)
106 for (const auto &FS : I.second) {
107 const LineLocation &Loc = I.first;
108 const FunctionSamples &CalleeSamples = FS.second;
109 JOS.object(Contents: [&] {
110 JOS.attribute(Key: "line", Contents: Loc.LineOffset);
111 if (Loc.Discriminator)
112 JOS.attribute(Key: "discriminator", Contents: Loc.Discriminator);
113 JOS.attributeArray(
114 Key: "samples", Contents: [&] { dumpFunctionProfileJson(S: CalleeSamples, JOS); });
115 });
116 }
117 };
118
119 JOS.object(Contents: [&] {
120 JOS.attribute(Key: "name", Contents: S.getFunction().str());
121 JOS.attribute(Key: "total", Contents: S.getTotalSamples());
122 if (TopLevel)
123 JOS.attribute(Key: "head", Contents: S.getHeadSamples());
124
125 const auto &BodySamples = S.getBodySamples();
126 if (!BodySamples.empty())
127 JOS.attributeArray(Key: "body", Contents: [&] { DumpBody(BodySamples); });
128
129 const auto &CallsiteSamples = S.getCallsiteSamples();
130 if (!CallsiteSamples.empty())
131 JOS.attributeArray(Key: "callsites",
132 Contents: [&] { DumpCallsiteSamples(CallsiteSamples); });
133 });
134}
135
136/// Dump all the function profiles found on stream \p OS in the JSON format.
137void SampleProfileReader::dumpJson(raw_ostream &OS) {
138 std::vector<NameFunctionSamples> V;
139 sortFuncProfiles(ProfileMap: Profiles, SortedProfiles&: V);
140 json::OStream JOS(OS, 2);
141 JOS.arrayBegin();
142 for (const auto &F : V)
143 dumpFunctionProfileJson(S: *F.second, JOS, TopLevel: true);
144 JOS.arrayEnd();
145
146 // Emit a newline character at the end as json::OStream doesn't emit one.
147 OS << "\n";
148}
149
150/// Parse \p Input as function head.
151///
152/// Parse one line of \p Input, and update function name in \p FName,
153/// function's total sample count in \p NumSamples, function's entry
154/// count in \p NumHeadSamples.
155///
156/// \returns true if parsing is successful.
157static bool ParseHead(const StringRef &Input, StringRef &FName,
158 uint64_t &NumSamples, uint64_t &NumHeadSamples) {
159 if (Input[0] == ' ')
160 return false;
161 size_t n2 = Input.rfind(C: ':');
162 size_t n1 = Input.rfind(C: ':', From: n2 - 1);
163 FName = Input.substr(Start: 0, N: n1);
164 if (Input.substr(Start: n1 + 1, N: n2 - n1 - 1).getAsInteger(Radix: 10, Result&: NumSamples))
165 return false;
166 if (Input.substr(Start: n2 + 1).getAsInteger(Radix: 10, Result&: NumHeadSamples))
167 return false;
168 return true;
169}
170
171/// Returns true if line offset \p L is legal (only has 16 bits).
172static bool isOffsetLegal(unsigned L) { return (L & 0xffff) == L; }
173
174/// Parse \p Input that contains metadata.
175/// Possible metadata:
176/// - CFG Checksum information:
177/// !CFGChecksum: 12345
178/// - CFG Checksum information:
179/// !Attributes: 1
180/// Stores the FunctionHash (a.k.a. CFG Checksum) into \p FunctionHash.
181static bool parseMetadata(const StringRef &Input, uint64_t &FunctionHash,
182 uint32_t &Attributes) {
183 if (Input.starts_with(Prefix: "!CFGChecksum:")) {
184 StringRef CFGInfo = Input.substr(Start: strlen(s: "!CFGChecksum:")).trim();
185 return !CFGInfo.getAsInteger(Radix: 10, Result&: FunctionHash);
186 }
187
188 if (Input.starts_with(Prefix: "!Attributes:")) {
189 StringRef Attrib = Input.substr(Start: strlen(s: "!Attributes:")).trim();
190 return !Attrib.getAsInteger(Radix: 10, Result&: Attributes);
191 }
192
193 return false;
194}
195
196enum class LineType {
197 CallSiteProfile,
198 BodyProfile,
199 Metadata,
200};
201
202/// Parse \p Input as line sample.
203///
204/// \param Input input line.
205/// \param LineTy Type of this line.
206/// \param Depth the depth of the inline stack.
207/// \param NumSamples total samples of the line/inlined callsite.
208/// \param LineOffset line offset to the start of the function.
209/// \param Discriminator discriminator of the line.
210/// \param TargetCountMap map from indirect call target to count.
211/// \param FunctionHash the function's CFG hash, used by pseudo probe.
212///
213/// returns true if parsing is successful.
214static bool ParseLine(const StringRef &Input, LineType &LineTy, uint32_t &Depth,
215 uint64_t &NumSamples, uint32_t &LineOffset,
216 uint32_t &Discriminator, StringRef &CalleeName,
217 DenseMap<StringRef, uint64_t> &TargetCountMap,
218 uint64_t &FunctionHash, uint32_t &Attributes) {
219 for (Depth = 0; Input[Depth] == ' '; Depth++)
220 ;
221 if (Depth == 0)
222 return false;
223
224 if (Input[Depth] == '!') {
225 LineTy = LineType::Metadata;
226 return parseMetadata(Input: Input.substr(Start: Depth), FunctionHash, Attributes);
227 }
228
229 size_t n1 = Input.find(C: ':');
230 StringRef Loc = Input.substr(Start: Depth, N: n1 - Depth);
231 size_t n2 = Loc.find(C: '.');
232 if (n2 == StringRef::npos) {
233 if (Loc.getAsInteger(Radix: 10, Result&: LineOffset) || !isOffsetLegal(L: LineOffset))
234 return false;
235 Discriminator = 0;
236 } else {
237 if (Loc.substr(Start: 0, N: n2).getAsInteger(Radix: 10, Result&: LineOffset))
238 return false;
239 if (Loc.substr(Start: n2 + 1).getAsInteger(Radix: 10, Result&: Discriminator))
240 return false;
241 }
242
243 StringRef Rest = Input.substr(Start: n1 + 2);
244 if (isDigit(C: Rest[0])) {
245 LineTy = LineType::BodyProfile;
246 size_t n3 = Rest.find(C: ' ');
247 if (n3 == StringRef::npos) {
248 if (Rest.getAsInteger(Radix: 10, Result&: NumSamples))
249 return false;
250 } else {
251 if (Rest.substr(Start: 0, N: n3).getAsInteger(Radix: 10, Result&: NumSamples))
252 return false;
253 }
254 // Find call targets and their sample counts.
255 // Note: In some cases, there are symbols in the profile which are not
256 // mangled. To accommodate such cases, use colon + integer pairs as the
257 // anchor points.
258 // An example:
259 // _M_construct<char *>:1000 string_view<std::allocator<char> >:437
260 // ":1000" and ":437" are used as anchor points so the string above will
261 // be interpreted as
262 // target: _M_construct<char *>
263 // count: 1000
264 // target: string_view<std::allocator<char> >
265 // count: 437
266 while (n3 != StringRef::npos) {
267 n3 += Rest.substr(Start: n3).find_first_not_of(C: ' ');
268 Rest = Rest.substr(Start: n3);
269 n3 = Rest.find_first_of(C: ':');
270 if (n3 == StringRef::npos || n3 == 0)
271 return false;
272
273 StringRef Target;
274 uint64_t count, n4;
275 while (true) {
276 // Get the segment after the current colon.
277 StringRef AfterColon = Rest.substr(Start: n3 + 1);
278 // Get the target symbol before the current colon.
279 Target = Rest.substr(Start: 0, N: n3);
280 // Check if the word after the current colon is an integer.
281 n4 = AfterColon.find_first_of(C: ' ');
282 n4 = (n4 != StringRef::npos) ? n3 + n4 + 1 : Rest.size();
283 StringRef WordAfterColon = Rest.substr(Start: n3 + 1, N: n4 - n3 - 1);
284 if (!WordAfterColon.getAsInteger(Radix: 10, Result&: count))
285 break;
286
287 // Try to find the next colon.
288 uint64_t n5 = AfterColon.find_first_of(C: ':');
289 if (n5 == StringRef::npos)
290 return false;
291 n3 += n5 + 1;
292 }
293
294 // An anchor point is found. Save the {target, count} pair
295 TargetCountMap[Target] = count;
296 if (n4 == Rest.size())
297 break;
298 // Change n3 to the next blank space after colon + integer pair.
299 n3 = n4;
300 }
301 } else {
302 LineTy = LineType::CallSiteProfile;
303 size_t n3 = Rest.find_last_of(C: ':');
304 CalleeName = Rest.substr(Start: 0, N: n3);
305 if (Rest.substr(Start: n3 + 1).getAsInteger(Radix: 10, Result&: NumSamples))
306 return false;
307 }
308 return true;
309}
310
311/// Load samples from a text file.
312///
313/// See the documentation at the top of the file for an explanation of
314/// the expected format.
315///
316/// \returns true if the file was loaded successfully, false otherwise.
317std::error_code SampleProfileReaderText::readImpl() {
318 line_iterator LineIt(*Buffer, /*SkipBlanks=*/true, '#');
319 sampleprof_error Result = sampleprof_error::success;
320
321 InlineCallStack InlineStack;
322 uint32_t TopLevelProbeProfileCount = 0;
323
324 // DepthMetadata tracks whether we have processed metadata for the current
325 // top-level or nested function profile.
326 uint32_t DepthMetadata = 0;
327
328 ProfileIsFS = ProfileIsFSDisciminator;
329 FunctionSamples::ProfileIsFS = ProfileIsFS;
330 for (; !LineIt.is_at_eof(); ++LineIt) {
331 size_t pos = LineIt->find_first_not_of(C: ' ');
332 if (pos == LineIt->npos || (*LineIt)[pos] == '#')
333 continue;
334 // Read the header of each function.
335 //
336 // Note that for function identifiers we are actually expecting
337 // mangled names, but we may not always get them. This happens when
338 // the compiler decides not to emit the function (e.g., it was inlined
339 // and removed). In this case, the binary will not have the linkage
340 // name for the function, so the profiler will emit the function's
341 // unmangled name, which may contain characters like ':' and '>' in its
342 // name (member functions, templates, etc).
343 //
344 // The only requirement we place on the identifier, then, is that it
345 // should not begin with a number.
346 if ((*LineIt)[0] != ' ') {
347 uint64_t NumSamples, NumHeadSamples;
348 StringRef FName;
349 if (!ParseHead(Input: *LineIt, FName, NumSamples, NumHeadSamples)) {
350 reportError(LineNumber: LineIt.line_number(),
351 Msg: "Expected 'mangled_name:NUM:NUM', found " + *LineIt);
352 return sampleprof_error::malformed;
353 }
354 DepthMetadata = 0;
355 SampleContext FContext(FName, CSNameTable);
356 if (FContext.hasContext())
357 ++CSProfileCount;
358 FunctionSamples &FProfile = Profiles.create(Ctx: FContext);
359 mergeSampleProfErrors(Accumulator&: Result, Result: FProfile.addTotalSamples(Num: NumSamples));
360 mergeSampleProfErrors(Accumulator&: Result, Result: FProfile.addHeadSamples(Num: NumHeadSamples));
361 InlineStack.clear();
362 InlineStack.push_back(Elt: &FProfile);
363 } else {
364 uint64_t NumSamples;
365 StringRef FName;
366 DenseMap<StringRef, uint64_t> TargetCountMap;
367 uint32_t Depth, LineOffset, Discriminator;
368 LineType LineTy;
369 uint64_t FunctionHash = 0;
370 uint32_t Attributes = 0;
371 if (!ParseLine(Input: *LineIt, LineTy, Depth, NumSamples, LineOffset,
372 Discriminator, CalleeName&: FName, TargetCountMap, FunctionHash,
373 Attributes)) {
374 reportError(LineNumber: LineIt.line_number(),
375 Msg: "Expected 'NUM[.NUM]: NUM[ mangled_name:NUM]*', found " +
376 *LineIt);
377 return sampleprof_error::malformed;
378 }
379 if (LineTy != LineType::Metadata && Depth == DepthMetadata) {
380 // Metadata must be put at the end of a function profile.
381 reportError(LineNumber: LineIt.line_number(),
382 Msg: "Found non-metadata after metadata: " + *LineIt);
383 return sampleprof_error::malformed;
384 }
385
386 // Here we handle FS discriminators.
387 Discriminator &= getDiscriminatorMask();
388
389 while (InlineStack.size() > Depth) {
390 InlineStack.pop_back();
391 }
392 switch (LineTy) {
393 case LineType::CallSiteProfile: {
394 FunctionSamples &FSamples = InlineStack.back()->functionSamplesAt(
395 Loc: LineLocation(LineOffset, Discriminator))[FunctionId(FName)];
396 FSamples.setFunction(FunctionId(FName));
397 mergeSampleProfErrors(Accumulator&: Result, Result: FSamples.addTotalSamples(Num: NumSamples));
398 InlineStack.push_back(Elt: &FSamples);
399 DepthMetadata = 0;
400 break;
401 }
402 case LineType::BodyProfile: {
403 while (InlineStack.size() > Depth) {
404 InlineStack.pop_back();
405 }
406 FunctionSamples &FProfile = *InlineStack.back();
407 for (const auto &name_count : TargetCountMap) {
408 mergeSampleProfErrors(Accumulator&: Result, Result: FProfile.addCalledTargetSamples(
409 LineOffset, Discriminator,
410 Func: FunctionId(name_count.first),
411 Num: name_count.second));
412 }
413 mergeSampleProfErrors(
414 Accumulator&: Result,
415 Result: FProfile.addBodySamples(LineOffset, Discriminator, Num: NumSamples));
416 break;
417 }
418 case LineType::Metadata: {
419 FunctionSamples &FProfile = *InlineStack.back();
420 if (FunctionHash) {
421 FProfile.setFunctionHash(FunctionHash);
422 if (Depth == 1)
423 ++TopLevelProbeProfileCount;
424 }
425 FProfile.getContext().setAllAttributes(Attributes);
426 if (Attributes & (uint32_t)ContextShouldBeInlined)
427 ProfileIsPreInlined = true;
428 DepthMetadata = Depth;
429 break;
430 }
431 }
432 }
433 }
434
435 assert((CSProfileCount == 0 || CSProfileCount == Profiles.size()) &&
436 "Cannot have both context-sensitive and regular profile");
437 ProfileIsCS = (CSProfileCount > 0);
438 assert((TopLevelProbeProfileCount == 0 ||
439 TopLevelProbeProfileCount == Profiles.size()) &&
440 "Cannot have both probe-based profiles and regular profiles");
441 ProfileIsProbeBased = (TopLevelProbeProfileCount > 0);
442 FunctionSamples::ProfileIsProbeBased = ProfileIsProbeBased;
443 FunctionSamples::ProfileIsCS = ProfileIsCS;
444 FunctionSamples::ProfileIsPreInlined = ProfileIsPreInlined;
445
446 if (Result == sampleprof_error::success)
447 computeSummary();
448
449 return Result;
450}
451
452bool SampleProfileReaderText::hasFormat(const MemoryBuffer &Buffer) {
453 bool result = false;
454
455 // Check that the first non-comment line is a valid function header.
456 line_iterator LineIt(Buffer, /*SkipBlanks=*/true, '#');
457 if (!LineIt.is_at_eof()) {
458 if ((*LineIt)[0] != ' ') {
459 uint64_t NumSamples, NumHeadSamples;
460 StringRef FName;
461 result = ParseHead(Input: *LineIt, FName, NumSamples, NumHeadSamples);
462 }
463 }
464
465 return result;
466}
467
468template <typename T> ErrorOr<T> SampleProfileReaderBinary::readNumber() {
469 unsigned NumBytesRead = 0;
470 uint64_t Val = decodeULEB128(p: Data, n: &NumBytesRead);
471
472 if (Val > std::numeric_limits<T>::max()) {
473 std::error_code EC = sampleprof_error::malformed;
474 reportError(LineNumber: 0, Msg: EC.message());
475 return EC;
476 } else if (Data + NumBytesRead > End) {
477 std::error_code EC = sampleprof_error::truncated;
478 reportError(LineNumber: 0, Msg: EC.message());
479 return EC;
480 }
481
482 Data += NumBytesRead;
483 return static_cast<T>(Val);
484}
485
486ErrorOr<StringRef> SampleProfileReaderBinary::readString() {
487 StringRef Str(reinterpret_cast<const char *>(Data));
488 if (Data + Str.size() + 1 > End) {
489 std::error_code EC = sampleprof_error::truncated;
490 reportError(LineNumber: 0, Msg: EC.message());
491 return EC;
492 }
493
494 Data += Str.size() + 1;
495 return Str;
496}
497
498template <typename T>
499ErrorOr<T> SampleProfileReaderBinary::readUnencodedNumber() {
500 if (Data + sizeof(T) > End) {
501 std::error_code EC = sampleprof_error::truncated;
502 reportError(LineNumber: 0, Msg: EC.message());
503 return EC;
504 }
505
506 using namespace support;
507 T Val = endian::readNext<T, llvm::endianness::little>(Data);
508 return Val;
509}
510
511template <typename T>
512inline ErrorOr<size_t> SampleProfileReaderBinary::readStringIndex(T &Table) {
513 auto Idx = readNumber<size_t>();
514 if (std::error_code EC = Idx.getError())
515 return EC;
516 if (*Idx >= Table.size())
517 return sampleprof_error::truncated_name_table;
518 return *Idx;
519}
520
521ErrorOr<FunctionId>
522SampleProfileReaderBinary::readStringFromTable(size_t *RetIdx) {
523 auto Idx = readStringIndex(Table&: NameTable);
524 if (std::error_code EC = Idx.getError())
525 return EC;
526 if (RetIdx)
527 *RetIdx = *Idx;
528 return NameTable[*Idx];
529}
530
531ErrorOr<SampleContextFrames>
532SampleProfileReaderBinary::readContextFromTable(size_t *RetIdx) {
533 auto ContextIdx = readNumber<size_t>();
534 if (std::error_code EC = ContextIdx.getError())
535 return EC;
536 if (*ContextIdx >= CSNameTable.size())
537 return sampleprof_error::truncated_name_table;
538 if (RetIdx)
539 *RetIdx = *ContextIdx;
540 return CSNameTable[*ContextIdx];
541}
542
543ErrorOr<std::pair<SampleContext, uint64_t>>
544SampleProfileReaderBinary::readSampleContextFromTable() {
545 SampleContext Context;
546 size_t Idx;
547 if (ProfileIsCS) {
548 auto FContext(readContextFromTable(RetIdx: &Idx));
549 if (std::error_code EC = FContext.getError())
550 return EC;
551 Context = SampleContext(*FContext);
552 } else {
553 auto FName(readStringFromTable(RetIdx: &Idx));
554 if (std::error_code EC = FName.getError())
555 return EC;
556 Context = SampleContext(*FName);
557 }
558 // Since MD5SampleContextStart may point to the profile's file data, need to
559 // make sure it is reading the same value on big endian CPU.
560 uint64_t Hash = support::endian::read64le(P: MD5SampleContextStart + Idx);
561 // Lazy computing of hash value, write back to the table to cache it. Only
562 // compute the context's hash value if it is being referenced for the first
563 // time.
564 if (Hash == 0) {
565 assert(MD5SampleContextStart == MD5SampleContextTable.data());
566 Hash = Context.getHashCode();
567 support::endian::write64le(P: &MD5SampleContextTable[Idx], V: Hash);
568 }
569 return std::make_pair(x&: Context, y&: Hash);
570}
571
572std::error_code
573SampleProfileReaderBinary::readProfile(FunctionSamples &FProfile) {
574 auto NumSamples = readNumber<uint64_t>();
575 if (std::error_code EC = NumSamples.getError())
576 return EC;
577 FProfile.addTotalSamples(Num: *NumSamples);
578
579 // Read the samples in the body.
580 auto NumRecords = readNumber<uint32_t>();
581 if (std::error_code EC = NumRecords.getError())
582 return EC;
583
584 for (uint32_t I = 0; I < *NumRecords; ++I) {
585 auto LineOffset = readNumber<uint64_t>();
586 if (std::error_code EC = LineOffset.getError())
587 return EC;
588
589 if (!isOffsetLegal(L: *LineOffset)) {
590 return std::error_code();
591 }
592
593 auto Discriminator = readNumber<uint64_t>();
594 if (std::error_code EC = Discriminator.getError())
595 return EC;
596
597 auto NumSamples = readNumber<uint64_t>();
598 if (std::error_code EC = NumSamples.getError())
599 return EC;
600
601 auto NumCalls = readNumber<uint32_t>();
602 if (std::error_code EC = NumCalls.getError())
603 return EC;
604
605 // Here we handle FS discriminators:
606 uint32_t DiscriminatorVal = (*Discriminator) & getDiscriminatorMask();
607
608 for (uint32_t J = 0; J < *NumCalls; ++J) {
609 auto CalledFunction(readStringFromTable());
610 if (std::error_code EC = CalledFunction.getError())
611 return EC;
612
613 auto CalledFunctionSamples = readNumber<uint64_t>();
614 if (std::error_code EC = CalledFunctionSamples.getError())
615 return EC;
616
617 FProfile.addCalledTargetSamples(LineOffset: *LineOffset, Discriminator: DiscriminatorVal,
618 Func: *CalledFunction, Num: *CalledFunctionSamples);
619 }
620
621 FProfile.addBodySamples(LineOffset: *LineOffset, Discriminator: DiscriminatorVal, Num: *NumSamples);
622 }
623
624 // Read all the samples for inlined function calls.
625 auto NumCallsites = readNumber<uint32_t>();
626 if (std::error_code EC = NumCallsites.getError())
627 return EC;
628
629 for (uint32_t J = 0; J < *NumCallsites; ++J) {
630 auto LineOffset = readNumber<uint64_t>();
631 if (std::error_code EC = LineOffset.getError())
632 return EC;
633
634 auto Discriminator = readNumber<uint64_t>();
635 if (std::error_code EC = Discriminator.getError())
636 return EC;
637
638 auto FName(readStringFromTable());
639 if (std::error_code EC = FName.getError())
640 return EC;
641
642 // Here we handle FS discriminators:
643 uint32_t DiscriminatorVal = (*Discriminator) & getDiscriminatorMask();
644
645 FunctionSamples &CalleeProfile = FProfile.functionSamplesAt(
646 Loc: LineLocation(*LineOffset, DiscriminatorVal))[*FName];
647 CalleeProfile.setFunction(*FName);
648 if (std::error_code EC = readProfile(FProfile&: CalleeProfile))
649 return EC;
650 }
651
652 return sampleprof_error::success;
653}
654
655std::error_code
656SampleProfileReaderBinary::readFuncProfile(const uint8_t *Start) {
657 Data = Start;
658 auto NumHeadSamples = readNumber<uint64_t>();
659 if (std::error_code EC = NumHeadSamples.getError())
660 return EC;
661
662 auto FContextHash(readSampleContextFromTable());
663 if (std::error_code EC = FContextHash.getError())
664 return EC;
665
666 auto &[FContext, Hash] = *FContextHash;
667 // Use the cached hash value for insertion instead of recalculating it.
668 auto Res = Profiles.try_emplace(Hash, Key: FContext, Args: FunctionSamples());
669 FunctionSamples &FProfile = Res.first->second;
670 FProfile.setContext(FContext);
671 FProfile.addHeadSamples(Num: *NumHeadSamples);
672
673 if (FContext.hasContext())
674 CSProfileCount++;
675
676 if (std::error_code EC = readProfile(FProfile))
677 return EC;
678 return sampleprof_error::success;
679}
680
681std::error_code SampleProfileReaderBinary::readImpl() {
682 ProfileIsFS = ProfileIsFSDisciminator;
683 FunctionSamples::ProfileIsFS = ProfileIsFS;
684 while (Data < End) {
685 if (std::error_code EC = readFuncProfile(Start: Data))
686 return EC;
687 }
688
689 return sampleprof_error::success;
690}
691
692std::error_code SampleProfileReaderExtBinaryBase::readOneSection(
693 const uint8_t *Start, uint64_t Size, const SecHdrTableEntry &Entry) {
694 Data = Start;
695 End = Start + Size;
696 switch (Entry.Type) {
697 case SecProfSummary:
698 if (std::error_code EC = readSummary())
699 return EC;
700 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagPartial))
701 Summary->setPartialProfile(true);
702 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagFullContext))
703 FunctionSamples::ProfileIsCS = ProfileIsCS = true;
704 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagIsPreInlined))
705 FunctionSamples::ProfileIsPreInlined = ProfileIsPreInlined = true;
706 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagFSDiscriminator))
707 FunctionSamples::ProfileIsFS = ProfileIsFS = true;
708 break;
709 case SecNameTable: {
710 bool FixedLengthMD5 =
711 hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagFixedLengthMD5);
712 bool UseMD5 = hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagMD5Name);
713 // UseMD5 means if THIS section uses MD5, ProfileIsMD5 means if the entire
714 // profile uses MD5 for function name matching in IPO passes.
715 ProfileIsMD5 = ProfileIsMD5 || UseMD5;
716 FunctionSamples::HasUniqSuffix =
717 hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagUniqSuffix);
718 if (std::error_code EC = readNameTableSec(IsMD5: UseMD5, FixedLengthMD5))
719 return EC;
720 break;
721 }
722 case SecCSNameTable: {
723 if (std::error_code EC = readCSNameTableSec())
724 return EC;
725 break;
726 }
727 case SecLBRProfile:
728 if (std::error_code EC = readFuncProfiles())
729 return EC;
730 break;
731 case SecFuncOffsetTable:
732 // If module is absent, we are using LLVM tools, and need to read all
733 // profiles, so skip reading the function offset table.
734 if (!M) {
735 Data = End;
736 } else {
737 assert((!ProfileIsCS ||
738 hasSecFlag(Entry, SecFuncOffsetFlags::SecFlagOrdered)) &&
739 "func offset table should always be sorted in CS profile");
740 if (std::error_code EC = readFuncOffsetTable())
741 return EC;
742 }
743 break;
744 case SecFuncMetadata: {
745 ProfileIsProbeBased =
746 hasSecFlag(Entry, Flag: SecFuncMetadataFlags::SecFlagIsProbeBased);
747 FunctionSamples::ProfileIsProbeBased = ProfileIsProbeBased;
748 bool HasAttribute =
749 hasSecFlag(Entry, Flag: SecFuncMetadataFlags::SecFlagHasAttribute);
750 if (std::error_code EC = readFuncMetadata(ProfileHasAttribute: HasAttribute))
751 return EC;
752 break;
753 }
754 case SecProfileSymbolList:
755 if (std::error_code EC = readProfileSymbolList())
756 return EC;
757 break;
758 default:
759 if (std::error_code EC = readCustomSection(Entry))
760 return EC;
761 break;
762 }
763 return sampleprof_error::success;
764}
765
766bool SampleProfileReaderExtBinaryBase::useFuncOffsetList() const {
767 // If profile is CS, the function offset section is expected to consist of
768 // sequences of contexts in pre-order layout
769 // (e.g. [A, A:1 @ B, A:1 @ B:2.3 @ C] [D, D:1 @ E]), so that when a matched
770 // context in the module is found, the profiles of all its callees are
771 // recursively loaded. A list is needed since the order of profiles matters.
772 if (ProfileIsCS)
773 return true;
774
775 // If the profile is MD5, use the map container to lookup functions in
776 // the module. A remapper has no use on MD5 names.
777 if (useMD5())
778 return false;
779
780 // Profile is not MD5 and if a remapper is present, the remapped name of
781 // every function needed to be matched against the module, so use the list
782 // container since each entry is accessed.
783 if (Remapper)
784 return true;
785
786 // Otherwise use the map container for faster lookup.
787 // TODO: If the cardinality of the function offset section is much smaller
788 // than the number of functions in the module, using the list container can
789 // be always faster, but we need to figure out the constant factor to
790 // determine the cutoff.
791 return false;
792}
793
794
795bool SampleProfileReaderExtBinaryBase::collectFuncsFromModule() {
796 if (!M)
797 return false;
798 FuncsToUse.clear();
799 for (auto &F : *M)
800 FuncsToUse.insert(V: FunctionSamples::getCanonicalFnName(F));
801 return true;
802}
803
804std::error_code SampleProfileReaderExtBinaryBase::readFuncOffsetTable() {
805 // If there are more than one function offset section, the profile associated
806 // with the previous section has to be done reading before next one is read.
807 FuncOffsetTable.clear();
808 FuncOffsetList.clear();
809
810 auto Size = readNumber<uint64_t>();
811 if (std::error_code EC = Size.getError())
812 return EC;
813
814 bool UseFuncOffsetList = useFuncOffsetList();
815 if (UseFuncOffsetList)
816 FuncOffsetList.reserve(n: *Size);
817 else
818 FuncOffsetTable.reserve(NumEntries: *Size);
819
820 for (uint64_t I = 0; I < *Size; ++I) {
821 auto FContextHash(readSampleContextFromTable());
822 if (std::error_code EC = FContextHash.getError())
823 return EC;
824
825 auto &[FContext, Hash] = *FContextHash;
826 auto Offset = readNumber<uint64_t>();
827 if (std::error_code EC = Offset.getError())
828 return EC;
829
830 if (UseFuncOffsetList)
831 FuncOffsetList.emplace_back(args&: FContext, args&: *Offset);
832 else
833 // Because Porfiles replace existing value with new value if collision
834 // happens, we also use the latest offset so that they are consistent.
835 FuncOffsetTable[Hash] = *Offset;
836 }
837
838 return sampleprof_error::success;
839}
840
841std::error_code SampleProfileReaderExtBinaryBase::readFuncProfiles() {
842 // Collect functions used by current module if the Reader has been
843 // given a module.
844 // collectFuncsFromModule uses FunctionSamples::getCanonicalFnName
845 // which will query FunctionSamples::HasUniqSuffix, so it has to be
846 // called after FunctionSamples::HasUniqSuffix is set, i.e. after
847 // NameTable section is read.
848 bool LoadFuncsToBeUsed = collectFuncsFromModule();
849
850 // When LoadFuncsToBeUsed is false, we are using LLVM tool, need to read all
851 // profiles.
852 const uint8_t *Start = Data;
853 if (!LoadFuncsToBeUsed) {
854 while (Data < End) {
855 if (std::error_code EC = readFuncProfile(Start: Data))
856 return EC;
857 }
858 assert(Data == End && "More data is read than expected");
859 } else {
860 // Load function profiles on demand.
861 if (Remapper) {
862 for (auto Name : FuncsToUse) {
863 Remapper->insert(FunctionName: Name);
864 }
865 }
866
867 if (ProfileIsCS) {
868 assert(useFuncOffsetList());
869 DenseSet<uint64_t> FuncGuidsToUse;
870 if (useMD5()) {
871 for (auto Name : FuncsToUse)
872 FuncGuidsToUse.insert(V: Function::getGUID(GlobalName: Name));
873 }
874
875 // For each function in current module, load all context profiles for
876 // the function as well as their callee contexts which can help profile
877 // guided importing for ThinLTO. This can be achieved by walking
878 // through an ordered context container, where contexts are laid out
879 // as if they were walked in preorder of a context trie. While
880 // traversing the trie, a link to the highest common ancestor node is
881 // kept so that all of its decendants will be loaded.
882 const SampleContext *CommonContext = nullptr;
883 for (const auto &NameOffset : FuncOffsetList) {
884 const auto &FContext = NameOffset.first;
885 FunctionId FName = FContext.getFunction();
886 StringRef FNameString;
887 if (!useMD5())
888 FNameString = FName.stringRef();
889
890 // For function in the current module, keep its farthest ancestor
891 // context. This can be used to load itself and its child and
892 // sibling contexts.
893 if ((useMD5() && FuncGuidsToUse.count(V: FName.getHashCode())) ||
894 (!useMD5() && (FuncsToUse.count(V: FNameString) ||
895 (Remapper && Remapper->exist(FunctionName: FNameString))))) {
896 if (!CommonContext || !CommonContext->isPrefixOf(That: FContext))
897 CommonContext = &FContext;
898 }
899
900 if (CommonContext == &FContext ||
901 (CommonContext && CommonContext->isPrefixOf(That: FContext))) {
902 // Load profile for the current context which originated from
903 // the common ancestor.
904 const uint8_t *FuncProfileAddr = Start + NameOffset.second;
905 if (std::error_code EC = readFuncProfile(Start: FuncProfileAddr))
906 return EC;
907 }
908 }
909 } else if (useMD5()) {
910 assert(!useFuncOffsetList());
911 for (auto Name : FuncsToUse) {
912 auto GUID = MD5Hash(Str: Name);
913 auto iter = FuncOffsetTable.find(Val: GUID);
914 if (iter == FuncOffsetTable.end())
915 continue;
916 const uint8_t *FuncProfileAddr = Start + iter->second;
917 if (std::error_code EC = readFuncProfile(Start: FuncProfileAddr))
918 return EC;
919 }
920 } else if (Remapper) {
921 assert(useFuncOffsetList());
922 for (auto NameOffset : FuncOffsetList) {
923 SampleContext FContext(NameOffset.first);
924 auto FuncName = FContext.getFunction();
925 StringRef FuncNameStr = FuncName.stringRef();
926 if (!FuncsToUse.count(V: FuncNameStr) && !Remapper->exist(FunctionName: FuncNameStr))
927 continue;
928 const uint8_t *FuncProfileAddr = Start + NameOffset.second;
929 if (std::error_code EC = readFuncProfile(Start: FuncProfileAddr))
930 return EC;
931 }
932 } else {
933 assert(!useFuncOffsetList());
934 for (auto Name : FuncsToUse) {
935 auto iter = FuncOffsetTable.find(Val: MD5Hash(Str: Name));
936 if (iter == FuncOffsetTable.end())
937 continue;
938 const uint8_t *FuncProfileAddr = Start + iter->second;
939 if (std::error_code EC = readFuncProfile(Start: FuncProfileAddr))
940 return EC;
941 }
942 }
943 Data = End;
944 }
945 assert((CSProfileCount == 0 || CSProfileCount == Profiles.size()) &&
946 "Cannot have both context-sensitive and regular profile");
947 assert((!CSProfileCount || ProfileIsCS) &&
948 "Section flag should be consistent with actual profile");
949 return sampleprof_error::success;
950}
951
952std::error_code SampleProfileReaderExtBinaryBase::readProfileSymbolList() {
953 if (!ProfSymList)
954 ProfSymList = std::make_unique<ProfileSymbolList>();
955
956 if (std::error_code EC = ProfSymList->read(Data, ListSize: End - Data))
957 return EC;
958
959 Data = End;
960 return sampleprof_error::success;
961}
962
963std::error_code SampleProfileReaderExtBinaryBase::decompressSection(
964 const uint8_t *SecStart, const uint64_t SecSize,
965 const uint8_t *&DecompressBuf, uint64_t &DecompressBufSize) {
966 Data = SecStart;
967 End = SecStart + SecSize;
968 auto DecompressSize = readNumber<uint64_t>();
969 if (std::error_code EC = DecompressSize.getError())
970 return EC;
971 DecompressBufSize = *DecompressSize;
972
973 auto CompressSize = readNumber<uint64_t>();
974 if (std::error_code EC = CompressSize.getError())
975 return EC;
976
977 if (!llvm::compression::zlib::isAvailable())
978 return sampleprof_error::zlib_unavailable;
979
980 uint8_t *Buffer = Allocator.Allocate<uint8_t>(Num: DecompressBufSize);
981 size_t UCSize = DecompressBufSize;
982 llvm::Error E = compression::zlib::decompress(Input: ArrayRef(Data, *CompressSize),
983 Output: Buffer, UncompressedSize&: UCSize);
984 if (E)
985 return sampleprof_error::uncompress_failed;
986 DecompressBuf = reinterpret_cast<const uint8_t *>(Buffer);
987 return sampleprof_error::success;
988}
989
990std::error_code SampleProfileReaderExtBinaryBase::readImpl() {
991 const uint8_t *BufStart =
992 reinterpret_cast<const uint8_t *>(Buffer->getBufferStart());
993
994 for (auto &Entry : SecHdrTable) {
995 // Skip empty section.
996 if (!Entry.Size)
997 continue;
998
999 // Skip sections without context when SkipFlatProf is true.
1000 if (SkipFlatProf && hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagFlat))
1001 continue;
1002
1003 const uint8_t *SecStart = BufStart + Entry.Offset;
1004 uint64_t SecSize = Entry.Size;
1005
1006 // If the section is compressed, decompress it into a buffer
1007 // DecompressBuf before reading the actual data. The pointee of
1008 // 'Data' will be changed to buffer hold by DecompressBuf
1009 // temporarily when reading the actual data.
1010 bool isCompressed = hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagCompress);
1011 if (isCompressed) {
1012 const uint8_t *DecompressBuf;
1013 uint64_t DecompressBufSize;
1014 if (std::error_code EC = decompressSection(
1015 SecStart, SecSize, DecompressBuf, DecompressBufSize))
1016 return EC;
1017 SecStart = DecompressBuf;
1018 SecSize = DecompressBufSize;
1019 }
1020
1021 if (std::error_code EC = readOneSection(Start: SecStart, Size: SecSize, Entry))
1022 return EC;
1023 if (Data != SecStart + SecSize)
1024 return sampleprof_error::malformed;
1025
1026 // Change the pointee of 'Data' from DecompressBuf to original Buffer.
1027 if (isCompressed) {
1028 Data = BufStart + Entry.Offset;
1029 End = BufStart + Buffer->getBufferSize();
1030 }
1031 }
1032
1033 return sampleprof_error::success;
1034}
1035
1036std::error_code SampleProfileReaderRawBinary::verifySPMagic(uint64_t Magic) {
1037 if (Magic == SPMagic())
1038 return sampleprof_error::success;
1039 return sampleprof_error::bad_magic;
1040}
1041
1042std::error_code SampleProfileReaderExtBinary::verifySPMagic(uint64_t Magic) {
1043 if (Magic == SPMagic(Format: SPF_Ext_Binary))
1044 return sampleprof_error::success;
1045 return sampleprof_error::bad_magic;
1046}
1047
1048std::error_code SampleProfileReaderBinary::readNameTable() {
1049 auto Size = readNumber<size_t>();
1050 if (std::error_code EC = Size.getError())
1051 return EC;
1052
1053 // Normally if useMD5 is true, the name table should have MD5 values, not
1054 // strings, however in the case that ExtBinary profile has multiple name
1055 // tables mixing string and MD5, all of them have to be normalized to use MD5,
1056 // because optimization passes can only handle either type.
1057 bool UseMD5 = useMD5();
1058
1059 NameTable.clear();
1060 NameTable.reserve(n: *Size);
1061 if (!ProfileIsCS) {
1062 MD5SampleContextTable.clear();
1063 if (UseMD5)
1064 MD5SampleContextTable.reserve(n: *Size);
1065 else
1066 // If we are using strings, delay MD5 computation since only a portion of
1067 // names are used by top level functions. Use 0 to indicate MD5 value is
1068 // to be calculated as no known string has a MD5 value of 0.
1069 MD5SampleContextTable.resize(new_size: *Size);
1070 }
1071 for (size_t I = 0; I < *Size; ++I) {
1072 auto Name(readString());
1073 if (std::error_code EC = Name.getError())
1074 return EC;
1075 if (UseMD5) {
1076 FunctionId FID(*Name);
1077 if (!ProfileIsCS)
1078 MD5SampleContextTable.emplace_back(args: FID.getHashCode());
1079 NameTable.emplace_back(args&: FID);
1080 } else
1081 NameTable.push_back(x: FunctionId(*Name));
1082 }
1083 if (!ProfileIsCS)
1084 MD5SampleContextStart = MD5SampleContextTable.data();
1085 return sampleprof_error::success;
1086}
1087
1088std::error_code
1089SampleProfileReaderExtBinaryBase::readNameTableSec(bool IsMD5,
1090 bool FixedLengthMD5) {
1091 if (FixedLengthMD5) {
1092 if (!IsMD5)
1093 errs() << "If FixedLengthMD5 is true, UseMD5 has to be true";
1094 auto Size = readNumber<size_t>();
1095 if (std::error_code EC = Size.getError())
1096 return EC;
1097
1098 assert(Data + (*Size) * sizeof(uint64_t) == End &&
1099 "Fixed length MD5 name table does not contain specified number of "
1100 "entries");
1101 if (Data + (*Size) * sizeof(uint64_t) > End)
1102 return sampleprof_error::truncated;
1103
1104 NameTable.clear();
1105 NameTable.reserve(n: *Size);
1106 for (size_t I = 0; I < *Size; ++I) {
1107 using namespace support;
1108 uint64_t FID = endian::read<uint64_t, endianness::little, unaligned>(
1109 memory: Data + I * sizeof(uint64_t));
1110 NameTable.emplace_back(args: FunctionId(FID));
1111 }
1112 if (!ProfileIsCS)
1113 MD5SampleContextStart = reinterpret_cast<const uint64_t *>(Data);
1114 Data = Data + (*Size) * sizeof(uint64_t);
1115 return sampleprof_error::success;
1116 }
1117
1118 if (IsMD5) {
1119 assert(!FixedLengthMD5 && "FixedLengthMD5 should be unreachable here");
1120 auto Size = readNumber<size_t>();
1121 if (std::error_code EC = Size.getError())
1122 return EC;
1123
1124 NameTable.clear();
1125 NameTable.reserve(n: *Size);
1126 if (!ProfileIsCS)
1127 MD5SampleContextTable.resize(new_size: *Size);
1128 for (size_t I = 0; I < *Size; ++I) {
1129 auto FID = readNumber<uint64_t>();
1130 if (std::error_code EC = FID.getError())
1131 return EC;
1132 if (!ProfileIsCS)
1133 support::endian::write64le(P: &MD5SampleContextTable[I], V: *FID);
1134 NameTable.emplace_back(args: FunctionId(*FID));
1135 }
1136 if (!ProfileIsCS)
1137 MD5SampleContextStart = MD5SampleContextTable.data();
1138 return sampleprof_error::success;
1139 }
1140
1141 return SampleProfileReaderBinary::readNameTable();
1142}
1143
1144// Read in the CS name table section, which basically contains a list of context
1145// vectors. Each element of a context vector, aka a frame, refers to the
1146// underlying raw function names that are stored in the name table, as well as
1147// a callsite identifier that only makes sense for non-leaf frames.
1148std::error_code SampleProfileReaderExtBinaryBase::readCSNameTableSec() {
1149 auto Size = readNumber<size_t>();
1150 if (std::error_code EC = Size.getError())
1151 return EC;
1152
1153 CSNameTable.clear();
1154 CSNameTable.reserve(n: *Size);
1155 if (ProfileIsCS) {
1156 // Delay MD5 computation of CS context until they are needed. Use 0 to
1157 // indicate MD5 value is to be calculated as no known string has a MD5
1158 // value of 0.
1159 MD5SampleContextTable.clear();
1160 MD5SampleContextTable.resize(new_size: *Size);
1161 MD5SampleContextStart = MD5SampleContextTable.data();
1162 }
1163 for (size_t I = 0; I < *Size; ++I) {
1164 CSNameTable.emplace_back(args: SampleContextFrameVector());
1165 auto ContextSize = readNumber<uint32_t>();
1166 if (std::error_code EC = ContextSize.getError())
1167 return EC;
1168 for (uint32_t J = 0; J < *ContextSize; ++J) {
1169 auto FName(readStringFromTable());
1170 if (std::error_code EC = FName.getError())
1171 return EC;
1172 auto LineOffset = readNumber<uint64_t>();
1173 if (std::error_code EC = LineOffset.getError())
1174 return EC;
1175
1176 if (!isOffsetLegal(L: *LineOffset))
1177 return std::error_code();
1178
1179 auto Discriminator = readNumber<uint64_t>();
1180 if (std::error_code EC = Discriminator.getError())
1181 return EC;
1182
1183 CSNameTable.back().emplace_back(
1184 Args&: FName.get(), Args: LineLocation(LineOffset.get(), Discriminator.get()));
1185 }
1186 }
1187
1188 return sampleprof_error::success;
1189}
1190
1191std::error_code
1192SampleProfileReaderExtBinaryBase::readFuncMetadata(bool ProfileHasAttribute,
1193 FunctionSamples *FProfile) {
1194 if (Data < End) {
1195 if (ProfileIsProbeBased) {
1196 auto Checksum = readNumber<uint64_t>();
1197 if (std::error_code EC = Checksum.getError())
1198 return EC;
1199 if (FProfile)
1200 FProfile->setFunctionHash(*Checksum);
1201 }
1202
1203 if (ProfileHasAttribute) {
1204 auto Attributes = readNumber<uint32_t>();
1205 if (std::error_code EC = Attributes.getError())
1206 return EC;
1207 if (FProfile)
1208 FProfile->getContext().setAllAttributes(*Attributes);
1209 }
1210
1211 if (!ProfileIsCS) {
1212 // Read all the attributes for inlined function calls.
1213 auto NumCallsites = readNumber<uint32_t>();
1214 if (std::error_code EC = NumCallsites.getError())
1215 return EC;
1216
1217 for (uint32_t J = 0; J < *NumCallsites; ++J) {
1218 auto LineOffset = readNumber<uint64_t>();
1219 if (std::error_code EC = LineOffset.getError())
1220 return EC;
1221
1222 auto Discriminator = readNumber<uint64_t>();
1223 if (std::error_code EC = Discriminator.getError())
1224 return EC;
1225
1226 auto FContextHash(readSampleContextFromTable());
1227 if (std::error_code EC = FContextHash.getError())
1228 return EC;
1229
1230 auto &[FContext, Hash] = *FContextHash;
1231 FunctionSamples *CalleeProfile = nullptr;
1232 if (FProfile) {
1233 CalleeProfile = const_cast<FunctionSamples *>(
1234 &FProfile->functionSamplesAt(Loc: LineLocation(
1235 *LineOffset,
1236 *Discriminator))[FContext.getFunction()]);
1237 }
1238 if (std::error_code EC =
1239 readFuncMetadata(ProfileHasAttribute, FProfile: CalleeProfile))
1240 return EC;
1241 }
1242 }
1243 }
1244
1245 return sampleprof_error::success;
1246}
1247
1248std::error_code
1249SampleProfileReaderExtBinaryBase::readFuncMetadata(bool ProfileHasAttribute) {
1250 while (Data < End) {
1251 auto FContextHash(readSampleContextFromTable());
1252 if (std::error_code EC = FContextHash.getError())
1253 return EC;
1254 auto &[FContext, Hash] = *FContextHash;
1255 FunctionSamples *FProfile = nullptr;
1256 auto It = Profiles.find(Ctx: FContext);
1257 if (It != Profiles.end())
1258 FProfile = &It->second;
1259
1260 if (std::error_code EC = readFuncMetadata(ProfileHasAttribute, FProfile))
1261 return EC;
1262 }
1263
1264 assert(Data == End && "More data is read than expected");
1265 return sampleprof_error::success;
1266}
1267
1268std::error_code
1269SampleProfileReaderExtBinaryBase::readSecHdrTableEntry(uint64_t Idx) {
1270 SecHdrTableEntry Entry;
1271 auto Type = readUnencodedNumber<uint64_t>();
1272 if (std::error_code EC = Type.getError())
1273 return EC;
1274 Entry.Type = static_cast<SecType>(*Type);
1275
1276 auto Flags = readUnencodedNumber<uint64_t>();
1277 if (std::error_code EC = Flags.getError())
1278 return EC;
1279 Entry.Flags = *Flags;
1280
1281 auto Offset = readUnencodedNumber<uint64_t>();
1282 if (std::error_code EC = Offset.getError())
1283 return EC;
1284 Entry.Offset = *Offset;
1285
1286 auto Size = readUnencodedNumber<uint64_t>();
1287 if (std::error_code EC = Size.getError())
1288 return EC;
1289 Entry.Size = *Size;
1290
1291 Entry.LayoutIndex = Idx;
1292 SecHdrTable.push_back(x: std::move(Entry));
1293 return sampleprof_error::success;
1294}
1295
1296std::error_code SampleProfileReaderExtBinaryBase::readSecHdrTable() {
1297 auto EntryNum = readUnencodedNumber<uint64_t>();
1298 if (std::error_code EC = EntryNum.getError())
1299 return EC;
1300
1301 for (uint64_t i = 0; i < (*EntryNum); i++)
1302 if (std::error_code EC = readSecHdrTableEntry(Idx: i))
1303 return EC;
1304
1305 return sampleprof_error::success;
1306}
1307
1308std::error_code SampleProfileReaderExtBinaryBase::readHeader() {
1309 const uint8_t *BufStart =
1310 reinterpret_cast<const uint8_t *>(Buffer->getBufferStart());
1311 Data = BufStart;
1312 End = BufStart + Buffer->getBufferSize();
1313
1314 if (std::error_code EC = readMagicIdent())
1315 return EC;
1316
1317 if (std::error_code EC = readSecHdrTable())
1318 return EC;
1319
1320 return sampleprof_error::success;
1321}
1322
1323uint64_t SampleProfileReaderExtBinaryBase::getSectionSize(SecType Type) {
1324 uint64_t Size = 0;
1325 for (auto &Entry : SecHdrTable) {
1326 if (Entry.Type == Type)
1327 Size += Entry.Size;
1328 }
1329 return Size;
1330}
1331
1332uint64_t SampleProfileReaderExtBinaryBase::getFileSize() {
1333 // Sections in SecHdrTable is not necessarily in the same order as
1334 // sections in the profile because section like FuncOffsetTable needs
1335 // to be written after section LBRProfile but needs to be read before
1336 // section LBRProfile, so we cannot simply use the last entry in
1337 // SecHdrTable to calculate the file size.
1338 uint64_t FileSize = 0;
1339 for (auto &Entry : SecHdrTable) {
1340 FileSize = std::max(a: Entry.Offset + Entry.Size, b: FileSize);
1341 }
1342 return FileSize;
1343}
1344
1345static std::string getSecFlagsStr(const SecHdrTableEntry &Entry) {
1346 std::string Flags;
1347 if (hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagCompress))
1348 Flags.append(s: "{compressed,");
1349 else
1350 Flags.append(s: "{");
1351
1352 if (hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagFlat))
1353 Flags.append(s: "flat,");
1354
1355 switch (Entry.Type) {
1356 case SecNameTable:
1357 if (hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagFixedLengthMD5))
1358 Flags.append(s: "fixlenmd5,");
1359 else if (hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagMD5Name))
1360 Flags.append(s: "md5,");
1361 if (hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagUniqSuffix))
1362 Flags.append(s: "uniq,");
1363 break;
1364 case SecProfSummary:
1365 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagPartial))
1366 Flags.append(s: "partial,");
1367 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagFullContext))
1368 Flags.append(s: "context,");
1369 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagIsPreInlined))
1370 Flags.append(s: "preInlined,");
1371 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagFSDiscriminator))
1372 Flags.append(s: "fs-discriminator,");
1373 break;
1374 case SecFuncOffsetTable:
1375 if (hasSecFlag(Entry, Flag: SecFuncOffsetFlags::SecFlagOrdered))
1376 Flags.append(s: "ordered,");
1377 break;
1378 case SecFuncMetadata:
1379 if (hasSecFlag(Entry, Flag: SecFuncMetadataFlags::SecFlagIsProbeBased))
1380 Flags.append(s: "probe,");
1381 if (hasSecFlag(Entry, Flag: SecFuncMetadataFlags::SecFlagHasAttribute))
1382 Flags.append(s: "attr,");
1383 break;
1384 default:
1385 break;
1386 }
1387 char &last = Flags.back();
1388 if (last == ',')
1389 last = '}';
1390 else
1391 Flags.append(s: "}");
1392 return Flags;
1393}
1394
1395bool SampleProfileReaderExtBinaryBase::dumpSectionInfo(raw_ostream &OS) {
1396 uint64_t TotalSecsSize = 0;
1397 for (auto &Entry : SecHdrTable) {
1398 OS << getSecName(Type: Entry.Type) << " - Offset: " << Entry.Offset
1399 << ", Size: " << Entry.Size << ", Flags: " << getSecFlagsStr(Entry)
1400 << "\n";
1401 ;
1402 TotalSecsSize += Entry.Size;
1403 }
1404 uint64_t HeaderSize = SecHdrTable.front().Offset;
1405 assert(HeaderSize + TotalSecsSize == getFileSize() &&
1406 "Size of 'header + sections' doesn't match the total size of profile");
1407
1408 OS << "Header Size: " << HeaderSize << "\n";
1409 OS << "Total Sections Size: " << TotalSecsSize << "\n";
1410 OS << "File Size: " << getFileSize() << "\n";
1411 return true;
1412}
1413
1414std::error_code SampleProfileReaderBinary::readMagicIdent() {
1415 // Read and check the magic identifier.
1416 auto Magic = readNumber<uint64_t>();
1417 if (std::error_code EC = Magic.getError())
1418 return EC;
1419 else if (std::error_code EC = verifySPMagic(Magic: *Magic))
1420 return EC;
1421
1422 // Read the version number.
1423 auto Version = readNumber<uint64_t>();
1424 if (std::error_code EC = Version.getError())
1425 return EC;
1426 else if (*Version != SPVersion())
1427 return sampleprof_error::unsupported_version;
1428
1429 return sampleprof_error::success;
1430}
1431
1432std::error_code SampleProfileReaderBinary::readHeader() {
1433 Data = reinterpret_cast<const uint8_t *>(Buffer->getBufferStart());
1434 End = Data + Buffer->getBufferSize();
1435
1436 if (std::error_code EC = readMagicIdent())
1437 return EC;
1438
1439 if (std::error_code EC = readSummary())
1440 return EC;
1441
1442 if (std::error_code EC = readNameTable())
1443 return EC;
1444 return sampleprof_error::success;
1445}
1446
1447std::error_code SampleProfileReaderBinary::readSummaryEntry(
1448 std::vector<ProfileSummaryEntry> &Entries) {
1449 auto Cutoff = readNumber<uint64_t>();
1450 if (std::error_code EC = Cutoff.getError())
1451 return EC;
1452
1453 auto MinBlockCount = readNumber<uint64_t>();
1454 if (std::error_code EC = MinBlockCount.getError())
1455 return EC;
1456
1457 auto NumBlocks = readNumber<uint64_t>();
1458 if (std::error_code EC = NumBlocks.getError())
1459 return EC;
1460
1461 Entries.emplace_back(args&: *Cutoff, args&: *MinBlockCount, args&: *NumBlocks);
1462 return sampleprof_error::success;
1463}
1464
1465std::error_code SampleProfileReaderBinary::readSummary() {
1466 auto TotalCount = readNumber<uint64_t>();
1467 if (std::error_code EC = TotalCount.getError())
1468 return EC;
1469
1470 auto MaxBlockCount = readNumber<uint64_t>();
1471 if (std::error_code EC = MaxBlockCount.getError())
1472 return EC;
1473
1474 auto MaxFunctionCount = readNumber<uint64_t>();
1475 if (std::error_code EC = MaxFunctionCount.getError())
1476 return EC;
1477
1478 auto NumBlocks = readNumber<uint64_t>();
1479 if (std::error_code EC = NumBlocks.getError())
1480 return EC;
1481
1482 auto NumFunctions = readNumber<uint64_t>();
1483 if (std::error_code EC = NumFunctions.getError())
1484 return EC;
1485
1486 auto NumSummaryEntries = readNumber<uint64_t>();
1487 if (std::error_code EC = NumSummaryEntries.getError())
1488 return EC;
1489
1490 std::vector<ProfileSummaryEntry> Entries;
1491 for (unsigned i = 0; i < *NumSummaryEntries; i++) {
1492 std::error_code EC = readSummaryEntry(Entries);
1493 if (EC != sampleprof_error::success)
1494 return EC;
1495 }
1496 Summary = std::make_unique<ProfileSummary>(
1497 args: ProfileSummary::PSK_Sample, args&: Entries, args&: *TotalCount, args&: *MaxBlockCount, args: 0,
1498 args&: *MaxFunctionCount, args&: *NumBlocks, args&: *NumFunctions);
1499
1500 return sampleprof_error::success;
1501}
1502
1503bool SampleProfileReaderRawBinary::hasFormat(const MemoryBuffer &Buffer) {
1504 const uint8_t *Data =
1505 reinterpret_cast<const uint8_t *>(Buffer.getBufferStart());
1506 uint64_t Magic = decodeULEB128(p: Data);
1507 return Magic == SPMagic();
1508}
1509
1510bool SampleProfileReaderExtBinary::hasFormat(const MemoryBuffer &Buffer) {
1511 const uint8_t *Data =
1512 reinterpret_cast<const uint8_t *>(Buffer.getBufferStart());
1513 uint64_t Magic = decodeULEB128(p: Data);
1514 return Magic == SPMagic(Format: SPF_Ext_Binary);
1515}
1516
1517std::error_code SampleProfileReaderGCC::skipNextWord() {
1518 uint32_t dummy;
1519 if (!GcovBuffer.readInt(Val&: dummy))
1520 return sampleprof_error::truncated;
1521 return sampleprof_error::success;
1522}
1523
1524template <typename T> ErrorOr<T> SampleProfileReaderGCC::readNumber() {
1525 if (sizeof(T) <= sizeof(uint32_t)) {
1526 uint32_t Val;
1527 if (GcovBuffer.readInt(Val) && Val <= std::numeric_limits<T>::max())
1528 return static_cast<T>(Val);
1529 } else if (sizeof(T) <= sizeof(uint64_t)) {
1530 uint64_t Val;
1531 if (GcovBuffer.readInt64(Val) && Val <= std::numeric_limits<T>::max())
1532 return static_cast<T>(Val);
1533 }
1534
1535 std::error_code EC = sampleprof_error::malformed;
1536 reportError(LineNumber: 0, Msg: EC.message());
1537 return EC;
1538}
1539
1540ErrorOr<StringRef> SampleProfileReaderGCC::readString() {
1541 StringRef Str;
1542 if (!GcovBuffer.readString(str&: Str))
1543 return sampleprof_error::truncated;
1544 return Str;
1545}
1546
1547std::error_code SampleProfileReaderGCC::readHeader() {
1548 // Read the magic identifier.
1549 if (!GcovBuffer.readGCDAFormat())
1550 return sampleprof_error::unrecognized_format;
1551
1552 // Read the version number. Note - the GCC reader does not validate this
1553 // version, but the profile creator generates v704.
1554 GCOV::GCOVVersion version;
1555 if (!GcovBuffer.readGCOVVersion(version))
1556 return sampleprof_error::unrecognized_format;
1557
1558 if (version != GCOV::V407)
1559 return sampleprof_error::unsupported_version;
1560
1561 // Skip the empty integer.
1562 if (std::error_code EC = skipNextWord())
1563 return EC;
1564
1565 return sampleprof_error::success;
1566}
1567
1568std::error_code SampleProfileReaderGCC::readSectionTag(uint32_t Expected) {
1569 uint32_t Tag;
1570 if (!GcovBuffer.readInt(Val&: Tag))
1571 return sampleprof_error::truncated;
1572
1573 if (Tag != Expected)
1574 return sampleprof_error::malformed;
1575
1576 if (std::error_code EC = skipNextWord())
1577 return EC;
1578
1579 return sampleprof_error::success;
1580}
1581
1582std::error_code SampleProfileReaderGCC::readNameTable() {
1583 if (std::error_code EC = readSectionTag(Expected: GCOVTagAFDOFileNames))
1584 return EC;
1585
1586 uint32_t Size;
1587 if (!GcovBuffer.readInt(Val&: Size))
1588 return sampleprof_error::truncated;
1589
1590 for (uint32_t I = 0; I < Size; ++I) {
1591 StringRef Str;
1592 if (!GcovBuffer.readString(str&: Str))
1593 return sampleprof_error::truncated;
1594 Names.push_back(x: std::string(Str));
1595 }
1596
1597 return sampleprof_error::success;
1598}
1599
1600std::error_code SampleProfileReaderGCC::readFunctionProfiles() {
1601 if (std::error_code EC = readSectionTag(Expected: GCOVTagAFDOFunction))
1602 return EC;
1603
1604 uint32_t NumFunctions;
1605 if (!GcovBuffer.readInt(Val&: NumFunctions))
1606 return sampleprof_error::truncated;
1607
1608 InlineCallStack Stack;
1609 for (uint32_t I = 0; I < NumFunctions; ++I)
1610 if (std::error_code EC = readOneFunctionProfile(InlineStack: Stack, Update: true, Offset: 0))
1611 return EC;
1612
1613 computeSummary();
1614 return sampleprof_error::success;
1615}
1616
1617std::error_code SampleProfileReaderGCC::readOneFunctionProfile(
1618 const InlineCallStack &InlineStack, bool Update, uint32_t Offset) {
1619 uint64_t HeadCount = 0;
1620 if (InlineStack.size() == 0)
1621 if (!GcovBuffer.readInt64(Val&: HeadCount))
1622 return sampleprof_error::truncated;
1623
1624 uint32_t NameIdx;
1625 if (!GcovBuffer.readInt(Val&: NameIdx))
1626 return sampleprof_error::truncated;
1627
1628 StringRef Name(Names[NameIdx]);
1629
1630 uint32_t NumPosCounts;
1631 if (!GcovBuffer.readInt(Val&: NumPosCounts))
1632 return sampleprof_error::truncated;
1633
1634 uint32_t NumCallsites;
1635 if (!GcovBuffer.readInt(Val&: NumCallsites))
1636 return sampleprof_error::truncated;
1637
1638 FunctionSamples *FProfile = nullptr;
1639 if (InlineStack.size() == 0) {
1640 // If this is a top function that we have already processed, do not
1641 // update its profile again. This happens in the presence of
1642 // function aliases. Since these aliases share the same function
1643 // body, there will be identical replicated profiles for the
1644 // original function. In this case, we simply not bother updating
1645 // the profile of the original function.
1646 FProfile = &Profiles[FunctionId(Name)];
1647 FProfile->addHeadSamples(Num: HeadCount);
1648 if (FProfile->getTotalSamples() > 0)
1649 Update = false;
1650 } else {
1651 // Otherwise, we are reading an inlined instance. The top of the
1652 // inline stack contains the profile of the caller. Insert this
1653 // callee in the caller's CallsiteMap.
1654 FunctionSamples *CallerProfile = InlineStack.front();
1655 uint32_t LineOffset = Offset >> 16;
1656 uint32_t Discriminator = Offset & 0xffff;
1657 FProfile = &CallerProfile->functionSamplesAt(
1658 Loc: LineLocation(LineOffset, Discriminator))[FunctionId(Name)];
1659 }
1660 FProfile->setFunction(FunctionId(Name));
1661
1662 for (uint32_t I = 0; I < NumPosCounts; ++I) {
1663 uint32_t Offset;
1664 if (!GcovBuffer.readInt(Val&: Offset))
1665 return sampleprof_error::truncated;
1666
1667 uint32_t NumTargets;
1668 if (!GcovBuffer.readInt(Val&: NumTargets))
1669 return sampleprof_error::truncated;
1670
1671 uint64_t Count;
1672 if (!GcovBuffer.readInt64(Val&: Count))
1673 return sampleprof_error::truncated;
1674
1675 // The line location is encoded in the offset as:
1676 // high 16 bits: line offset to the start of the function.
1677 // low 16 bits: discriminator.
1678 uint32_t LineOffset = Offset >> 16;
1679 uint32_t Discriminator = Offset & 0xffff;
1680
1681 InlineCallStack NewStack;
1682 NewStack.push_back(Elt: FProfile);
1683 llvm::append_range(C&: NewStack, R: InlineStack);
1684 if (Update) {
1685 // Walk up the inline stack, adding the samples on this line to
1686 // the total sample count of the callers in the chain.
1687 for (auto *CallerProfile : NewStack)
1688 CallerProfile->addTotalSamples(Num: Count);
1689
1690 // Update the body samples for the current profile.
1691 FProfile->addBodySamples(LineOffset, Discriminator, Num: Count);
1692 }
1693
1694 // Process the list of functions called at an indirect call site.
1695 // These are all the targets that a function pointer (or virtual
1696 // function) resolved at runtime.
1697 for (uint32_t J = 0; J < NumTargets; J++) {
1698 uint32_t HistVal;
1699 if (!GcovBuffer.readInt(Val&: HistVal))
1700 return sampleprof_error::truncated;
1701
1702 if (HistVal != HIST_TYPE_INDIR_CALL_TOPN)
1703 return sampleprof_error::malformed;
1704
1705 uint64_t TargetIdx;
1706 if (!GcovBuffer.readInt64(Val&: TargetIdx))
1707 return sampleprof_error::truncated;
1708 StringRef TargetName(Names[TargetIdx]);
1709
1710 uint64_t TargetCount;
1711 if (!GcovBuffer.readInt64(Val&: TargetCount))
1712 return sampleprof_error::truncated;
1713
1714 if (Update)
1715 FProfile->addCalledTargetSamples(LineOffset, Discriminator,
1716 Func: FunctionId(TargetName),
1717 Num: TargetCount);
1718 }
1719 }
1720
1721 // Process all the inlined callers into the current function. These
1722 // are all the callsites that were inlined into this function.
1723 for (uint32_t I = 0; I < NumCallsites; I++) {
1724 // The offset is encoded as:
1725 // high 16 bits: line offset to the start of the function.
1726 // low 16 bits: discriminator.
1727 uint32_t Offset;
1728 if (!GcovBuffer.readInt(Val&: Offset))
1729 return sampleprof_error::truncated;
1730 InlineCallStack NewStack;
1731 NewStack.push_back(Elt: FProfile);
1732 llvm::append_range(C&: NewStack, R: InlineStack);
1733 if (std::error_code EC = readOneFunctionProfile(InlineStack: NewStack, Update, Offset))
1734 return EC;
1735 }
1736
1737 return sampleprof_error::success;
1738}
1739
1740/// Read a GCC AutoFDO profile.
1741///
1742/// This format is generated by the Linux Perf conversion tool at
1743/// https://github.com/google/autofdo.
1744std::error_code SampleProfileReaderGCC::readImpl() {
1745 assert(!ProfileIsFSDisciminator && "Gcc profiles not support FSDisciminator");
1746 // Read the string table.
1747 if (std::error_code EC = readNameTable())
1748 return EC;
1749
1750 // Read the source profile.
1751 if (std::error_code EC = readFunctionProfiles())
1752 return EC;
1753
1754 return sampleprof_error::success;
1755}
1756
1757bool SampleProfileReaderGCC::hasFormat(const MemoryBuffer &Buffer) {
1758 StringRef Magic(reinterpret_cast<const char *>(Buffer.getBufferStart()));
1759 return Magic == "adcg*704";
1760}
1761
1762void SampleProfileReaderItaniumRemapper::applyRemapping(LLVMContext &Ctx) {
1763 // If the reader uses MD5 to represent string, we can't remap it because
1764 // we don't know what the original function names were.
1765 if (Reader.useMD5()) {
1766 Ctx.diagnose(DI: DiagnosticInfoSampleProfile(
1767 Reader.getBuffer()->getBufferIdentifier(),
1768 "Profile data remapping cannot be applied to profile data "
1769 "using MD5 names (original mangled names are not available).",
1770 DS_Warning));
1771 return;
1772 }
1773
1774 // CSSPGO-TODO: Remapper is not yet supported.
1775 // We will need to remap the entire context string.
1776 assert(Remappings && "should be initialized while creating remapper");
1777 for (auto &Sample : Reader.getProfiles()) {
1778 DenseSet<FunctionId> NamesInSample;
1779 Sample.second.findAllNames(NameSet&: NamesInSample);
1780 for (auto &Name : NamesInSample) {
1781 StringRef NameStr = Name.stringRef();
1782 if (auto Key = Remappings->insert(FunctionName: NameStr))
1783 NameMap.insert(KV: {Key, NameStr});
1784 }
1785 }
1786
1787 RemappingApplied = true;
1788}
1789
1790std::optional<StringRef>
1791SampleProfileReaderItaniumRemapper::lookUpNameInProfile(StringRef Fname) {
1792 if (auto Key = Remappings->lookup(FunctionName: Fname)) {
1793 StringRef Result = NameMap.lookup(Val: Key);
1794 if (!Result.empty())
1795 return Result;
1796 }
1797 return std::nullopt;
1798}
1799
1800/// Prepare a memory buffer for the contents of \p Filename.
1801///
1802/// \returns an error code indicating the status of the buffer.
1803static ErrorOr<std::unique_ptr<MemoryBuffer>>
1804setupMemoryBuffer(const Twine &Filename, vfs::FileSystem &FS) {
1805 auto BufferOrErr = Filename.str() == "-" ? MemoryBuffer::getSTDIN()
1806 : FS.getBufferForFile(Name: Filename);
1807 if (std::error_code EC = BufferOrErr.getError())
1808 return EC;
1809 auto Buffer = std::move(BufferOrErr.get());
1810
1811 return std::move(Buffer);
1812}
1813
1814/// Create a sample profile reader based on the format of the input file.
1815///
1816/// \param Filename The file to open.
1817///
1818/// \param C The LLVM context to use to emit diagnostics.
1819///
1820/// \param P The FSDiscriminatorPass.
1821///
1822/// \param RemapFilename The file used for profile remapping.
1823///
1824/// \returns an error code indicating the status of the created reader.
1825ErrorOr<std::unique_ptr<SampleProfileReader>>
1826SampleProfileReader::create(StringRef Filename, LLVMContext &C,
1827 vfs::FileSystem &FS, FSDiscriminatorPass P,
1828 StringRef RemapFilename) {
1829 auto BufferOrError = setupMemoryBuffer(Filename, FS);
1830 if (std::error_code EC = BufferOrError.getError())
1831 return EC;
1832 return create(B&: BufferOrError.get(), C, FS, P, RemapFilename);
1833}
1834
1835/// Create a sample profile remapper from the given input, to remap the
1836/// function names in the given profile data.
1837///
1838/// \param Filename The file to open.
1839///
1840/// \param Reader The profile reader the remapper is going to be applied to.
1841///
1842/// \param C The LLVM context to use to emit diagnostics.
1843///
1844/// \returns an error code indicating the status of the created reader.
1845ErrorOr<std::unique_ptr<SampleProfileReaderItaniumRemapper>>
1846SampleProfileReaderItaniumRemapper::create(StringRef Filename,
1847 vfs::FileSystem &FS,
1848 SampleProfileReader &Reader,
1849 LLVMContext &C) {
1850 auto BufferOrError = setupMemoryBuffer(Filename, FS);
1851 if (std::error_code EC = BufferOrError.getError())
1852 return EC;
1853 return create(B&: BufferOrError.get(), Reader, C);
1854}
1855
1856/// Create a sample profile remapper from the given input, to remap the
1857/// function names in the given profile data.
1858///
1859/// \param B The memory buffer to create the reader from (assumes ownership).
1860///
1861/// \param C The LLVM context to use to emit diagnostics.
1862///
1863/// \param Reader The profile reader the remapper is going to be applied to.
1864///
1865/// \returns an error code indicating the status of the created reader.
1866ErrorOr<std::unique_ptr<SampleProfileReaderItaniumRemapper>>
1867SampleProfileReaderItaniumRemapper::create(std::unique_ptr<MemoryBuffer> &B,
1868 SampleProfileReader &Reader,
1869 LLVMContext &C) {
1870 auto Remappings = std::make_unique<SymbolRemappingReader>();
1871 if (Error E = Remappings->read(B&: *B)) {
1872 handleAllErrors(
1873 E: std::move(E), Handlers: [&](const SymbolRemappingParseError &ParseError) {
1874 C.diagnose(DI: DiagnosticInfoSampleProfile(B->getBufferIdentifier(),
1875 ParseError.getLineNum(),
1876 ParseError.getMessage()));
1877 });
1878 return sampleprof_error::malformed;
1879 }
1880
1881 return std::make_unique<SampleProfileReaderItaniumRemapper>(
1882 args: std::move(B), args: std::move(Remappings), args&: Reader);
1883}
1884
1885/// Create a sample profile reader based on the format of the input data.
1886///
1887/// \param B The memory buffer to create the reader from (assumes ownership).
1888///
1889/// \param C The LLVM context to use to emit diagnostics.
1890///
1891/// \param P The FSDiscriminatorPass.
1892///
1893/// \param RemapFilename The file used for profile remapping.
1894///
1895/// \returns an error code indicating the status of the created reader.
1896ErrorOr<std::unique_ptr<SampleProfileReader>>
1897SampleProfileReader::create(std::unique_ptr<MemoryBuffer> &B, LLVMContext &C,
1898 vfs::FileSystem &FS, FSDiscriminatorPass P,
1899 StringRef RemapFilename) {
1900 std::unique_ptr<SampleProfileReader> Reader;
1901 if (SampleProfileReaderRawBinary::hasFormat(Buffer: *B))
1902 Reader.reset(p: new SampleProfileReaderRawBinary(std::move(B), C));
1903 else if (SampleProfileReaderExtBinary::hasFormat(Buffer: *B))
1904 Reader.reset(p: new SampleProfileReaderExtBinary(std::move(B), C));
1905 else if (SampleProfileReaderGCC::hasFormat(Buffer: *B))
1906 Reader.reset(p: new SampleProfileReaderGCC(std::move(B), C));
1907 else if (SampleProfileReaderText::hasFormat(Buffer: *B))
1908 Reader.reset(p: new SampleProfileReaderText(std::move(B), C));
1909 else
1910 return sampleprof_error::unrecognized_format;
1911
1912 if (!RemapFilename.empty()) {
1913 auto ReaderOrErr = SampleProfileReaderItaniumRemapper::create(
1914 Filename: RemapFilename, FS, Reader&: *Reader, C);
1915 if (std::error_code EC = ReaderOrErr.getError()) {
1916 std::string Msg = "Could not create remapper: " + EC.message();
1917 C.diagnose(DI: DiagnosticInfoSampleProfile(RemapFilename, Msg));
1918 return EC;
1919 }
1920 Reader->Remapper = std::move(ReaderOrErr.get());
1921 }
1922
1923 if (std::error_code EC = Reader->readHeader()) {
1924 return EC;
1925 }
1926
1927 Reader->setDiscriminatorMaskedBitFrom(P);
1928
1929 return std::move(Reader);
1930}
1931
1932// For text and GCC file formats, we compute the summary after reading the
1933// profile. Binary format has the profile summary in its header.
1934void SampleProfileReader::computeSummary() {
1935 SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs);
1936 Summary = Builder.computeSummaryForProfiles(Profiles);
1937}
1938