1//===- SampleProfReader.cpp - Read LLVM sample profile data ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the class that reads LLVM sample profiles. It
10// supports three file formats: text, binary and gcov.
11//
12// The textual representation is useful for debugging and testing purposes. The
13// binary representation is more compact, resulting in smaller file sizes.
14//
15// The gcov encoding is the one generated by GCC's AutoFDO profile creation
16// tool (https://github.com/google/autofdo)
17//
18// All three encodings can be used interchangeably as an input sample profile.
19//
20//===----------------------------------------------------------------------===//
21
22#include "llvm/ProfileData/SampleProfReader.h"
23#include "llvm/ADT/DenseMap.h"
24#include "llvm/ADT/STLExtras.h"
25#include "llvm/ADT/SmallSet.h"
26#include "llvm/ADT/StringRef.h"
27#include "llvm/IR/Module.h"
28#include "llvm/IR/ProfileSummary.h"
29#include "llvm/ProfileData/ProfileCommon.h"
30#include "llvm/ProfileData/SampleProf.h"
31#include "llvm/Support/CommandLine.h"
32#include "llvm/Support/Compression.h"
33#include "llvm/Support/ErrorOr.h"
34#include "llvm/Support/JSON.h"
35#include "llvm/Support/LEB128.h"
36#include "llvm/Support/LineIterator.h"
37#include "llvm/Support/MD5.h"
38#include "llvm/Support/MemoryBuffer.h"
39#include "llvm/Support/SaveAndRestore.h"
40#include "llvm/Support/VirtualFileSystem.h"
41#include "llvm/Support/raw_ostream.h"
42#include <algorithm>
43#include <cstddef>
44#include <cstdint>
45#include <cstring>
46#include <limits>
47#include <memory>
48#include <system_error>
49#include <vector>
50
51using namespace llvm;
52using namespace sampleprof;
53
54#define DEBUG_TYPE "samplepgo-reader"
55
56// This internal option specifies if the profile uses FS discriminators.
57// It only applies to text, and binary format profiles.
58// For ext-binary format profiles, the flag is set in the summary.
59static cl::opt<bool> ProfileIsFSDisciminator(
60 "profile-isfs", cl::Hidden, cl::init(Val: false),
61 cl::desc("Profile uses flow sensitive discriminators"));
62
63static cl::opt<bool>
64 LazyLoadNameTable("sample-profile-lazy-load-name-table", cl::init(Val: true),
65 cl::Hidden,
66 cl::desc("Lazy load the name table from the profile."));
67
68/// Dump the function profile for \p FName.
69///
70/// \param FContext Name + context of the function to print.
71/// \param OS Stream to emit the output to.
72void SampleProfileReader::dumpFunctionProfile(const FunctionSamples &FS,
73 raw_ostream &OS) {
74 OS << "Function: " << FS.getContext().toString() << ": " << FS;
75}
76
77/// Dump all the function profiles found on stream \p OS.
78void SampleProfileReader::dump(raw_ostream &OS) {
79 std::vector<NameFunctionSamples> V;
80 sortFuncProfiles(ProfileMap: Profiles, SortedProfiles&: V);
81 for (const auto &I : V)
82 dumpFunctionProfile(FS: *I.second, OS);
83}
84
85static void dumpFunctionProfileJson(const FunctionSamples &S,
86 json::OStream &JOS, bool TopLevel = false) {
87 auto DumpBody = [&](const BodySampleMap &BodySamples) {
88 for (const auto &I : BodySamples) {
89 const LineLocation &Loc = I.first;
90 const SampleRecord &Sample = I.second;
91 JOS.object(Contents: [&] {
92 JOS.attribute(Key: "line", Contents: Loc.LineOffset);
93 if (Loc.Discriminator)
94 JOS.attribute(Key: "discriminator", Contents: Loc.Discriminator);
95 JOS.attribute(Key: "samples", Contents: Sample.getSamples());
96
97 auto CallTargets = Sample.getSortedCallTargets();
98 if (!CallTargets.empty()) {
99 JOS.attributeArray(Key: "calls", Contents: [&] {
100 for (const auto &J : CallTargets) {
101 JOS.object(Contents: [&] {
102 JOS.attribute(Key: "function", Contents: J.first.str());
103 JOS.attribute(Key: "samples", Contents: J.second);
104 });
105 }
106 });
107 }
108 });
109 }
110 };
111
112 auto DumpCallsiteSamples = [&](const CallsiteSampleMap &CallsiteSamples) {
113 for (const auto &I : CallsiteSamples)
114 for (const auto &FS : I.second) {
115 const LineLocation &Loc = I.first;
116 const FunctionSamples &CalleeSamples = FS.second;
117 JOS.object(Contents: [&] {
118 JOS.attribute(Key: "line", Contents: Loc.LineOffset);
119 if (Loc.Discriminator)
120 JOS.attribute(Key: "discriminator", Contents: Loc.Discriminator);
121 JOS.attributeArray(
122 Key: "samples", Contents: [&] { dumpFunctionProfileJson(S: CalleeSamples, JOS); });
123 });
124 }
125 };
126
127 JOS.object(Contents: [&] {
128 JOS.attribute(Key: "name", Contents: S.getFunction().str());
129 JOS.attribute(Key: "total", Contents: S.getTotalSamples());
130 if (TopLevel)
131 JOS.attribute(Key: "head", Contents: S.getHeadSamples());
132
133 const auto &BodySamples = S.getBodySamples();
134 if (!BodySamples.empty())
135 JOS.attributeArray(Key: "body", Contents: [&] { DumpBody(BodySamples); });
136
137 const auto &CallsiteSamples = S.getCallsiteSamples();
138 if (!CallsiteSamples.empty())
139 JOS.attributeArray(Key: "callsites",
140 Contents: [&] { DumpCallsiteSamples(CallsiteSamples); });
141 });
142}
143
144/// Dump all the function profiles found on stream \p OS in the JSON format.
145void SampleProfileReader::dumpJson(raw_ostream &OS) {
146 std::vector<NameFunctionSamples> V;
147 sortFuncProfiles(ProfileMap: Profiles, SortedProfiles&: V);
148 json::OStream JOS(OS, 2);
149 JOS.arrayBegin();
150 for (const auto &F : V)
151 dumpFunctionProfileJson(S: *F.second, JOS, TopLevel: true);
152 JOS.arrayEnd();
153
154 // Emit a newline character at the end as json::OStream doesn't emit one.
155 OS << "\n";
156}
157
158/// Parse \p Input as function head.
159///
160/// Parse one line of \p Input, and update function name in \p FName,
161/// function's total sample count in \p NumSamples, function's entry
162/// count in \p NumHeadSamples.
163///
164/// \returns true if parsing is successful.
165static bool ParseHead(const StringRef &Input, StringRef &FName,
166 uint64_t &NumSamples, uint64_t &NumHeadSamples) {
167 if (Input[0] == ' ')
168 return false;
169 size_t n2 = Input.rfind(C: ':');
170 size_t n1 = Input.rfind(C: ':', From: n2 - 1);
171 FName = Input.substr(Start: 0, N: n1);
172 if (Input.substr(Start: n1 + 1, N: n2 - n1 - 1).getAsInteger(Radix: 10, Result&: NumSamples))
173 return false;
174 if (Input.substr(Start: n2 + 1).getAsInteger(Radix: 10, Result&: NumHeadSamples))
175 return false;
176 return true;
177}
178
179/// Returns true if line offset \p L is legal (only has 16 bits).
180static bool isOffsetLegal(unsigned L) { return (L & 0xffff) == L; }
181
182/// Parse \p Input that contains metadata.
183/// Possible metadata:
184/// - CFG Checksum information:
185/// !CFGChecksum: 12345
186/// - CFG Checksum information:
187/// !Attributes: 1
188/// Stores the FunctionHash (a.k.a. CFG Checksum) into \p FunctionHash.
189static bool parseMetadata(const StringRef &Input, uint64_t &FunctionHash,
190 uint32_t &Attributes) {
191 if (Input.starts_with(Prefix: "!CFGChecksum:")) {
192 StringRef CFGInfo = Input.substr(Start: strlen(s: "!CFGChecksum:")).trim();
193 return !CFGInfo.getAsInteger(Radix: 10, Result&: FunctionHash);
194 }
195
196 if (Input.starts_with(Prefix: "!Attributes:")) {
197 StringRef Attrib = Input.substr(Start: strlen(s: "!Attributes:")).trim();
198 return !Attrib.getAsInteger(Radix: 10, Result&: Attributes);
199 }
200
201 return false;
202}
203
204enum class LineType {
205 CallSiteProfile,
206 BodyProfile,
207 Metadata,
208 VirtualCallTypeProfile,
209};
210
211// Parse `Input` as a white-space separated list of `vtable:count` pairs. An
212// example input line is `_ZTVbar:1471 _ZTVfoo:630`.
213static bool parseTypeCountMap(StringRef Input,
214 DenseMap<StringRef, uint64_t> &TypeCountMap) {
215 for (size_t Index = Input.find_first_not_of(C: ' '); Index != StringRef::npos;) {
216 size_t ColonIndex = Input.find(C: ':', From: Index);
217 if (ColonIndex == StringRef::npos)
218 return false; // No colon found, invalid format.
219 StringRef TypeName = Input.substr(Start: Index, N: ColonIndex - Index);
220 // CountIndex is the start index of count.
221 size_t CountStartIndex = ColonIndex + 1;
222 // NextIndex is the start index after the 'target:count' pair.
223 size_t NextIndex = Input.find_first_of(C: ' ', From: CountStartIndex);
224 uint64_t Count;
225 if (Input.substr(Start: CountStartIndex, N: NextIndex - CountStartIndex)
226 .getAsInteger(Radix: 10, Result&: Count))
227 return false; // Invalid count.
228 // Error on duplicated type names in one line of input.
229 auto [Iter, Inserted] = TypeCountMap.insert(KV: {TypeName, Count});
230 if (!Inserted)
231 return false;
232 Index = (NextIndex == StringRef::npos)
233 ? StringRef::npos
234 : Input.find_first_not_of(C: ' ', From: NextIndex);
235 }
236 return true;
237}
238
239/// Parse \p Input as line sample.
240///
241/// \param Input input line.
242/// \param LineTy Type of this line.
243/// \param Depth the depth of the inline stack.
244/// \param NumSamples total samples of the line/inlined callsite.
245/// \param LineOffset line offset to the start of the function.
246/// \param Discriminator discriminator of the line.
247/// \param TargetCountMap map from indirect call target to count.
248/// \param FunctionHash the function's CFG hash, used by pseudo probe.
249///
250/// returns true if parsing is successful.
251static bool ParseLine(const StringRef &Input, LineType &LineTy, uint32_t &Depth,
252 uint64_t &NumSamples, uint32_t &LineOffset,
253 uint32_t &Discriminator, StringRef &CalleeName,
254 DenseMap<StringRef, uint64_t> &TargetCountMap,
255 DenseMap<StringRef, uint64_t> &TypeCountMap,
256 uint64_t &FunctionHash, uint32_t &Attributes,
257 bool &IsFlat) {
258 for (Depth = 0; Input[Depth] == ' '; Depth++)
259 ;
260 if (Depth == 0)
261 return false;
262
263 if (Input[Depth] == '!') {
264 LineTy = LineType::Metadata;
265 // This metadata is only for manual inspection only. We already created a
266 // FunctionSamples and put it in the profile map, so there is no point
267 // to skip profiles even they have no use for ThinLTO.
268 if (Input == StringRef(" !Flat")) {
269 IsFlat = true;
270 return true;
271 }
272 return parseMetadata(Input: Input.substr(Start: Depth), FunctionHash, Attributes);
273 }
274
275 size_t n1 = Input.find(C: ':');
276 StringRef Loc = Input.substr(Start: Depth, N: n1 - Depth);
277 size_t n2 = Loc.find(C: '.');
278 if (n2 == StringRef::npos) {
279 if (Loc.getAsInteger(Radix: 10, Result&: LineOffset) || !isOffsetLegal(L: LineOffset))
280 return false;
281 Discriminator = 0;
282 } else {
283 if (Loc.substr(Start: 0, N: n2).getAsInteger(Radix: 10, Result&: LineOffset))
284 return false;
285 if (Loc.substr(Start: n2 + 1).getAsInteger(Radix: 10, Result&: Discriminator))
286 return false;
287 }
288
289 StringRef Rest = Input.substr(Start: n1 + 2);
290 if (isDigit(C: Rest[0])) {
291 LineTy = LineType::BodyProfile;
292 size_t n3 = Rest.find(C: ' ');
293 if (n3 == StringRef::npos) {
294 if (Rest.getAsInteger(Radix: 10, Result&: NumSamples))
295 return false;
296 } else {
297 if (Rest.substr(Start: 0, N: n3).getAsInteger(Radix: 10, Result&: NumSamples))
298 return false;
299 }
300 // Find call targets and their sample counts.
301 // Note: In some cases, there are symbols in the profile which are not
302 // mangled. To accommodate such cases, use colon + integer pairs as the
303 // anchor points.
304 // An example:
305 // _M_construct<char *>:1000 string_view<std::allocator<char> >:437
306 // ":1000" and ":437" are used as anchor points so the string above will
307 // be interpreted as
308 // target: _M_construct<char *>
309 // count: 1000
310 // target: string_view<std::allocator<char> >
311 // count: 437
312 while (n3 != StringRef::npos) {
313 n3 += Rest.substr(Start: n3).find_first_not_of(C: ' ');
314 Rest = Rest.substr(Start: n3);
315 n3 = Rest.find_first_of(C: ':');
316 if (n3 == StringRef::npos || n3 == 0)
317 return false;
318
319 StringRef Target;
320 uint64_t count, n4;
321 while (true) {
322 // Get the segment after the current colon.
323 StringRef AfterColon = Rest.substr(Start: n3 + 1);
324 // Get the target symbol before the current colon.
325 Target = Rest.substr(Start: 0, N: n3);
326 // Check if the word after the current colon is an integer.
327 n4 = AfterColon.find_first_of(C: ' ');
328 n4 = (n4 != StringRef::npos) ? n3 + n4 + 1 : Rest.size();
329 StringRef WordAfterColon = Rest.substr(Start: n3 + 1, N: n4 - n3 - 1);
330 if (!WordAfterColon.getAsInteger(Radix: 10, Result&: count))
331 break;
332
333 // Try to find the next colon.
334 uint64_t n5 = AfterColon.find_first_of(C: ':');
335 if (n5 == StringRef::npos)
336 return false;
337 n3 += n5 + 1;
338 }
339
340 // An anchor point is found. Save the {target, count} pair
341 TargetCountMap[Target] = count;
342 if (n4 == Rest.size())
343 break;
344 // Change n3 to the next blank space after colon + integer pair.
345 n3 = n4;
346 }
347 } else if (Rest.starts_with(Prefix: kVTableProfPrefix)) {
348 LineTy = LineType::VirtualCallTypeProfile;
349 return parseTypeCountMap(Input: Rest.substr(Start: strlen(s: kVTableProfPrefix)),
350 TypeCountMap);
351 } else {
352 LineTy = LineType::CallSiteProfile;
353 size_t n3 = Rest.find_last_of(C: ':');
354 CalleeName = Rest.substr(Start: 0, N: n3);
355 if (Rest.substr(Start: n3 + 1).getAsInteger(Radix: 10, Result&: NumSamples))
356 return false;
357 }
358 return true;
359}
360
361/// Load samples from a text file.
362///
363/// See the documentation at the top of the file for an explanation of
364/// the expected format.
365///
366/// \returns true if the file was loaded successfully, false otherwise.
367std::error_code SampleProfileReaderText::readImpl() {
368 line_iterator LineIt(*Buffer, /*SkipBlanks=*/true, '#');
369 sampleprof_error Result = sampleprof_error::success;
370
371 InlineCallStack InlineStack;
372 uint32_t TopLevelProbeProfileCount = 0;
373
374 // DepthMetadata tracks whether we have processed metadata for the current
375 // top-level or nested function profile.
376 uint32_t DepthMetadata = 0;
377
378 std::vector<SampleContext *> FlatSamples;
379
380 ProfileIsFS = ProfileIsFSDisciminator;
381 FunctionSamples::ProfileIsFS = ProfileIsFS;
382 for (; !LineIt.is_at_eof(); ++LineIt) {
383 size_t pos = LineIt->find_first_not_of(C: ' ');
384 if (pos == LineIt->npos || (*LineIt)[pos] == '#')
385 continue;
386 // Read the header of each function.
387 //
388 // Note that for function identifiers we are actually expecting
389 // mangled names, but we may not always get them. This happens when
390 // the compiler decides not to emit the function (e.g., it was inlined
391 // and removed). In this case, the binary will not have the linkage
392 // name for the function, so the profiler will emit the function's
393 // unmangled name, which may contain characters like ':' and '>' in its
394 // name (member functions, templates, etc).
395 //
396 // The only requirement we place on the identifier, then, is that it
397 // should not begin with a number.
398 if ((*LineIt)[0] != ' ') {
399 uint64_t NumSamples, NumHeadSamples;
400 StringRef FName;
401 if (!ParseHead(Input: *LineIt, FName, NumSamples, NumHeadSamples)) {
402 reportError(LineNumber: LineIt.line_number(),
403 Msg: "Expected 'mangled_name:NUM:NUM', found " + *LineIt);
404 return sampleprof_error::malformed;
405 }
406 DepthMetadata = 0;
407 SampleContext FContext(FName, CSNameTable);
408 if (FContext.hasContext())
409 ++CSProfileCount;
410 FunctionSamples &FProfile = Profiles.create(Ctx: FContext);
411 mergeSampleProfErrors(Accumulator&: Result, Result: FProfile.addTotalSamples(Num: NumSamples));
412 mergeSampleProfErrors(Accumulator&: Result, Result: FProfile.addHeadSamples(Num: NumHeadSamples));
413 InlineStack.clear();
414 InlineStack.push_back(Elt: &FProfile);
415 } else {
416 uint64_t NumSamples;
417 StringRef FName;
418 DenseMap<StringRef, uint64_t> TargetCountMap;
419 DenseMap<StringRef, uint64_t> TypeCountMap;
420 uint32_t Depth, LineOffset, Discriminator;
421 LineType LineTy = LineType::BodyProfile;
422 uint64_t FunctionHash = 0;
423 uint32_t Attributes = 0;
424 bool IsFlat = false;
425 // TODO: Update ParseLine to return an error code instead of a bool and
426 // report it.
427 if (!ParseLine(Input: *LineIt, LineTy, Depth, NumSamples, LineOffset,
428 Discriminator, CalleeName&: FName, TargetCountMap, TypeCountMap,
429 FunctionHash, Attributes, IsFlat)) {
430 switch (LineTy) {
431 case LineType::Metadata:
432 reportError(LineNumber: LineIt.line_number(),
433 Msg: "Cannot parse metadata: " + *LineIt);
434 break;
435 case LineType::VirtualCallTypeProfile:
436 reportError(LineNumber: LineIt.line_number(),
437 Msg: "Expected 'vtables [mangled_vtable:NUM]+', found " +
438 *LineIt);
439 break;
440 default:
441 reportError(LineNumber: LineIt.line_number(),
442 Msg: "Expected 'NUM[.NUM]: NUM[ mangled_name:NUM]*', found " +
443 *LineIt);
444 }
445 return sampleprof_error::malformed;
446 }
447 if (LineTy != LineType::Metadata && Depth == DepthMetadata) {
448 // Metadata must be put at the end of a function profile.
449 reportError(LineNumber: LineIt.line_number(),
450 Msg: "Found non-metadata after metadata: " + *LineIt);
451 return sampleprof_error::malformed;
452 }
453
454 // Here we handle FS discriminators.
455 Discriminator &= getDiscriminatorMask();
456
457 while (InlineStack.size() > Depth) {
458 InlineStack.pop_back();
459 }
460 switch (LineTy) {
461 case LineType::CallSiteProfile: {
462 FunctionSamples &FSamples = InlineStack.back()->functionSamplesAt(
463 Loc: LineLocation(LineOffset, Discriminator))[FunctionId(FName)];
464 FSamples.setFunction(FunctionId(FName));
465 mergeSampleProfErrors(Accumulator&: Result, Result: FSamples.addTotalSamples(Num: NumSamples));
466 InlineStack.push_back(Elt: &FSamples);
467 DepthMetadata = 0;
468 break;
469 }
470
471 case LineType::VirtualCallTypeProfile: {
472 mergeSampleProfErrors(
473 Accumulator&: Result, Result: InlineStack.back()->addCallsiteVTableTypeProfAt(
474 Loc: LineLocation(LineOffset, Discriminator), Other: TypeCountMap));
475 break;
476 }
477
478 case LineType::BodyProfile: {
479 FunctionSamples &FProfile = *InlineStack.back();
480 for (const auto &name_count : TargetCountMap) {
481 mergeSampleProfErrors(Accumulator&: Result, Result: FProfile.addCalledTargetSamples(
482 LineOffset, Discriminator,
483 Func: FunctionId(name_count.first),
484 Num: name_count.second));
485 }
486 mergeSampleProfErrors(
487 Accumulator&: Result,
488 Result: FProfile.addBodySamples(LineOffset, Discriminator, Num: NumSamples));
489 break;
490 }
491 case LineType::Metadata: {
492 FunctionSamples &FProfile = *InlineStack.back();
493 if (FunctionHash) {
494 FProfile.setFunctionHash(FunctionHash);
495 if (Depth == 1)
496 ++TopLevelProbeProfileCount;
497 }
498 FProfile.getContext().setAllAttributes(Attributes);
499 if (Attributes & (uint32_t)ContextShouldBeInlined)
500 ProfileIsPreInlined = true;
501 DepthMetadata = Depth;
502 if (IsFlat) {
503 if (Depth == 1)
504 FlatSamples.push_back(x: &FProfile.getContext());
505 else
506 Ctx.diagnose(DI: DiagnosticInfoSampleProfile(
507 Buffer->getBufferIdentifier(), LineIt.line_number(),
508 "!Flat may only be used at top level function.", DS_Warning));
509 }
510 break;
511 }
512 }
513 }
514 }
515
516 // Honor the option to skip flat functions. Since they are already added to
517 // the profile map, remove them all here.
518 if (SkipFlatProf)
519 for (SampleContext *FlatSample : FlatSamples)
520 Profiles.erase(Ctx: *FlatSample);
521
522 assert((CSProfileCount == 0 || CSProfileCount == Profiles.size()) &&
523 "Cannot have both context-sensitive and regular profile");
524 ProfileIsCS = (CSProfileCount > 0);
525 assert((TopLevelProbeProfileCount == 0 ||
526 TopLevelProbeProfileCount == Profiles.size()) &&
527 "Cannot have both probe-based profiles and regular profiles");
528 ProfileIsProbeBased = (TopLevelProbeProfileCount > 0);
529 FunctionSamples::ProfileIsProbeBased = ProfileIsProbeBased;
530 FunctionSamples::ProfileIsCS = ProfileIsCS;
531 FunctionSamples::ProfileIsPreInlined = ProfileIsPreInlined;
532
533 if (Result == sampleprof_error::success)
534 computeSummary();
535
536 return Result;
537}
538
539bool SampleProfileReaderText::hasFormat(const MemoryBuffer &Buffer) {
540 bool result = false;
541
542 // Check that the first non-comment line is a valid function header.
543 line_iterator LineIt(Buffer, /*SkipBlanks=*/true, '#');
544 if (!LineIt.is_at_eof()) {
545 if ((*LineIt)[0] != ' ') {
546 uint64_t NumSamples, NumHeadSamples;
547 StringRef FName;
548 result = ParseHead(Input: *LineIt, FName, NumSamples, NumHeadSamples);
549 }
550 }
551
552 return result;
553}
554
555/// Emit a reader diagnostic for \p ProfError and return its error code.
556static std::error_code diagnoseReaderError(const SampleProfileReader &Reader,
557 sampleprof_error ProfError) {
558 std::error_code EC = ProfError;
559 Reader.reportError(LineNumber: 0, Msg: EC.message());
560 return EC;
561}
562
563template <typename T> ErrorOr<T> SampleProfileReaderBinary::readNumber() {
564 if (Data >= End)
565 return diagnoseReaderError(Reader: *this, ProfError: sampleprof_error::truncated);
566
567 unsigned NumBytesRead = 0;
568 ULEB128DecodeError DecodeError = ULEB128DecodeError::None;
569 uint64_t Val = decodeULEB128(p: Data, n: &NumBytesRead, end: End, error: nullptr, errorCode: &DecodeError);
570
571 // Preserve the distinction between incomplete input and an invalid value.
572 switch (DecodeError) {
573 case ULEB128DecodeError::None:
574 break;
575 case ULEB128DecodeError::UnexpectedEnd:
576 return diagnoseReaderError(Reader: *this, ProfError: sampleprof_error::truncated);
577 case ULEB128DecodeError::TooBig:
578 return diagnoseReaderError(Reader: *this, ProfError: sampleprof_error::malformed);
579 }
580
581 if (Val > std::numeric_limits<T>::max())
582 return diagnoseReaderError(Reader: *this, ProfError: sampleprof_error::malformed);
583
584 Data += NumBytesRead;
585 return static_cast<T>(Val);
586}
587
588ErrorOr<StringRef> SampleProfileReaderBinary::readString() {
589 if (Data >= End)
590 return diagnoseReaderError(Reader: *this, ProfError: sampleprof_error::truncated);
591
592 const auto *Terminator = static_cast<const uint8_t *>(
593 std::memchr(s: Data, c: 0, n: static_cast<size_t>(End - Data)));
594 if (!Terminator)
595 return diagnoseReaderError(Reader: *this, ProfError: sampleprof_error::truncated);
596
597 StringRef Str(reinterpret_cast<const char *>(Data), Terminator - Data);
598 Data = Terminator + 1;
599 return Str;
600}
601
602template <typename T>
603ErrorOr<T> SampleProfileReaderBinary::readUnencodedNumber() {
604 if (Data > End || static_cast<size_t>(End - Data) < sizeof(T))
605 return diagnoseReaderError(Reader: *this, ProfError: sampleprof_error::truncated);
606
607 using namespace support;
608 T Val = endian::readNext<T, llvm::endianness::little>(Data);
609 return Val;
610}
611
612template <typename T>
613inline ErrorOr<size_t> SampleProfileReaderBinary::readStringIndex(T &Table) {
614 auto Idx = readNumber<size_t>();
615 if (std::error_code EC = Idx.getError())
616 return EC;
617 if (*Idx >= Table.size())
618 return sampleprof_error::truncated_name_table;
619 return *Idx;
620}
621
622ErrorOr<FunctionId>
623SampleProfileReaderBinary::readStringFromTable(size_t *RetIdx) {
624 if (!NameTable)
625 return sampleprof_error::truncated_name_table;
626 auto Idx = readStringIndex(Table&: *NameTable);
627 if (std::error_code EC = Idx.getError())
628 return EC;
629 if (RetIdx)
630 *RetIdx = *Idx;
631 return (*NameTable)[*Idx];
632}
633
634ErrorOr<SampleContextFrames>
635SampleProfileReaderBinary::readContextFromTable(size_t *RetIdx) {
636 auto ContextIdx = readNumber<size_t>();
637 if (std::error_code EC = ContextIdx.getError())
638 return EC;
639 if (*ContextIdx >= CSNameTable.size())
640 return sampleprof_error::truncated_name_table;
641 if (RetIdx)
642 *RetIdx = *ContextIdx;
643 return CSNameTable[*ContextIdx];
644}
645
646ErrorOr<std::pair<SampleContext, uint64_t>>
647SampleProfileReaderBinary::readSampleContextFromTable() {
648 SampleContext Context;
649 size_t Idx;
650 if (ProfileIsCS) {
651 auto FContext(readContextFromTable(RetIdx: &Idx));
652 if (std::error_code EC = FContext.getError())
653 return EC;
654 Context = SampleContext(*FContext);
655 } else {
656 auto FName(readStringFromTable(RetIdx: &Idx));
657 if (std::error_code EC = FName.getError())
658 return EC;
659 Context = SampleContext(*FName);
660 }
661 // Since MD5SampleContextStart may point to the profile's file data, need to
662 // make sure it is reading the same value on big endian CPU.
663 uint64_t Hash = support::endian::read64le(P: MD5SampleContextStart + Idx);
664 // Lazy computing of hash value, write back to the table to cache it. Only
665 // compute the context's hash value if it is being referenced for the first
666 // time.
667 if (Hash == 0) {
668 assert(MD5SampleContextStart == MD5SampleContextTable.data());
669 Hash = Context.getHashCode();
670 support::endian::write64le(P: &MD5SampleContextTable[Idx], V: Hash);
671 }
672 return std::make_pair(x&: Context, y&: Hash);
673}
674
675std::error_code
676SampleProfileReaderBinary::readVTableTypeCountMap(TypeCountMap &M) {
677 auto NumVTableTypes = readNumber<uint32_t>();
678 if (std::error_code EC = NumVTableTypes.getError())
679 return EC;
680 M.reserve(Cap: *NumVTableTypes);
681
682 for (uint32_t I = 0; I < *NumVTableTypes; ++I) {
683 auto VTableType(readStringFromTable());
684 if (std::error_code EC = VTableType.getError())
685 return EC;
686
687 auto VTableSamples = readNumber<uint64_t>();
688 if (std::error_code EC = VTableSamples.getError())
689 return EC;
690 // The source profile should not have duplicate vtable records at the same
691 // location. In case duplicate vtables are found, reader can emit a warning
692 // but continue processing the profile.
693 if (!M.insert(KV: std::make_pair(x&: *VTableType, y&: *VTableSamples)).second) {
694 Ctx.diagnose(DI: DiagnosticInfoSampleProfile(
695 Buffer->getBufferIdentifier(), 0,
696 "Duplicate vtable type " + VTableType->str() +
697 " at the same location. Additional counters will be ignored.",
698 DS_Warning));
699 continue;
700 }
701 }
702 return sampleprof_error::success;
703}
704
705std::error_code
706SampleProfileReaderBinary::readCallsiteVTableProf(FunctionSamples &FProfile) {
707 assert(ReadVTableProf &&
708 "Cannot read vtable profiles if ReadVTableProf is false");
709
710 // Read the vtable type profile for the callsite.
711 auto NumCallsites = readNumber<uint32_t>();
712 if (std::error_code EC = NumCallsites.getError())
713 return EC;
714 FProfile.reserveCallsiteTypeCounts(NumEntries: *NumCallsites);
715
716 for (uint32_t I = 0; I < *NumCallsites; ++I) {
717 auto LineOffset = readNumber<uint64_t>();
718 if (std::error_code EC = LineOffset.getError())
719 return EC;
720
721 if (!isOffsetLegal(L: *LineOffset))
722 return sampleprof_error::illegal_line_offset;
723
724 auto Discriminator = readNumber<uint64_t>();
725 if (std::error_code EC = Discriminator.getError())
726 return EC;
727
728 // Here we handle FS discriminators:
729 const uint32_t DiscriminatorVal = (*Discriminator) & getDiscriminatorMask();
730
731 if (std::error_code EC = readVTableTypeCountMap(M&: FProfile.getTypeSamplesAt(
732 Loc: LineLocation(*LineOffset, DiscriminatorVal))))
733 return EC;
734 }
735 return sampleprof_error::success;
736}
737
738std::error_code
739SampleProfileReaderBinary::readLBRProfile(FunctionSamples &FProfile,
740 bool IsNested) {
741 if (ProfileSecRange.IsComposite && !IsNested) {
742 auto NumHeadSamples = readNumber<uint64_t>();
743 if (std::error_code EC = NumHeadSamples.getError())
744 return EC;
745 FProfile.addHeadSamples(Num: *NumHeadSamples);
746 }
747 auto NumSamples = readNumber<uint64_t>();
748 if (std::error_code EC = NumSamples.getError())
749 return EC;
750 FProfile.addTotalSamples(Num: *NumSamples);
751
752 // Read the samples in the body.
753 auto NumRecords = readNumber<uint32_t>();
754 if (std::error_code EC = NumRecords.getError())
755 return EC;
756 FProfile.reserveBodySamples(NumEntries: *NumRecords);
757
758 for (uint32_t I = 0; I < *NumRecords; ++I) {
759 auto LineOffset = readNumber<uint64_t>();
760 if (std::error_code EC = LineOffset.getError())
761 return EC;
762
763 if (!isOffsetLegal(L: *LineOffset)) {
764 return sampleprof_error::illegal_line_offset;
765 }
766
767 auto Discriminator = readNumber<uint64_t>();
768 if (std::error_code EC = Discriminator.getError())
769 return EC;
770
771 auto NumSamples = readNumber<uint64_t>();
772 if (std::error_code EC = NumSamples.getError())
773 return EC;
774
775 auto NumCalls = readNumber<uint32_t>();
776 if (std::error_code EC = NumCalls.getError())
777 return EC;
778
779 // Here we handle FS discriminators:
780 uint32_t DiscriminatorVal = (*Discriminator) & getDiscriminatorMask();
781
782 for (uint32_t J = 0; J < *NumCalls; ++J) {
783 auto CalledFunction(readStringFromTable());
784 if (std::error_code EC = CalledFunction.getError())
785 return EC;
786
787 auto CalledFunctionSamples = readNumber<uint64_t>();
788 if (std::error_code EC = CalledFunctionSamples.getError())
789 return EC;
790
791 FProfile.addCalledTargetSamples(LineOffset: *LineOffset, Discriminator: DiscriminatorVal,
792 Func: *CalledFunction, Num: *CalledFunctionSamples);
793 }
794
795 FProfile.addBodySamples(LineOffset: *LineOffset, Discriminator: DiscriminatorVal, Num: *NumSamples);
796 }
797
798 return sampleprof_error::success;
799}
800
801std::error_code
802SampleProfileReaderBinary::readCompositeProfile(FunctionSamples &FProfile,
803 bool IsNested) {
804 // Read the number of profile types.
805 auto ProfNum = readNumber<uint64_t>();
806 if (std::error_code EC = ProfNum.getError())
807 return EC;
808 if (ProfileTypeInfoOS)
809 *ProfileTypeInfoOS << (IsNested ? "Nested function: " : "Function: ")
810 << FProfile.getContext().toString()
811 << "\n Profile blocks: " << *ProfNum << "\n";
812
813 // Each type identifies one logical payload for the function. Decoding the
814 // same type twice would merge absolute counters from malformed input.
815 SmallSet<uint64_t, 4> SeenTypes;
816
817 // Read the specified number of composite profiles.
818 for (uint64_t I = 0; I < *ProfNum; ++I) {
819 auto Type = readNumber<uint64_t>();
820 if (std::error_code EC = Type.getError())
821 return EC;
822 // Report the conflicting ID so malformed profiles can be diagnosed
823 // without inspecting their binary encoding.
824 if (!SeenTypes.insert(V: *Type).second) {
825 reportError(LineNumber: 0, Msg: "Duplicate profile type ID: " + Twine(*Type));
826 return sampleprof_error::malformed;
827 }
828 auto Size = readNumber<uint64_t>();
829 if (std::error_code EC = Size.getError())
830 return EC;
831 if (ProfileTypeInfoOS)
832 *ProfileTypeInfoOS << " Type: " << *Type << " ("
833 << getProfTypeName(Type: *Type)
834 << "), Payload size: " << *Size << "\n";
835 const uint64_t RemainingSize = End - Data;
836 // Diagnose a size that would let the payload cross its containing section.
837 if (*Size > RemainingSize) {
838 reportError(LineNumber: 0, Msg: "Profile type ID " + Twine(*Type) +
839 " declares payload size " + Twine(*Size) +
840 ", but only " + Twine(RemainingSize) +
841 " bytes remain");
842 return sampleprof_error::truncated;
843 }
844
845 const uint8_t *PayloadEnd = Data + *Size;
846 std::error_code EC = sampleprof_error::success;
847 // Restrict field readers to the current payload so they reject fields that
848 // extend into the following payload.
849 SaveAndRestore<const uint8_t *> RestoreEnd(End, PayloadEnd);
850 switch (*Type) {
851 case ProfTypeLBR:
852 EC = readLBRProfile(FProfile, IsNested);
853 break;
854 default:
855 // Skip unknown profile types for forward compatibility.
856 HasUnknownProfileTypes = true;
857 Data = PayloadEnd;
858 break;
859 }
860
861 if (EC)
862 return EC;
863 // Reject trailing bytes because every known decoder must consume exactly
864 // the payload declared for its type.
865 if (Data != PayloadEnd) {
866 reportError(LineNumber: 0,
867 Msg: "Profile type ID " + Twine(*Type) +
868 " did not consume its complete payload; unread bytes: " +
869 Twine(PayloadEnd - Data));
870 return sampleprof_error::malformed;
871 }
872 }
873
874 return sampleprof_error::success;
875}
876
877std::error_code
878SampleProfileReaderBinary::readProfile(FunctionSamples &FProfile,
879 bool IsNested) {
880 if (ProfileSecRange.IsComposite) {
881 if (std::error_code EC = readCompositeProfile(FProfile, IsNested))
882 return EC;
883 } else {
884 if (std::error_code EC = readLBRProfile(FProfile, IsNested))
885 return EC;
886 }
887
888 // Read all the samples for inlined function calls.
889 auto NumCallsites = readNumber<uint32_t>();
890 if (std::error_code EC = NumCallsites.getError())
891 return EC;
892
893 for (uint32_t J = 0; J < *NumCallsites; ++J) {
894 auto LineOffset = readNumber<uint64_t>();
895 if (std::error_code EC = LineOffset.getError())
896 return EC;
897
898 auto Discriminator = readNumber<uint64_t>();
899 if (std::error_code EC = Discriminator.getError())
900 return EC;
901
902 auto FName(readStringFromTable());
903 if (std::error_code EC = FName.getError())
904 return EC;
905
906 // Here we handle FS discriminators:
907 uint32_t DiscriminatorVal = (*Discriminator) & getDiscriminatorMask();
908
909 FunctionSamples &CalleeProfile = FProfile.functionSamplesAt(
910 Loc: LineLocation(*LineOffset, DiscriminatorVal))[*FName];
911 CalleeProfile.setFunction(*FName);
912 if (std::error_code EC = readProfile(FProfile&: CalleeProfile, /*IsNested=*/true))
913 return EC;
914 }
915
916 if (ReadVTableProf)
917 return readCallsiteVTableProf(FProfile);
918
919 return sampleprof_error::success;
920}
921
922std::error_code
923SampleProfileReaderBinary::readFuncProfile(const uint8_t *Start,
924 SampleProfileMap &Profiles) {
925 Data = Start;
926 ErrorOr<uint64_t> NumHeadSamples = 0;
927 if (!ProfileSecRange.IsComposite) {
928 NumHeadSamples = readNumber<uint64_t>();
929 if (std::error_code EC = NumHeadSamples.getError())
930 return EC;
931 }
932 auto FContextHash(readSampleContextFromTable());
933 if (std::error_code EC = FContextHash.getError())
934 return EC;
935
936 auto &[FContext, Hash] = *FContextHash;
937 // Use the cached hash value for insertion instead of recalculating it.
938 auto Res = Profiles.try_emplace(Hash, Key: FContext, Args: FunctionSamples());
939 FunctionSamples &FProfile = Res.first->second;
940 FProfile.setContext(FContext);
941 if (!ProfileSecRange.IsComposite)
942 FProfile.addHeadSamples(Num: *NumHeadSamples);
943
944 if (FContext.hasContext())
945 CSProfileCount++;
946
947 if (std::error_code EC = readProfile(FProfile, /*IsNested=*/false))
948 return EC;
949 return sampleprof_error::success;
950}
951
952std::error_code
953SampleProfileReaderBinary::readFuncProfile(const uint8_t *Start) {
954 return readFuncProfile(Start, Profiles);
955}
956
957std::error_code SampleProfileReaderBinary::readImpl() {
958 ProfileIsFS = ProfileIsFSDisciminator;
959 FunctionSamples::ProfileIsFS = ProfileIsFS;
960 while (Data < End) {
961 if (std::error_code EC = readFuncProfile(Start: Data))
962 return EC;
963 }
964
965 return sampleprof_error::success;
966}
967
968std::error_code SampleProfileReaderExtBinaryBase::readOneSection(
969 const uint8_t *Start, uint64_t Size, const SecHdrTableEntry &Entry) {
970 Data = Start;
971 End = Start + Size;
972 switch (Entry.Type) {
973 case SecProfSummary:
974 if (std::error_code EC = readSummary())
975 return EC;
976 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagPartial))
977 Summary->setPartialProfile(true);
978 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagFullContext))
979 FunctionSamples::ProfileIsCS = ProfileIsCS = true;
980 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagIsPreInlined))
981 FunctionSamples::ProfileIsPreInlined = ProfileIsPreInlined = true;
982 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagFSDiscriminator))
983 FunctionSamples::ProfileIsFS = ProfileIsFS = true;
984 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagHasVTableTypeProf))
985 ReadVTableProf = true;
986 break;
987 case SecNameTable: {
988 bool FixedLengthMD5 =
989 hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagFixedLengthMD5);
990 bool UseMD5 = hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagMD5Name);
991 // UseMD5 means if THIS section uses MD5, ProfileIsMD5 means if the entire
992 // profile uses MD5 for function name matching in IPO passes.
993 ProfileIsMD5 = ProfileIsMD5 || UseMD5;
994 FunctionSamples::HasUniqSuffix =
995 hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagUniqSuffix);
996 bool IsEytzinger = hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagEytzinger);
997 if (std::error_code EC =
998 readNameTableSec(IsMD5: UseMD5, FixedLengthMD5, IsEytzinger))
999 return EC;
1000 break;
1001 }
1002 case SecCSNameTable: {
1003 if (std::error_code EC = readCSNameTableSec())
1004 return EC;
1005 break;
1006 }
1007 case SecLBRProfile:
1008 case SecCompositeProfile:
1009 // Retain the section and its encoding for subsequent on-demand reads.
1010 ProfileSecRange = {.Start: Data, .End: End, .IsComposite: Entry.Type == SecCompositeProfile};
1011 if (std::error_code EC = readFuncProfiles())
1012 return EC;
1013 break;
1014 case SecFuncOffsetTable:
1015 case SecCompositeFuncOffsetTable:
1016 // If module is absent, we are using LLVM tools, and need to read all
1017 // profiles, so skip reading the function offset table.
1018 if (!M) {
1019 Data = End;
1020 } else {
1021 bool IsEytzinger =
1022 hasSecFlag(Entry, Flag: SecFuncOffsetFlags::SecFlagEytzinger);
1023 bool IsFlat = hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagFlat);
1024 // An unflagged function offset table inherently indexes the primary
1025 // Nested symbol span.
1026 bool IsNested = !IsFlat;
1027 assert((!ProfileIsCS ||
1028 hasSecFlag(Entry, SecFuncOffsetFlags::SecFlagOrdered) ||
1029 IsEytzinger) &&
1030 "func offset table should always be sorted or in Eytzinger BFS "
1031 "order in CS profile");
1032 if (std::error_code EC = readFuncOffsetTable(IsEytzinger, IsNested))
1033 return EC;
1034 }
1035 break;
1036 case SecFuncMetadata: {
1037 ProfileIsProbeBased =
1038 hasSecFlag(Entry, Flag: SecFuncMetadataFlags::SecFlagIsProbeBased);
1039 FunctionSamples::ProfileIsProbeBased = ProfileIsProbeBased;
1040 ProfileHasAttribute =
1041 hasSecFlag(Entry, Flag: SecFuncMetadataFlags::SecFlagHasAttribute);
1042 if (std::error_code EC = readFuncMetadata())
1043 return EC;
1044 break;
1045 }
1046 case SecProfileSymbolList:
1047 if (std::error_code EC = readProfileSymbolList(
1048 IsMD5: hasSecFlag(Entry, Flag: SecProfileSymbolListFlags::SecFlagMD5)))
1049 return EC;
1050 break;
1051 default:
1052 if (std::error_code EC = readCustomSection(Entry))
1053 return EC;
1054 break;
1055 }
1056 return sampleprof_error::success;
1057}
1058
1059bool SampleProfileReaderExtBinaryBase::useFuncOffsetList() const {
1060 // If profile is CS, the function offset section is expected to consist of
1061 // sequences of contexts in pre-order layout
1062 // (e.g. [A, A:1 @ B, A:1 @ B:2.3 @ C] [D, D:1 @ E]), so that when a matched
1063 // context in the module is found, the profiles of all its callees are
1064 // recursively loaded. A list is needed since the order of profiles matters.
1065 if (ProfileIsCS)
1066 return true;
1067
1068 // If the profile is MD5, use the map container to lookup functions in
1069 // the module. A remapper has no use on MD5 names.
1070 if (useMD5())
1071 return false;
1072
1073 // Profile is not MD5 and if a remapper is present, the remapped name of
1074 // every function needed to be matched against the module, so use the list
1075 // container since each entry is accessed.
1076 if (Remapper)
1077 return true;
1078
1079 // Otherwise use the map container for faster lookup.
1080 // TODO: If the cardinality of the function offset section is much smaller
1081 // than the number of functions in the module, using the list container can
1082 // be always faster, but we need to figure out the constant factor to
1083 // determine the cutoff.
1084 return false;
1085}
1086
1087std::error_code
1088SampleProfileReaderExtBinaryBase::read(const DenseSet<StringRef> &FuncsToUse,
1089 SampleProfileMap &Profiles) {
1090 if (FuncsToUse.empty())
1091 return sampleprof_error::success;
1092
1093 Data = ProfileSecRange.Start;
1094 End = ProfileSecRange.End;
1095 if (std::error_code EC = readFuncProfiles(FuncsToUse, Profiles))
1096 return EC;
1097 End = Data;
1098 DenseSet<FunctionSamples *> ProfilesToReadMetadata;
1099 for (auto FName : FuncsToUse) {
1100 auto I = Profiles.find(Ctx: FName);
1101 if (I != Profiles.end())
1102 ProfilesToReadMetadata.insert(V: &I->second);
1103 }
1104
1105 if (std::error_code EC = readFuncMetadata(Profiles&: ProfilesToReadMetadata))
1106 return EC;
1107 return sampleprof_error::success;
1108}
1109
1110bool SampleProfileReaderExtBinaryBase::collectFuncsFromModule() {
1111 if (!M)
1112 return false;
1113 FuncsToUse.clear();
1114 for (auto &F : *M)
1115 FuncsToUse.insert(V: FunctionSamples::getCanonicalFnName(F));
1116 return true;
1117}
1118
1119std::error_code
1120SampleProfileReaderExtBinaryBase::readFuncOffsetTable(bool IsEytzinger,
1121 bool IsNested) {
1122 if (IsEytzinger)
1123 return readEytzingerFuncOffsetTable(IsNested);
1124 return readLegacyFuncOffsetTable();
1125}
1126
1127std::error_code
1128SampleProfileReaderExtBinaryBase::readEytzingerFuncOffsetTable(bool IsNested) {
1129 // If there are more than one function offset section, the profile associated
1130 // with the previous section has to be done reading before next one is read.
1131 FuncOffsetTable.reset();
1132
1133 size_t Size = End - Data;
1134 size_t SpanSize = NameTable->getEytzingerSpan(IsNested).size();
1135 if (Size != SpanSize * sizeof(uint32_t))
1136 return sampleprof_error::malformed;
1137
1138 auto *Array = reinterpret_cast<const support::ulittle32_t *>(Data);
1139 ArrayRef<support::ulittle32_t> Offsets(Array, SpanSize);
1140
1141 FuncOffsetTable.emplace(args: EytzingerMode, args: NameTable->getEytzingerSpan(IsNested),
1142 args&: Offsets);
1143
1144 Data = End;
1145 return sampleprof_error::success;
1146}
1147
1148std::error_code SampleProfileReaderExtBinaryBase::readLegacyFuncOffsetTable() {
1149 // If there are more than one function offset section, the profile associated
1150 // with the previous section has to be done reading before next one is read.
1151 FuncOffsetTable.reset();
1152 FuncOffsetList.clear();
1153
1154 auto Size = readNumber<uint64_t>();
1155 if (std::error_code EC = Size.getError())
1156 return EC;
1157
1158 bool UseFuncOffsetList = useFuncOffsetList();
1159 if (UseFuncOffsetList)
1160 FuncOffsetList.reserve(n: *Size);
1161 else
1162 FuncOffsetTable.emplace(args: InMemoryMode, args&: *Size);
1163
1164 for (uint64_t I = 0; I < *Size; ++I) {
1165 auto FContextHash(readSampleContextFromTable());
1166 if (std::error_code EC = FContextHash.getError())
1167 return EC;
1168
1169 auto &[FContext, Hash] = *FContextHash;
1170 auto Offset = readNumber<uint64_t>();
1171 if (std::error_code EC = Offset.getError())
1172 return EC;
1173
1174 if (UseFuncOffsetList)
1175 FuncOffsetList.emplace_back(args&: FContext, args&: *Offset);
1176 else
1177 // Because Porfiles replace existing value with new value if collision
1178 // happens, we also use the latest offset so that they are consistent.
1179 FuncOffsetTable->insert(GUID: Hash, Offset: *Offset);
1180 }
1181
1182 return sampleprof_error::success;
1183}
1184
1185std::error_code SampleProfileReaderExtBinaryBase::readFuncProfiles(
1186 const DenseSet<StringRef> &FuncsToUse, SampleProfileMap &Profiles) {
1187 const uint8_t *Start = Data;
1188
1189 if (Remapper) {
1190 for (auto Name : FuncsToUse) {
1191 Remapper->insert(FunctionName: Name);
1192 }
1193 }
1194
1195 if (FuncOffsetTable && FuncOffsetTable->isEytzinger() &&
1196 useFuncOffsetList()) {
1197 ArrayRef<support::ulittle32_t> Offsets = FuncOffsetTable->getFuncOffsets();
1198 if (Offsets.size() != FuncOffsetTable->getExpectedSize())
1199 return sampleprof_error::malformed;
1200 for (const auto &[LocalIdx, RelOffset] : llvm::enumerate(First&: Offsets)) {
1201 if (RelOffset == UINT32_MAX)
1202 continue;
1203 const uint8_t *FuncProfileAddr = Start + RelOffset;
1204 if (std::error_code EC = readFuncProfile(Start: FuncProfileAddr, Profiles))
1205 return EC;
1206 }
1207 return sampleprof_error::success;
1208 }
1209
1210 if (ProfileIsCS) {
1211 assert(useFuncOffsetList());
1212 DenseSet<uint64_t> FuncGuidsToUse;
1213 if (useMD5()) {
1214 for (auto Name : FuncsToUse)
1215 FuncGuidsToUse.insert(V: Function::getGUIDAssumingExternalLinkage(GlobalName: Name));
1216 }
1217
1218 // For each function in current module, load all context profiles for
1219 // the function as well as their callee contexts which can help profile
1220 // guided importing for ThinLTO. This can be achieved by walking
1221 // through an ordered context container, where contexts are laid out
1222 // as if they were walked in preorder of a context trie. While
1223 // traversing the trie, a link to the highest common ancestor node is
1224 // kept so that all of its decendants will be loaded.
1225 const SampleContext *CommonContext = nullptr;
1226 for (const auto &NameOffset : FuncOffsetList) {
1227 const auto &FContext = NameOffset.first;
1228 FunctionId FName = FContext.getFunction();
1229 StringRef FNameString;
1230 if (!useMD5())
1231 FNameString = FName.stringRef();
1232
1233 // For function in the current module, keep its farthest ancestor
1234 // context. This can be used to load itself and its child and
1235 // sibling contexts.
1236 if ((useMD5() && FuncGuidsToUse.count(V: FName.getHashCode())) ||
1237 (!useMD5() && (FuncsToUse.count(V: FNameString) ||
1238 (Remapper && Remapper->exist(FunctionName: FNameString))))) {
1239 if (!CommonContext || !CommonContext->isPrefixOf(That: FContext))
1240 CommonContext = &FContext;
1241 }
1242
1243 if (CommonContext == &FContext ||
1244 (CommonContext && CommonContext->isPrefixOf(That: FContext))) {
1245 // Load profile for the current context which originated from
1246 // the common ancestor.
1247 const uint8_t *FuncProfileAddr = Start + NameOffset.second;
1248 if (std::error_code EC = readFuncProfile(Start: FuncProfileAddr))
1249 return EC;
1250 }
1251 }
1252 } else if (useMD5()) {
1253 assert(!useFuncOffsetList());
1254 for (auto Name : FuncsToUse) {
1255 auto GUID = MD5Hash(Str: Name);
1256 if (auto Offset = FuncOffsetTable->lookup(GUID)) {
1257 const uint8_t *FuncProfileAddr = Start + *Offset;
1258 if (std::error_code EC = readFuncProfile(Start: FuncProfileAddr, Profiles))
1259 return EC;
1260 }
1261 }
1262 } else if (Remapper) {
1263 assert(useFuncOffsetList());
1264 for (auto NameOffset : FuncOffsetList) {
1265 SampleContext FContext(NameOffset.first);
1266 auto FuncName = FContext.getFunction();
1267 StringRef FuncNameStr = FuncName.stringRef();
1268 if (!FuncsToUse.count(V: FuncNameStr) && !Remapper->exist(FunctionName: FuncNameStr))
1269 continue;
1270 const uint8_t *FuncProfileAddr = Start + NameOffset.second;
1271 if (std::error_code EC = readFuncProfile(Start: FuncProfileAddr, Profiles))
1272 return EC;
1273 }
1274 } else {
1275 assert(!useFuncOffsetList());
1276 for (auto Name : FuncsToUse) {
1277 if (auto Offset = FuncOffsetTable->lookup(GUID: MD5Hash(Str: Name))) {
1278 const uint8_t *FuncProfileAddr = Start + *Offset;
1279 if (std::error_code EC = readFuncProfile(Start: FuncProfileAddr, Profiles))
1280 return EC;
1281 }
1282 }
1283 }
1284
1285 return sampleprof_error::success;
1286}
1287
1288std::error_code SampleProfileReaderExtBinaryBase::readFuncProfiles() {
1289 // Collect functions used by current module if the Reader has been
1290 // given a module.
1291 // collectFuncsFromModule uses FunctionSamples::getCanonicalFnName
1292 // which will query FunctionSamples::HasUniqSuffix, so it has to be
1293 // called after FunctionSamples::HasUniqSuffix is set, i.e. after
1294 // NameTable section is read.
1295 bool LoadFuncsToBeUsed = collectFuncsFromModule();
1296
1297 // When LoadFuncsToBeUsed is false, we are using LLVM tool, need to read all
1298 // profiles.
1299 if (!LoadFuncsToBeUsed) {
1300 while (Data < End) {
1301 if (std::error_code EC = readFuncProfile(Start: Data))
1302 return EC;
1303 }
1304 assert(Data == End && "More data is read than expected");
1305 } else {
1306 // Load function profiles on demand.
1307 if (std::error_code EC = readFuncProfiles(FuncsToUse, Profiles))
1308 return EC;
1309 Data = End;
1310 }
1311 assert((CSProfileCount == 0 || CSProfileCount == Profiles.size()) &&
1312 "Cannot have both context-sensitive and regular profile");
1313 assert((!CSProfileCount || ProfileIsCS) &&
1314 "Section flag should be consistent with actual profile");
1315 return sampleprof_error::success;
1316}
1317
1318std::error_code
1319SampleProfileReaderExtBinaryBase::readProfileSymbolList(bool IsMD5) {
1320 if (IsMD5)
1321 return readMD5ProfileSymbolList();
1322 return readStringBasedProfileSymbolList();
1323}
1324
1325std::error_code SampleProfileReaderExtBinaryBase::readMD5ProfileSymbolList() {
1326 size_t Size = End - Data;
1327 if (Size % sizeof(uint64_t) != 0)
1328 return sampleprof_error::truncated;
1329 const auto *Table = reinterpret_cast<const support::ulittle64_t *>(Data);
1330 size_t NumEntries = Size / sizeof(uint64_t);
1331 if (!ProfSymList)
1332 ProfSymList = std::make_unique<ProfileSymbolList>();
1333 ProfSymList->setColdGUIDTable(
1334 EytzingerTableSpan<support::ulittle64_t>(Table, NumEntries));
1335 Data = End;
1336 return sampleprof_error::success;
1337}
1338
1339std::error_code
1340SampleProfileReaderExtBinaryBase::readStringBasedProfileSymbolList() {
1341 if (!ProfSymList)
1342 ProfSymList = std::make_unique<ProfileSymbolList>();
1343
1344 if (std::error_code EC = ProfSymList->read(Data, ListSize: End - Data))
1345 return EC;
1346
1347 Data = End;
1348 return sampleprof_error::success;
1349}
1350
1351std::error_code SampleProfileReaderExtBinaryBase::decompressSection(
1352 const uint8_t *SecStart, const uint64_t SecSize,
1353 const uint8_t *&DecompressBuf, uint64_t &DecompressBufSize) {
1354 Data = SecStart;
1355 End = SecStart + SecSize;
1356 auto DecompressSize = readNumber<uint64_t>();
1357 if (std::error_code EC = DecompressSize.getError())
1358 return EC;
1359 DecompressBufSize = *DecompressSize;
1360
1361 auto CompressSize = readNumber<uint64_t>();
1362 if (std::error_code EC = CompressSize.getError())
1363 return EC;
1364
1365 if (!llvm::compression::zlib::isAvailable())
1366 return sampleprof_error::zlib_unavailable;
1367
1368 uint8_t *Buffer = Allocator.Allocate<uint8_t>(Num: DecompressBufSize);
1369 size_t UCSize = DecompressBufSize;
1370 llvm::Error E = compression::zlib::decompress(Input: ArrayRef(Data, *CompressSize),
1371 Output: Buffer, UncompressedSize&: UCSize);
1372 if (E)
1373 return sampleprof_error::uncompress_failed;
1374 DecompressBuf = reinterpret_cast<const uint8_t *>(Buffer);
1375 return sampleprof_error::success;
1376}
1377
1378std::error_code SampleProfileReaderExtBinaryBase::readImpl() {
1379 const uint8_t *BufStart =
1380 reinterpret_cast<const uint8_t *>(Buffer->getBufferStart());
1381
1382 for (auto &Entry : SecHdrTable) {
1383 // Skip empty section.
1384 if (!Entry.Size)
1385 continue;
1386
1387 // Skip sections without inlined functions when SkipFlatProf is true.
1388 if (SkipFlatProf && hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagFlat))
1389 continue;
1390
1391 const uint8_t *SecStart = BufStart + Entry.Offset;
1392 uint64_t SecSize = Entry.Size;
1393
1394 // If the section is compressed, decompress it into a buffer
1395 // DecompressBuf before reading the actual data. The pointee of
1396 // 'Data' will be changed to buffer hold by DecompressBuf
1397 // temporarily when reading the actual data.
1398 bool isCompressed = hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagCompress);
1399 if (isCompressed) {
1400 const uint8_t *DecompressBuf;
1401 uint64_t DecompressBufSize;
1402 if (std::error_code EC = decompressSection(
1403 SecStart, SecSize, DecompressBuf, DecompressBufSize))
1404 return EC;
1405 SecStart = DecompressBuf;
1406 SecSize = DecompressBufSize;
1407 }
1408
1409 if (std::error_code EC = readOneSection(Start: SecStart, Size: SecSize, Entry))
1410 return EC;
1411 if (Data != SecStart + SecSize)
1412 return sampleprof_error::malformed;
1413
1414 // Change the pointee of 'Data' from DecompressBuf to original Buffer.
1415 if (isCompressed) {
1416 Data = BufStart + Entry.Offset;
1417 End = BufStart + Buffer->getBufferSize();
1418 }
1419 }
1420
1421 return sampleprof_error::success;
1422}
1423
1424std::error_code SampleProfileReaderRawBinary::verifySPMagic(uint64_t Magic) {
1425 if (Magic == SPMagic())
1426 return sampleprof_error::success;
1427 return sampleprof_error::bad_magic;
1428}
1429
1430std::error_code SampleProfileReaderExtBinary::verifySPMagic(uint64_t Magic) {
1431 if (Magic == SPMagic(Format: SPF_Ext_Binary))
1432 return sampleprof_error::success;
1433 return sampleprof_error::bad_magic;
1434}
1435
1436std::error_code SampleProfileReaderBinary::readNameTable() {
1437 auto Size = readNumber<size_t>();
1438 if (std::error_code EC = Size.getError())
1439 return EC;
1440
1441 // Normally if useMD5 is true, the name table should have MD5 values, not
1442 // strings, however in the case that ExtBinary profile has multiple name
1443 // tables mixing string and MD5, all of them have to be normalized to use MD5,
1444 // because optimization passes can only handle either type.
1445 bool UseMD5 = useMD5();
1446
1447 std::vector<FunctionId> TableVec;
1448 TableVec.reserve(n: *Size);
1449 if (!ProfileIsCS) {
1450 MD5SampleContextTable.clear();
1451 if (UseMD5)
1452 MD5SampleContextTable.reserve(n: *Size);
1453 else
1454 // If we are using strings, delay MD5 computation since only a portion of
1455 // names are used by top level functions. Use 0 to indicate MD5 value is
1456 // to be calculated as no known string has a MD5 value of 0.
1457 MD5SampleContextTable.resize(new_size: *Size);
1458 }
1459 for (size_t I = 0; I < *Size; ++I) {
1460 auto Name(readString());
1461 if (std::error_code EC = Name.getError())
1462 return EC;
1463 if (UseMD5) {
1464 FunctionId FID(*Name);
1465 if (!ProfileIsCS)
1466 MD5SampleContextTable.emplace_back(args: FID.getHashCode());
1467 TableVec.emplace_back(args&: FID);
1468 } else
1469 TableVec.push_back(x: FunctionId(*Name));
1470 }
1471 if (!ProfileIsCS)
1472 MD5SampleContextStart = MD5SampleContextTable.data();
1473 if (UseMD5)
1474 NameTable =
1475 std::make_unique<MD5SampleProfileNameTable>(args: std::move(TableVec));
1476 else
1477 NameTable =
1478 std::make_unique<StringSampleProfileNameTable>(args: std::move(TableVec));
1479 return sampleprof_error::success;
1480}
1481
1482std::error_code SampleProfileReaderExtBinaryBase::readNameTableSec(
1483 bool IsMD5, bool FixedLengthMD5, bool IsEytzinger) {
1484 if (IsEytzinger)
1485 return readNameTableSecEytzinger(IsMD5, FixedLengthMD5);
1486 return readNameTableSecLegacy(IsMD5, FixedLengthMD5);
1487}
1488
1489// Read the Eytzinger layout for SecNameTable from an ExtBinary MD5 profile.
1490//
1491// The section consists of three sequential ULEB128 symbol counts (Nested, Flat,
1492// and Inlinees) followed by their corresponding arrays of 64-bit MD5 hash keys
1493// laid out in Eytzinger order.
1494std::error_code SampleProfileReaderExtBinaryBase::readNameTableSecEytzinger(
1495 bool IsMD5, bool FixedLengthMD5) {
1496 assert(IsMD5 && "Eytzinger name tables require MD5 representation");
1497 if (!IsMD5)
1498 return sampleprof_error::malformed;
1499
1500 // Read the table sizes for Nested, flat, and inlinee symbols.
1501 std::array<uint64_t, static_cast<size_t>(EytzingerSpan::NumSpans)> Counts;
1502 for (uint64_t &Count : Counts) {
1503 auto ValOrErr = readNumber<uint64_t>();
1504 if (std::error_code EC = ValOrErr.getError())
1505 return EC;
1506 Count = *ValOrErr;
1507 }
1508 auto [NumNested, NumFlat, NumInlinees] = Counts;
1509
1510 // Guard against unsigned overflow in total entry computation.
1511 if (NumNested > std::numeric_limits<uint32_t>::max() ||
1512 NumFlat > std::numeric_limits<uint32_t>::max() ||
1513 NumInlinees > std::numeric_limits<uint32_t>::max())
1514 return sampleprof_error::malformed;
1515
1516 uint64_t TotalEntries = NumNested + NumFlat + NumInlinees;
1517 if (static_cast<size_t>(End - Data) < TotalEntries * sizeof(uint64_t))
1518 return sampleprof_error::truncated;
1519
1520 NameTable = std::make_unique<EytzingerSampleProfileNameTable>(
1521 args: reinterpret_cast<const support::ulittle64_t *>(Data), args&: NumNested, args&: NumFlat,
1522 args&: NumInlinees);
1523
1524 if (!ProfileIsCS)
1525 MD5SampleContextStart = reinterpret_cast<const uint64_t *>(Data);
1526 Data = Data + TotalEntries * sizeof(uint64_t);
1527 return sampleprof_error::success;
1528}
1529
1530std::error_code
1531SampleProfileReaderExtBinaryBase::readNameTableSecLegacy(bool IsMD5,
1532 bool FixedLengthMD5) {
1533 if (FixedLengthMD5) {
1534 if (!IsMD5)
1535 errs() << "If FixedLengthMD5 is true, UseMD5 has to be true";
1536 auto Size = readNumber<size_t>();
1537 if (std::error_code EC = Size.getError())
1538 return EC;
1539
1540 assert(Data + (*Size) * sizeof(uint64_t) == End &&
1541 "Fixed length MD5 name table does not contain specified number of "
1542 "entries");
1543 if (Data + (*Size) * sizeof(uint64_t) > End)
1544 return sampleprof_error::truncated;
1545
1546 if (LazyLoadNameTable) {
1547 NameTable = std::make_unique<LazySampleProfileNameTable>(args&: Data, args&: *Size);
1548 } else {
1549 std::vector<FunctionId> TableVec;
1550 TableVec.reserve(n: *Size);
1551 for (size_t I = 0; I < *Size; ++I) {
1552 using namespace support;
1553 uint64_t FID = endian::read<uint64_t, unaligned>(
1554 memory: Data + I * sizeof(uint64_t), endian: endianness::little);
1555 TableVec.emplace_back(args: FunctionId(FID));
1556 }
1557 NameTable =
1558 std::make_unique<MD5SampleProfileNameTable>(args: std::move(TableVec));
1559 }
1560 if (!ProfileIsCS)
1561 MD5SampleContextStart = reinterpret_cast<const uint64_t *>(Data);
1562 Data = Data + (*Size) * sizeof(uint64_t);
1563 return sampleprof_error::success;
1564 }
1565
1566 if (IsMD5) {
1567 assert(!FixedLengthMD5 && "FixedLengthMD5 should be unreachable here");
1568 auto Size = readNumber<size_t>();
1569 if (std::error_code EC = Size.getError())
1570 return EC;
1571
1572 std::vector<FunctionId> TableVec;
1573 TableVec.reserve(n: *Size);
1574 if (!ProfileIsCS)
1575 MD5SampleContextTable.resize(new_size: *Size);
1576 for (size_t I = 0; I < *Size; ++I) {
1577 auto FID = readNumber<uint64_t>();
1578 if (std::error_code EC = FID.getError())
1579 return EC;
1580 if (!ProfileIsCS)
1581 support::endian::write64le(P: &MD5SampleContextTable[I], V: *FID);
1582 TableVec.emplace_back(args: FunctionId(*FID));
1583 }
1584 if (!ProfileIsCS)
1585 MD5SampleContextStart = MD5SampleContextTable.data();
1586 NameTable =
1587 std::make_unique<MD5SampleProfileNameTable>(args: std::move(TableVec));
1588 return sampleprof_error::success;
1589 }
1590
1591 return SampleProfileReaderBinary::readNameTable();
1592}
1593
1594// Read in the CS name table section, which basically contains a list of context
1595// vectors. Each element of a context vector, aka a frame, refers to the
1596// underlying raw function names that are stored in the name table, as well as
1597// a callsite identifier that only makes sense for non-leaf frames.
1598std::error_code SampleProfileReaderExtBinaryBase::readCSNameTableSec() {
1599 auto Size = readNumber<size_t>();
1600 if (std::error_code EC = Size.getError())
1601 return EC;
1602
1603 CSNameTable.clear();
1604 CSNameTable.reserve(n: *Size);
1605 if (ProfileIsCS) {
1606 // Delay MD5 computation of CS context until they are needed. Use 0 to
1607 // indicate MD5 value is to be calculated as no known string has a MD5
1608 // value of 0.
1609 MD5SampleContextTable.clear();
1610 MD5SampleContextTable.resize(new_size: *Size);
1611 MD5SampleContextStart = MD5SampleContextTable.data();
1612 }
1613 for (size_t I = 0; I < *Size; ++I) {
1614 CSNameTable.emplace_back(args: SampleContextFrameVector());
1615 auto ContextSize = readNumber<uint32_t>();
1616 if (std::error_code EC = ContextSize.getError())
1617 return EC;
1618 for (uint32_t J = 0; J < *ContextSize; ++J) {
1619 auto FName(readStringFromTable());
1620 if (std::error_code EC = FName.getError())
1621 return EC;
1622 auto LineOffset = readNumber<uint64_t>();
1623 if (std::error_code EC = LineOffset.getError())
1624 return EC;
1625
1626 if (!isOffsetLegal(L: *LineOffset))
1627 return sampleprof_error::illegal_line_offset;
1628
1629 auto Discriminator = readNumber<uint64_t>();
1630 if (std::error_code EC = Discriminator.getError())
1631 return EC;
1632
1633 CSNameTable.back().emplace_back(
1634 Args&: FName.get(), Args: LineLocation(LineOffset.get(), Discriminator.get()));
1635 }
1636 }
1637
1638 return sampleprof_error::success;
1639}
1640
1641std::error_code
1642SampleProfileReaderExtBinaryBase::readFuncMetadata(FunctionSamples *FProfile) {
1643 if (Data < End) {
1644 if (ProfileIsProbeBased) {
1645 auto Checksum = readNumber<uint64_t>();
1646 if (std::error_code EC = Checksum.getError())
1647 return EC;
1648 if (FProfile)
1649 FProfile->setFunctionHash(*Checksum);
1650 }
1651
1652 if (ProfileHasAttribute) {
1653 auto Attributes = readNumber<uint32_t>();
1654 if (std::error_code EC = Attributes.getError())
1655 return EC;
1656 if (FProfile)
1657 FProfile->getContext().setAllAttributes(*Attributes);
1658 }
1659
1660 if (!ProfileIsCS) {
1661 // Read all the attributes for inlined function calls.
1662 auto NumCallsites = readNumber<uint32_t>();
1663 if (std::error_code EC = NumCallsites.getError())
1664 return EC;
1665
1666 for (uint32_t J = 0; J < *NumCallsites; ++J) {
1667 auto LineOffset = readNumber<uint64_t>();
1668 if (std::error_code EC = LineOffset.getError())
1669 return EC;
1670
1671 auto Discriminator = readNumber<uint64_t>();
1672 if (std::error_code EC = Discriminator.getError())
1673 return EC;
1674
1675 auto FContextHash(readSampleContextFromTable());
1676 if (std::error_code EC = FContextHash.getError())
1677 return EC;
1678
1679 auto &[FContext, Hash] = *FContextHash;
1680 FunctionSamples *CalleeProfile = nullptr;
1681 if (FProfile) {
1682 CalleeProfile = const_cast<FunctionSamples *>(
1683 &FProfile->functionSamplesAt(Loc: LineLocation(
1684 *LineOffset, *Discriminator))[FContext.getFunction()]);
1685 }
1686 if (std::error_code EC = readFuncMetadata(FProfile: CalleeProfile))
1687 return EC;
1688 }
1689 }
1690 }
1691
1692 return sampleprof_error::success;
1693}
1694
1695std::error_code SampleProfileReaderExtBinaryBase::readFuncMetadata(
1696 DenseSet<FunctionSamples *> &Profiles) {
1697 if (FuncMetadataIndex.empty())
1698 return sampleprof_error::success;
1699
1700 for (auto *FProfile : Profiles) {
1701 auto R = FuncMetadataIndex.find(Val: FProfile->getContext().getHashCode());
1702 if (R == FuncMetadataIndex.end())
1703 continue;
1704
1705 Data = R->second.first;
1706 End = R->second.second;
1707 if (std::error_code EC = readFuncMetadata(FProfile))
1708 return EC;
1709 assert(Data == End && "More data is read than expected");
1710 }
1711 return sampleprof_error::success;
1712}
1713
1714std::error_code SampleProfileReaderExtBinaryBase::readFuncMetadata() {
1715 while (Data < End) {
1716 auto FContextHash(readSampleContextFromTable());
1717 if (std::error_code EC = FContextHash.getError())
1718 return EC;
1719 auto &[FContext, Hash] = *FContextHash;
1720 FunctionSamples *FProfile = nullptr;
1721 auto It = Profiles.find(Ctx: FContext);
1722 if (It != Profiles.end())
1723 FProfile = &It->second;
1724
1725 const uint8_t *Start = Data;
1726 if (std::error_code EC = readFuncMetadata(FProfile))
1727 return EC;
1728
1729 FuncMetadataIndex[FContext.getHashCode()] = {Start, Data};
1730 }
1731
1732 assert(Data == End && "More data is read than expected");
1733 return sampleprof_error::success;
1734}
1735
1736std::error_code
1737SampleProfileReaderExtBinaryBase::readSecHdrTableEntry(uint64_t Idx) {
1738 SecHdrTableEntry Entry;
1739 auto Type = readUnencodedNumber<uint64_t>();
1740 if (std::error_code EC = Type.getError())
1741 return EC;
1742 Entry.Type = static_cast<SecType>(*Type);
1743
1744 // Reject a section whose encoding is newer than the declared file version.
1745 if ((Entry.Type == SecCompositeProfile ||
1746 Entry.Type == SecCompositeFuncOffsetTable) &&
1747 FormatVersion < CompositeProfileVersion)
1748 return sampleprof_error::unsupported_version;
1749
1750 auto Flags = readUnencodedNumber<uint64_t>();
1751 if (std::error_code EC = Flags.getError())
1752 return EC;
1753 Entry.Flags = *Flags;
1754
1755 auto Offset = readUnencodedNumber<uint64_t>();
1756 if (std::error_code EC = Offset.getError())
1757 return EC;
1758 Entry.Offset = *Offset;
1759
1760 auto Size = readUnencodedNumber<uint64_t>();
1761 if (std::error_code EC = Size.getError())
1762 return EC;
1763 Entry.Size = *Size;
1764
1765 Entry.LayoutIndex = Idx;
1766 SecHdrTable.push_back(x: std::move(Entry));
1767 return sampleprof_error::success;
1768}
1769
1770std::error_code SampleProfileReaderExtBinaryBase::readSecHdrTable() {
1771 auto EntryNum = readUnencodedNumber<uint64_t>();
1772 if (std::error_code EC = EntryNum.getError())
1773 return EC;
1774
1775 for (uint64_t i = 0; i < (*EntryNum); i++)
1776 if (std::error_code EC = readSecHdrTableEntry(Idx: i))
1777 return EC;
1778
1779 return sampleprof_error::success;
1780}
1781
1782std::error_code SampleProfileReaderExtBinaryBase::readHeader() {
1783 const uint8_t *BufStart =
1784 reinterpret_cast<const uint8_t *>(Buffer->getBufferStart());
1785 Data = BufStart;
1786 End = BufStart + Buffer->getBufferSize();
1787
1788 if (std::error_code EC = readMagicIdent())
1789 return EC;
1790
1791 if (std::error_code EC = readSecHdrTable())
1792 return EC;
1793
1794 return sampleprof_error::success;
1795}
1796
1797uint64_t SampleProfileReaderExtBinaryBase::getSectionSize(SecType Type) {
1798 uint64_t Size = 0;
1799 for (auto &Entry : SecHdrTable) {
1800 if (Entry.Type == Type)
1801 Size += Entry.Size;
1802 }
1803 return Size;
1804}
1805
1806uint64_t SampleProfileReaderExtBinaryBase::getFileSize() {
1807 // Sections in SecHdrTable is not necessarily in the same order as
1808 // sections in the profile because section like FuncOffsetTable needs
1809 // to be written after section LBRProfile but needs to be read before
1810 // section LBRProfile, so we cannot simply use the last entry in
1811 // SecHdrTable to calculate the file size.
1812 uint64_t FileSize = 0;
1813 for (auto &Entry : SecHdrTable) {
1814 FileSize = std::max(a: Entry.Offset + Entry.Size, b: FileSize);
1815 }
1816 return FileSize;
1817}
1818
1819static std::string getSecFlagsStr(const SecHdrTableEntry &Entry) {
1820 std::string Flags;
1821 if (hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagCompress))
1822 Flags.append(s: "{compressed,");
1823 else
1824 Flags.append(s: "{");
1825
1826 if (hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagFlat))
1827 Flags.append(s: "flat,");
1828
1829 switch (Entry.Type) {
1830 case SecNameTable:
1831 if (hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagEytzinger))
1832 Flags.append(s: "eytzinger,");
1833 if (hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagFixedLengthMD5))
1834 Flags.append(s: "fixlenmd5,");
1835 else if (hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagMD5Name))
1836 Flags.append(s: "md5,");
1837 if (hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagUniqSuffix))
1838 Flags.append(s: "uniq,");
1839 break;
1840 case SecProfSummary:
1841 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagPartial))
1842 Flags.append(s: "partial,");
1843 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagFullContext))
1844 Flags.append(s: "context,");
1845 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagIsPreInlined))
1846 Flags.append(s: "preInlined,");
1847 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagFSDiscriminator))
1848 Flags.append(s: "fs-discriminator,");
1849 break;
1850 case SecFuncOffsetTable:
1851 case SecCompositeFuncOffsetTable:
1852 if (hasSecFlag(Entry, Flag: SecFuncOffsetFlags::SecFlagOrdered))
1853 Flags.append(s: "ordered,");
1854 if (hasSecFlag(Entry, Flag: SecFuncOffsetFlags::SecFlagEytzinger))
1855 Flags.append(s: "eytzinger,");
1856 break;
1857 case SecFuncMetadata:
1858 if (hasSecFlag(Entry, Flag: SecFuncMetadataFlags::SecFlagIsProbeBased))
1859 Flags.append(s: "probe,");
1860 if (hasSecFlag(Entry, Flag: SecFuncMetadataFlags::SecFlagHasAttribute))
1861 Flags.append(s: "attr,");
1862 break;
1863 case SecProfileSymbolList:
1864 if (hasSecFlag(Entry, Flag: SecProfileSymbolListFlags::SecFlagMD5))
1865 Flags.append(s: "md5,");
1866 break;
1867 default:
1868 break;
1869 }
1870 char &last = Flags.back();
1871 if (last == ',')
1872 last = '}';
1873 else
1874 Flags.append(s: "}");
1875 return Flags;
1876}
1877
1878bool SampleProfileReaderExtBinaryBase::dumpSectionInfo(raw_ostream &OS) {
1879 uint64_t TotalSecsSize = 0;
1880 for (auto &Entry : SecHdrTable) {
1881 OS << getSecName(Type: Entry.Type) << " - Offset: " << Entry.Offset
1882 << ", Size: " << Entry.Size << ", Flags: " << getSecFlagsStr(Entry)
1883 << "\n";
1884 ;
1885 TotalSecsSize += Entry.Size;
1886 }
1887 uint64_t HeaderSize = SecHdrTable.front().Offset;
1888 assert(HeaderSize + TotalSecsSize == getFileSize() &&
1889 "Size of 'header + sections' doesn't match the total size of profile");
1890
1891 OS << "Header Size: " << HeaderSize << "\n";
1892 OS << "Total Sections Size: " << TotalSecsSize << "\n";
1893 OS << "File Size: " << getFileSize() << "\n";
1894 return true;
1895}
1896
1897std::error_code SampleProfileReaderBinary::readMagicIdent() {
1898 // Read and check the magic identifier.
1899 auto Magic = readNumber<uint64_t>();
1900 if (std::error_code EC = Magic.getError())
1901 return EC;
1902 else if (std::error_code EC = verifySPMagic(Magic: *Magic))
1903 return EC;
1904
1905 // Read the version number.
1906 auto Version = readNumber<uint64_t>();
1907 if (std::error_code EC = Version.getError())
1908 return EC;
1909 else if (!formatVersionIsSupported(Version: *Version))
1910 return sampleprof_error::unsupported_version;
1911 FormatVersion = *Version;
1912
1913 return sampleprof_error::success;
1914}
1915
1916std::error_code SampleProfileReaderBinary::readHeader() {
1917 Data = reinterpret_cast<const uint8_t *>(Buffer->getBufferStart());
1918 End = Data + Buffer->getBufferSize();
1919
1920 if (std::error_code EC = readMagicIdent())
1921 return EC;
1922
1923 if (std::error_code EC = readSummary())
1924 return EC;
1925
1926 if (std::error_code EC = readNameTable())
1927 return EC;
1928 return sampleprof_error::success;
1929}
1930
1931std::error_code SampleProfileReaderBinary::readSummaryEntry(
1932 std::vector<ProfileSummaryEntry> &Entries) {
1933 auto Cutoff = readNumber<uint64_t>();
1934 if (std::error_code EC = Cutoff.getError())
1935 return EC;
1936
1937 auto MinBlockCount = readNumber<uint64_t>();
1938 if (std::error_code EC = MinBlockCount.getError())
1939 return EC;
1940
1941 auto NumBlocks = readNumber<uint64_t>();
1942 if (std::error_code EC = NumBlocks.getError())
1943 return EC;
1944
1945 Entries.emplace_back(args&: *Cutoff, args&: *MinBlockCount, args&: *NumBlocks);
1946 return sampleprof_error::success;
1947}
1948
1949std::error_code SampleProfileReaderBinary::readSummary() {
1950 auto TotalCount = readNumber<uint64_t>();
1951 if (std::error_code EC = TotalCount.getError())
1952 return EC;
1953
1954 auto MaxBlockCount = readNumber<uint64_t>();
1955 if (std::error_code EC = MaxBlockCount.getError())
1956 return EC;
1957
1958 auto MaxFunctionCount = readNumber<uint64_t>();
1959 if (std::error_code EC = MaxFunctionCount.getError())
1960 return EC;
1961
1962 auto NumBlocks = readNumber<uint64_t>();
1963 if (std::error_code EC = NumBlocks.getError())
1964 return EC;
1965
1966 auto NumFunctions = readNumber<uint64_t>();
1967 if (std::error_code EC = NumFunctions.getError())
1968 return EC;
1969
1970 auto NumSummaryEntries = readNumber<uint64_t>();
1971 if (std::error_code EC = NumSummaryEntries.getError())
1972 return EC;
1973
1974 std::vector<ProfileSummaryEntry> Entries;
1975 for (unsigned i = 0; i < *NumSummaryEntries; i++) {
1976 std::error_code EC = readSummaryEntry(Entries);
1977 if (EC != sampleprof_error::success)
1978 return EC;
1979 }
1980 Summary = std::make_unique<ProfileSummary>(
1981 args: ProfileSummary::PSK_Sample, args&: Entries, args&: *TotalCount, args&: *MaxBlockCount, args: 0,
1982 args&: *MaxFunctionCount, args&: *NumBlocks, args&: *NumFunctions);
1983
1984 return sampleprof_error::success;
1985}
1986
1987/// Return whether Buffer starts with ExpectedMagic without reading beyond it.
1988static bool hasBinaryFormat(const MemoryBuffer &Buffer,
1989 uint64_t ExpectedMagic) {
1990 const uint8_t *Data =
1991 reinterpret_cast<const uint8_t *>(Buffer.getBufferStart());
1992 const uint8_t *End = reinterpret_cast<const uint8_t *>(Buffer.getBufferEnd());
1993 ULEB128DecodeError DecodeError = ULEB128DecodeError::None;
1994 uint64_t Magic = decodeULEB128(p: Data, n: nullptr, end: End, error: nullptr, errorCode: &DecodeError);
1995 return DecodeError == ULEB128DecodeError::None && Magic == ExpectedMagic;
1996}
1997
1998bool SampleProfileReaderRawBinary::hasFormat(const MemoryBuffer &Buffer) {
1999 return hasBinaryFormat(Buffer, ExpectedMagic: SPMagic());
2000}
2001
2002bool SampleProfileReaderExtBinary::hasFormat(const MemoryBuffer &Buffer) {
2003 return hasBinaryFormat(Buffer, ExpectedMagic: SPMagic(Format: SPF_Ext_Binary));
2004}
2005
2006std::error_code SampleProfileReaderGCC::skipNextWord() {
2007 uint32_t dummy;
2008 if (!GcovBuffer.readInt(Val&: dummy))
2009 return sampleprof_error::truncated;
2010 return sampleprof_error::success;
2011}
2012
2013template <typename T> ErrorOr<T> SampleProfileReaderGCC::readNumber() {
2014 if (sizeof(T) <= sizeof(uint32_t)) {
2015 uint32_t Val;
2016 if (GcovBuffer.readInt(Val) && Val <= std::numeric_limits<T>::max())
2017 return static_cast<T>(Val);
2018 } else if (sizeof(T) <= sizeof(uint64_t)) {
2019 uint64_t Val;
2020 if (GcovBuffer.readInt64(Val) && Val <= std::numeric_limits<T>::max())
2021 return static_cast<T>(Val);
2022 }
2023
2024 std::error_code EC = sampleprof_error::malformed;
2025 reportError(LineNumber: 0, Msg: EC.message());
2026 return EC;
2027}
2028
2029ErrorOr<StringRef> SampleProfileReaderGCC::readString() {
2030 StringRef Str;
2031 if (!GcovBuffer.readString(str&: Str))
2032 return sampleprof_error::truncated;
2033 return Str;
2034}
2035
2036std::error_code SampleProfileReaderGCC::readHeader() {
2037 // Read the magic identifier.
2038 if (!GcovBuffer.readGCDAFormat())
2039 return sampleprof_error::unrecognized_format;
2040
2041 // Read the version number. Note - the GCC reader does not validate this
2042 // version, but the profile creator generates v704.
2043 GCOV::GCOVVersion version;
2044 if (!GcovBuffer.readGCOVVersion(version))
2045 return sampleprof_error::unrecognized_format;
2046
2047 if (version != GCOV::V407)
2048 return sampleprof_error::unsupported_version;
2049
2050 // Skip the empty integer.
2051 if (std::error_code EC = skipNextWord())
2052 return EC;
2053
2054 return sampleprof_error::success;
2055}
2056
2057std::error_code SampleProfileReaderGCC::readSectionTag(uint32_t Expected) {
2058 uint32_t Tag;
2059 if (!GcovBuffer.readInt(Val&: Tag))
2060 return sampleprof_error::truncated;
2061
2062 if (Tag != Expected)
2063 return sampleprof_error::malformed;
2064
2065 if (std::error_code EC = skipNextWord())
2066 return EC;
2067
2068 return sampleprof_error::success;
2069}
2070
2071std::error_code SampleProfileReaderGCC::readNameTable() {
2072 if (std::error_code EC = readSectionTag(Expected: GCOVTagAFDOFileNames))
2073 return EC;
2074
2075 uint32_t Size;
2076 if (!GcovBuffer.readInt(Val&: Size))
2077 return sampleprof_error::truncated;
2078
2079 for (uint32_t I = 0; I < Size; ++I) {
2080 StringRef Str;
2081 if (!GcovBuffer.readString(str&: Str))
2082 return sampleprof_error::truncated;
2083 Names.push_back(x: std::string(Str));
2084 }
2085
2086 return sampleprof_error::success;
2087}
2088
2089std::error_code SampleProfileReaderGCC::readFunctionProfiles() {
2090 if (std::error_code EC = readSectionTag(Expected: GCOVTagAFDOFunction))
2091 return EC;
2092
2093 uint32_t NumFunctions;
2094 if (!GcovBuffer.readInt(Val&: NumFunctions))
2095 return sampleprof_error::truncated;
2096
2097 InlineCallStack Stack;
2098 for (uint32_t I = 0; I < NumFunctions; ++I)
2099 if (std::error_code EC = readOneFunctionProfile(InlineStack: Stack, Update: true, Offset: 0))
2100 return EC;
2101
2102 computeSummary();
2103 return sampleprof_error::success;
2104}
2105
2106std::error_code SampleProfileReaderGCC::readOneFunctionProfile(
2107 const InlineCallStack &InlineStack, bool Update, uint32_t Offset) {
2108 uint64_t HeadCount = 0;
2109 if (InlineStack.size() == 0)
2110 if (!GcovBuffer.readInt64(Val&: HeadCount))
2111 return sampleprof_error::truncated;
2112
2113 uint32_t NameIdx;
2114 if (!GcovBuffer.readInt(Val&: NameIdx))
2115 return sampleprof_error::truncated;
2116
2117 StringRef Name(Names[NameIdx]);
2118
2119 uint32_t NumPosCounts;
2120 if (!GcovBuffer.readInt(Val&: NumPosCounts))
2121 return sampleprof_error::truncated;
2122
2123 uint32_t NumCallsites;
2124 if (!GcovBuffer.readInt(Val&: NumCallsites))
2125 return sampleprof_error::truncated;
2126
2127 FunctionSamples *FProfile = nullptr;
2128 if (InlineStack.size() == 0) {
2129 // If this is a top function that we have already processed, do not
2130 // update its profile again. This happens in the presence of
2131 // function aliases. Since these aliases share the same function
2132 // body, there will be identical replicated profiles for the
2133 // original function. In this case, we simply not bother updating
2134 // the profile of the original function.
2135 FProfile = &Profiles[FunctionId(Name)];
2136 FProfile->addHeadSamples(Num: HeadCount);
2137 if (FProfile->getTotalSamples() > 0)
2138 Update = false;
2139 } else {
2140 // Otherwise, we are reading an inlined instance. The top of the
2141 // inline stack contains the profile of the caller. Insert this
2142 // callee in the caller's CallsiteMap.
2143 FunctionSamples *CallerProfile = InlineStack.front();
2144 uint32_t LineOffset = Offset >> 16;
2145 uint32_t Discriminator = Offset & 0xffff;
2146 FProfile = &CallerProfile->functionSamplesAt(
2147 Loc: LineLocation(LineOffset, Discriminator))[FunctionId(Name)];
2148 }
2149 FProfile->setFunction(FunctionId(Name));
2150 FProfile->reserveBodySamples(NumEntries: NumPosCounts);
2151
2152 for (uint32_t I = 0; I < NumPosCounts; ++I) {
2153 uint32_t Offset;
2154 if (!GcovBuffer.readInt(Val&: Offset))
2155 return sampleprof_error::truncated;
2156
2157 uint32_t NumTargets;
2158 if (!GcovBuffer.readInt(Val&: NumTargets))
2159 return sampleprof_error::truncated;
2160
2161 uint64_t Count;
2162 if (!GcovBuffer.readInt64(Val&: Count))
2163 return sampleprof_error::truncated;
2164
2165 // The line location is encoded in the offset as:
2166 // high 16 bits: line offset to the start of the function.
2167 // low 16 bits: discriminator.
2168 uint32_t LineOffset = Offset >> 16;
2169 uint32_t Discriminator = Offset & 0xffff;
2170
2171 InlineCallStack NewStack;
2172 NewStack.push_back(Elt: FProfile);
2173 llvm::append_range(C&: NewStack, R: InlineStack);
2174 if (Update) {
2175 // Walk up the inline stack, adding the samples on this line to
2176 // the total sample count of the callers in the chain.
2177 for (auto *CallerProfile : NewStack)
2178 CallerProfile->addTotalSamples(Num: Count);
2179
2180 // Update the body samples for the current profile.
2181 FProfile->addBodySamples(LineOffset, Discriminator, Num: Count);
2182 }
2183
2184 // Process the list of functions called at an indirect call site.
2185 // These are all the targets that a function pointer (or virtual
2186 // function) resolved at runtime.
2187 for (uint32_t J = 0; J < NumTargets; J++) {
2188 uint32_t HistVal;
2189 if (!GcovBuffer.readInt(Val&: HistVal))
2190 return sampleprof_error::truncated;
2191
2192 if (HistVal != HIST_TYPE_INDIR_CALL_TOPN)
2193 return sampleprof_error::malformed;
2194
2195 uint64_t TargetIdx;
2196 if (!GcovBuffer.readInt64(Val&: TargetIdx))
2197 return sampleprof_error::truncated;
2198 StringRef TargetName(Names[TargetIdx]);
2199
2200 uint64_t TargetCount;
2201 if (!GcovBuffer.readInt64(Val&: TargetCount))
2202 return sampleprof_error::truncated;
2203
2204 if (Update)
2205 FProfile->addCalledTargetSamples(LineOffset, Discriminator,
2206 Func: FunctionId(TargetName), Num: TargetCount);
2207 }
2208 }
2209
2210 // Process all the inlined callers into the current function. These
2211 // are all the callsites that were inlined into this function.
2212 for (uint32_t I = 0; I < NumCallsites; I++) {
2213 // The offset is encoded as:
2214 // high 16 bits: line offset to the start of the function.
2215 // low 16 bits: discriminator.
2216 uint32_t Offset;
2217 if (!GcovBuffer.readInt(Val&: Offset))
2218 return sampleprof_error::truncated;
2219 InlineCallStack NewStack;
2220 NewStack.push_back(Elt: FProfile);
2221 llvm::append_range(C&: NewStack, R: InlineStack);
2222 if (std::error_code EC = readOneFunctionProfile(InlineStack: NewStack, Update, Offset))
2223 return EC;
2224 }
2225
2226 return sampleprof_error::success;
2227}
2228
2229/// Read a GCC AutoFDO profile.
2230///
2231/// This format is generated by the Linux Perf conversion tool at
2232/// https://github.com/google/autofdo.
2233std::error_code SampleProfileReaderGCC::readImpl() {
2234 assert(!ProfileIsFSDisciminator && "Gcc profiles not support FSDisciminator");
2235 // Read the string table.
2236 if (std::error_code EC = readNameTable())
2237 return EC;
2238
2239 // Read the source profile.
2240 if (std::error_code EC = readFunctionProfiles())
2241 return EC;
2242
2243 return sampleprof_error::success;
2244}
2245
2246bool SampleProfileReaderGCC::hasFormat(const MemoryBuffer &Buffer) {
2247 StringRef Contents = Buffer.getBuffer();
2248 // Preserve exact magic matching, including a magic-only eight-byte buffer.
2249 return Contents.starts_with(Prefix: "adcg*704") &&
2250 (Contents.size() == 8 || Contents[8] == '\0');
2251}
2252
2253void SampleProfileReaderItaniumRemapper::applyRemapping(LLVMContext &Ctx) {
2254 // If the reader uses MD5 to represent string, we can't remap it because
2255 // we don't know what the original function names were.
2256 if (Reader.useMD5()) {
2257 Ctx.diagnose(DI: DiagnosticInfoSampleProfile(
2258 Reader.getBuffer()->getBufferIdentifier(),
2259 "Profile data remapping cannot be applied to profile data "
2260 "using MD5 names (original mangled names are not available).",
2261 DS_Warning));
2262 return;
2263 }
2264
2265 // CSSPGO-TODO: Remapper is not yet supported.
2266 // We will need to remap the entire context string.
2267 assert(Remappings && "should be initialized while creating remapper");
2268 for (auto &Sample : Reader.getProfiles()) {
2269 DenseSet<FunctionId> NamesInSample;
2270 Sample.second.findAllNames(NameSet&: NamesInSample);
2271 for (auto &Name : NamesInSample) {
2272 StringRef NameStr = Name.stringRef();
2273 if (auto Key = Remappings->insert(FunctionName: NameStr))
2274 NameMap.insert(KV: {Key, NameStr});
2275 }
2276 }
2277
2278 RemappingApplied = true;
2279}
2280
2281std::optional<StringRef>
2282SampleProfileReaderItaniumRemapper::lookUpNameInProfile(StringRef Fname) {
2283 if (auto Key = Remappings->lookup(FunctionName: Fname)) {
2284 StringRef Result = NameMap.lookup(Val: Key);
2285 if (!Result.empty())
2286 return Result;
2287 }
2288 return std::nullopt;
2289}
2290
2291/// Prepare a memory buffer for the contents of \p Filename.
2292///
2293/// \returns an error code indicating the status of the buffer.
2294static ErrorOr<std::unique_ptr<MemoryBuffer>>
2295setupMemoryBuffer(const Twine &Filename, vfs::FileSystem &FS) {
2296 auto BufferOrErr = Filename.str() == "-" ? MemoryBuffer::getSTDIN()
2297 : FS.getBufferForFile(Name: Filename);
2298 if (std::error_code EC = BufferOrErr.getError())
2299 return EC;
2300 auto Buffer = std::move(BufferOrErr.get());
2301
2302 return std::move(Buffer);
2303}
2304
2305/// Create a sample profile reader based on the format of the input file.
2306///
2307/// \param Filename The file to open.
2308///
2309/// \param C The LLVM context to use to emit diagnostics.
2310///
2311/// \param P The FSDiscriminatorPass.
2312///
2313/// \param RemapFilename The file used for profile remapping.
2314///
2315/// \returns an error code indicating the status of the created reader.
2316ErrorOr<std::unique_ptr<SampleProfileReader>>
2317SampleProfileReader::create(StringRef Filename, LLVMContext &C,
2318 vfs::FileSystem &FS, FSDiscriminatorPass P,
2319 StringRef RemapFilename) {
2320 auto BufferOrError = setupMemoryBuffer(Filename, FS);
2321 if (std::error_code EC = BufferOrError.getError())
2322 return EC;
2323 return create(B&: BufferOrError.get(), C, FS, P, RemapFilename);
2324}
2325
2326/// Create a sample profile remapper from the given input, to remap the
2327/// function names in the given profile data.
2328///
2329/// \param Filename The file to open.
2330///
2331/// \param Reader The profile reader the remapper is going to be applied to.
2332///
2333/// \param C The LLVM context to use to emit diagnostics.
2334///
2335/// \returns an error code indicating the status of the created reader.
2336ErrorOr<std::unique_ptr<SampleProfileReaderItaniumRemapper>>
2337SampleProfileReaderItaniumRemapper::create(StringRef Filename,
2338 vfs::FileSystem &FS,
2339 SampleProfileReader &Reader,
2340 LLVMContext &C) {
2341 auto BufferOrError = setupMemoryBuffer(Filename, FS);
2342 if (std::error_code EC = BufferOrError.getError())
2343 return EC;
2344 return create(B&: BufferOrError.get(), Reader, C);
2345}
2346
2347/// Create a sample profile remapper from the given input, to remap the
2348/// function names in the given profile data.
2349///
2350/// \param B The memory buffer to create the reader from (assumes ownership).
2351///
2352/// \param C The LLVM context to use to emit diagnostics.
2353///
2354/// \param Reader The profile reader the remapper is going to be applied to.
2355///
2356/// \returns an error code indicating the status of the created reader.
2357ErrorOr<std::unique_ptr<SampleProfileReaderItaniumRemapper>>
2358SampleProfileReaderItaniumRemapper::create(std::unique_ptr<MemoryBuffer> &B,
2359 SampleProfileReader &Reader,
2360 LLVMContext &C) {
2361 auto Remappings = std::make_unique<SymbolRemappingReader>();
2362 if (Error E = Remappings->read(B&: *B)) {
2363 handleAllErrors(
2364 E: std::move(E), Handlers: [&](const SymbolRemappingParseError &ParseError) {
2365 C.diagnose(DI: DiagnosticInfoSampleProfile(B->getBufferIdentifier(),
2366 ParseError.getLineNum(),
2367 ParseError.getMessage()));
2368 });
2369 return sampleprof_error::malformed;
2370 }
2371
2372 return std::make_unique<SampleProfileReaderItaniumRemapper>(
2373 args: std::move(B), args: std::move(Remappings), args&: Reader);
2374}
2375
2376/// Create a sample profile reader based on the format of the input data.
2377///
2378/// \param B The memory buffer to create the reader from (assumes ownership).
2379///
2380/// \param C The LLVM context to use to emit diagnostics.
2381///
2382/// \param P The FSDiscriminatorPass.
2383///
2384/// \param RemapFilename The file used for profile remapping.
2385///
2386/// \returns an error code indicating the status of the created reader.
2387ErrorOr<std::unique_ptr<SampleProfileReader>>
2388SampleProfileReader::create(std::unique_ptr<MemoryBuffer> &B, LLVMContext &C,
2389 vfs::FileSystem &FS, FSDiscriminatorPass P,
2390 StringRef RemapFilename) {
2391 std::unique_ptr<SampleProfileReader> Reader;
2392 if (SampleProfileReaderRawBinary::hasFormat(Buffer: *B))
2393 Reader.reset(p: new SampleProfileReaderRawBinary(std::move(B), C));
2394 else if (SampleProfileReaderExtBinary::hasFormat(Buffer: *B))
2395 Reader.reset(p: new SampleProfileReaderExtBinary(std::move(B), C));
2396 else if (SampleProfileReaderGCC::hasFormat(Buffer: *B))
2397 Reader.reset(p: new SampleProfileReaderGCC(std::move(B), C));
2398 else if (SampleProfileReaderText::hasFormat(Buffer: *B))
2399 Reader.reset(p: new SampleProfileReaderText(std::move(B), C));
2400 else
2401 return sampleprof_error::unrecognized_format;
2402
2403 if (!RemapFilename.empty()) {
2404 auto ReaderOrErr = SampleProfileReaderItaniumRemapper::create(
2405 Filename: RemapFilename, FS, Reader&: *Reader, C);
2406 if (std::error_code EC = ReaderOrErr.getError()) {
2407 std::string Msg = "Could not create remapper: " + EC.message();
2408 C.diagnose(DI: DiagnosticInfoSampleProfile(RemapFilename, Msg));
2409 return EC;
2410 }
2411 Reader->Remapper = std::move(ReaderOrErr.get());
2412 }
2413
2414 if (std::error_code EC = Reader->readHeader()) {
2415 return EC;
2416 }
2417
2418 Reader->setDiscriminatorMaskedBitFrom(P);
2419
2420 return std::move(Reader);
2421}
2422
2423// For text and GCC file formats, we compute the summary after reading the
2424// profile. Binary format has the profile summary in its header.
2425void SampleProfileReader::computeSummary() {
2426 SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs);
2427 Summary = Builder.computeSummaryForProfiles(Profiles);
2428}
2429