1//===- SampleProfReader.cpp - Read LLVM sample profile data ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the class that reads LLVM sample profiles. It
10// supports three file formats: text, binary and gcov.
11//
12// The textual representation is useful for debugging and testing purposes. The
13// binary representation is more compact, resulting in smaller file sizes.
14//
15// The gcov encoding is the one generated by GCC's AutoFDO profile creation
16// tool (https://github.com/google/autofdo)
17//
18// All three encodings can be used interchangeably as an input sample profile.
19//
20//===----------------------------------------------------------------------===//
21
22#include "llvm/ProfileData/SampleProfReader.h"
23#include "llvm/ADT/DenseMap.h"
24#include "llvm/ADT/STLExtras.h"
25#include "llvm/ADT/SmallSet.h"
26#include "llvm/ADT/StringRef.h"
27#include "llvm/IR/Module.h"
28#include "llvm/IR/ProfileSummary.h"
29#include "llvm/ProfileData/ProfileCommon.h"
30#include "llvm/ProfileData/SampleProf.h"
31#include "llvm/Support/CommandLine.h"
32#include "llvm/Support/Compression.h"
33#include "llvm/Support/ErrorOr.h"
34#include "llvm/Support/JSON.h"
35#include "llvm/Support/LEB128.h"
36#include "llvm/Support/LineIterator.h"
37#include "llvm/Support/MD5.h"
38#include "llvm/Support/MemoryBuffer.h"
39#include "llvm/Support/SaveAndRestore.h"
40#include "llvm/Support/VirtualFileSystem.h"
41#include "llvm/Support/raw_ostream.h"
42#include <algorithm>
43#include <cstddef>
44#include <cstdint>
45#include <cstring>
46#include <limits>
47#include <memory>
48#include <system_error>
49#include <vector>
50
51using namespace llvm;
52using namespace sampleprof;
53
54#define DEBUG_TYPE "samplepgo-reader"
55
56// This internal option specifies if the profile uses FS discriminators.
57// It only applies to text, and binary format profiles.
58// For ext-binary format profiles, the flag is set in the summary.
59static cl::opt<bool> ProfileIsFSDisciminator(
60 "profile-isfs", cl::Hidden, cl::init(Val: false),
61 cl::desc("Profile uses flow sensitive discriminators"));
62
63static cl::opt<bool>
64 LazyLoadNameTable("sample-profile-lazy-load-name-table", cl::init(Val: true),
65 cl::Hidden,
66 cl::desc("Lazy load the name table from the profile."));
67
68/// Dump the function profile for \p FName.
69///
70/// \param FContext Name + context of the function to print.
71/// \param OS Stream to emit the output to.
72void SampleProfileReader::dumpFunctionProfile(const FunctionSamples &FS,
73 raw_ostream &OS) {
74 OS << "Function: " << FS.getContext().toString() << ": " << FS;
75}
76
77/// Dump all the function profiles found on stream \p OS.
78void SampleProfileReader::dump(raw_ostream &OS) {
79 std::vector<NameFunctionSamples> V;
80 sortFuncProfiles(ProfileMap: Profiles, SortedProfiles&: V);
81 for (const auto &I : V)
82 dumpFunctionProfile(FS: *I.second, OS);
83}
84
85static void dumpFunctionProfileJson(const FunctionSamples &S,
86 json::OStream &JOS, bool TopLevel = false) {
87 auto DumpBody = [&](const BodySampleMap &BodySamples) {
88 for (const auto &I : BodySamples) {
89 const LineLocation &Loc = I.first;
90 const SampleRecord &Sample = I.second;
91 JOS.object(Contents: [&] {
92 JOS.attribute(Key: "line", Contents: Loc.LineOffset);
93 if (Loc.Discriminator)
94 JOS.attribute(Key: "discriminator", Contents: Loc.Discriminator);
95 JOS.attribute(Key: "samples", Contents: Sample.getSamples());
96
97 auto CallTargets = Sample.getSortedCallTargets();
98 if (!CallTargets.empty()) {
99 JOS.attributeArray(Key: "calls", Contents: [&] {
100 for (const auto &J : CallTargets) {
101 JOS.object(Contents: [&] {
102 JOS.attribute(Key: "function", Contents: J.first.str());
103 JOS.attribute(Key: "samples", Contents: J.second);
104 });
105 }
106 });
107 }
108 });
109 }
110 };
111
112 auto DumpCallsiteSamples = [&](const CallsiteSampleMap &CallsiteSamples) {
113 for (const auto &I : CallsiteSamples)
114 for (const auto &FS : I.second) {
115 const LineLocation &Loc = I.first;
116 const FunctionSamples &CalleeSamples = FS.second;
117 JOS.object(Contents: [&] {
118 JOS.attribute(Key: "line", Contents: Loc.LineOffset);
119 if (Loc.Discriminator)
120 JOS.attribute(Key: "discriminator", Contents: Loc.Discriminator);
121 JOS.attributeArray(
122 Key: "samples", Contents: [&] { dumpFunctionProfileJson(S: CalleeSamples, JOS); });
123 });
124 }
125 };
126
127 JOS.object(Contents: [&] {
128 JOS.attribute(Key: "name", Contents: S.getFunction().str());
129 JOS.attribute(Key: "total", Contents: S.getTotalSamples());
130 if (TopLevel)
131 JOS.attribute(Key: "head", Contents: S.getHeadSamples());
132
133 const auto &BodySamples = S.getBodySamples();
134 if (!BodySamples.empty())
135 JOS.attributeArray(Key: "body", Contents: [&] { DumpBody(BodySamples); });
136
137 const auto &CallsiteSamples = S.getCallsiteSamples();
138 if (!CallsiteSamples.empty())
139 JOS.attributeArray(Key: "callsites",
140 Contents: [&] { DumpCallsiteSamples(CallsiteSamples); });
141 });
142}
143
144/// Dump all the function profiles found on stream \p OS in the JSON format.
145void SampleProfileReader::dumpJson(raw_ostream &OS) {
146 std::vector<NameFunctionSamples> V;
147 sortFuncProfiles(ProfileMap: Profiles, SortedProfiles&: V);
148 json::OStream JOS(OS, 2);
149 JOS.arrayBegin();
150 for (const auto &F : V)
151 dumpFunctionProfileJson(S: *F.second, JOS, TopLevel: true);
152 JOS.arrayEnd();
153
154 // Emit a newline character at the end as json::OStream doesn't emit one.
155 OS << "\n";
156}
157
158/// Parse \p Input as function head.
159///
160/// Parse one line of \p Input, and update function name in \p FName,
161/// function's total sample count in \p NumSamples, function's entry
162/// count in \p NumHeadSamples.
163///
164/// \returns true if parsing is successful.
165static bool ParseHead(const StringRef &Input, StringRef &FName,
166 uint64_t &NumSamples, uint64_t &NumHeadSamples) {
167 if (Input[0] == ' ')
168 return false;
169 size_t n2 = Input.rfind(C: ':');
170 size_t n1 = Input.rfind(C: ':', From: n2 - 1);
171 FName = Input.substr(Start: 0, N: n1);
172 if (Input.substr(Start: n1 + 1, N: n2 - n1 - 1).getAsInteger(Radix: 10, Result&: NumSamples))
173 return false;
174 if (Input.substr(Start: n2 + 1).getAsInteger(Radix: 10, Result&: NumHeadSamples))
175 return false;
176 return true;
177}
178
179/// Returns true if line offset \p L is legal (only has 16 bits).
180static bool isOffsetLegal(unsigned L) { return (L & 0xffff) == L; }
181
182/// Parse \p Input that contains metadata.
183/// Possible metadata:
184/// - CFG Checksum information:
185/// !CFGChecksum: 12345
186/// - CFG Checksum information:
187/// !Attributes: 1
188/// Stores the FunctionHash (a.k.a. CFG Checksum) into \p FunctionHash.
189static bool parseMetadata(const StringRef &Input, uint64_t &FunctionHash,
190 uint32_t &Attributes) {
191 if (Input.starts_with(Prefix: "!CFGChecksum:")) {
192 StringRef CFGInfo = Input.substr(Start: strlen(s: "!CFGChecksum:")).trim();
193 return !CFGInfo.getAsInteger(Radix: 10, Result&: FunctionHash);
194 }
195
196 if (Input.starts_with(Prefix: "!Attributes:")) {
197 StringRef Attrib = Input.substr(Start: strlen(s: "!Attributes:")).trim();
198 return !Attrib.getAsInteger(Radix: 10, Result&: Attributes);
199 }
200
201 return false;
202}
203
204enum class LineType {
205 CallSiteProfile,
206 BodyProfile,
207 Metadata,
208 VirtualCallTypeProfile,
209};
210
211// Parse `Input` as a white-space separated list of `vtable:count` pairs. An
212// example input line is `_ZTVbar:1471 _ZTVfoo:630`.
213static bool parseTypeCountMap(StringRef Input,
214 DenseMap<StringRef, uint64_t> &TypeCountMap) {
215 for (size_t Index = Input.find_first_not_of(C: ' '); Index != StringRef::npos;) {
216 size_t ColonIndex = Input.find(C: ':', From: Index);
217 if (ColonIndex == StringRef::npos)
218 return false; // No colon found, invalid format.
219 StringRef TypeName = Input.substr(Start: Index, N: ColonIndex - Index);
220 // CountIndex is the start index of count.
221 size_t CountStartIndex = ColonIndex + 1;
222 // NextIndex is the start index after the 'target:count' pair.
223 size_t NextIndex = Input.find_first_of(C: ' ', From: CountStartIndex);
224 uint64_t Count;
225 if (Input.substr(Start: CountStartIndex, N: NextIndex - CountStartIndex)
226 .getAsInteger(Radix: 10, Result&: Count))
227 return false; // Invalid count.
228 // Error on duplicated type names in one line of input.
229 auto [Iter, Inserted] = TypeCountMap.insert(KV: {TypeName, Count});
230 if (!Inserted)
231 return false;
232 Index = (NextIndex == StringRef::npos)
233 ? StringRef::npos
234 : Input.find_first_not_of(C: ' ', From: NextIndex);
235 }
236 return true;
237}
238
239/// Parse \p Input as line sample.
240///
241/// \param Input input line.
242/// \param LineTy Type of this line.
243/// \param Depth the depth of the inline stack.
244/// \param NumSamples total samples of the line/inlined callsite.
245/// \param LineOffset line offset to the start of the function.
246/// \param Discriminator discriminator of the line.
247/// \param TargetCountMap map from indirect call target to count.
248/// \param FunctionHash the function's CFG hash, used by pseudo probe.
249///
250/// returns true if parsing is successful.
251static bool ParseLine(const StringRef &Input, LineType &LineTy, uint32_t &Depth,
252 uint64_t &NumSamples, uint32_t &LineOffset,
253 uint32_t &Discriminator, StringRef &CalleeName,
254 DenseMap<StringRef, uint64_t> &TargetCountMap,
255 DenseMap<StringRef, uint64_t> &TypeCountMap,
256 uint64_t &FunctionHash, uint32_t &Attributes,
257 bool &IsFlat) {
258 for (Depth = 0; Input[Depth] == ' '; Depth++)
259 ;
260 if (Depth == 0)
261 return false;
262
263 if (Input[Depth] == '!') {
264 LineTy = LineType::Metadata;
265 // This metadata is only for manual inspection only. We already created a
266 // FunctionSamples and put it in the profile map, so there is no point
267 // to skip profiles even they have no use for ThinLTO.
268 if (Input == StringRef(" !Flat")) {
269 IsFlat = true;
270 return true;
271 }
272 return parseMetadata(Input: Input.substr(Start: Depth), FunctionHash, Attributes);
273 }
274
275 size_t n1 = Input.find(C: ':');
276 StringRef Loc = Input.substr(Start: Depth, N: n1 - Depth);
277 size_t n2 = Loc.find(C: '.');
278 if (n2 == StringRef::npos) {
279 if (Loc.getAsInteger(Radix: 10, Result&: LineOffset) || !isOffsetLegal(L: LineOffset))
280 return false;
281 Discriminator = 0;
282 } else {
283 if (Loc.substr(Start: 0, N: n2).getAsInteger(Radix: 10, Result&: LineOffset))
284 return false;
285 if (Loc.substr(Start: n2 + 1).getAsInteger(Radix: 10, Result&: Discriminator))
286 return false;
287 }
288
289 StringRef Rest = Input.substr(Start: n1 + 2);
290 if (isDigit(C: Rest[0])) {
291 LineTy = LineType::BodyProfile;
292 size_t n3 = Rest.find(C: ' ');
293 if (n3 == StringRef::npos) {
294 if (Rest.getAsInteger(Radix: 10, Result&: NumSamples))
295 return false;
296 } else {
297 if (Rest.substr(Start: 0, N: n3).getAsInteger(Radix: 10, Result&: NumSamples))
298 return false;
299 }
300 // Find call targets and their sample counts.
301 // Note: In some cases, there are symbols in the profile which are not
302 // mangled. To accommodate such cases, use colon + integer pairs as the
303 // anchor points.
304 // An example:
305 // _M_construct<char *>:1000 string_view<std::allocator<char> >:437
306 // ":1000" and ":437" are used as anchor points so the string above will
307 // be interpreted as
308 // target: _M_construct<char *>
309 // count: 1000
310 // target: string_view<std::allocator<char> >
311 // count: 437
312 while (n3 != StringRef::npos) {
313 n3 += Rest.substr(Start: n3).find_first_not_of(C: ' ');
314 Rest = Rest.substr(Start: n3);
315 n3 = Rest.find_first_of(C: ':');
316 if (n3 == StringRef::npos || n3 == 0)
317 return false;
318
319 StringRef Target;
320 uint64_t count, n4;
321 while (true) {
322 // Get the segment after the current colon.
323 StringRef AfterColon = Rest.substr(Start: n3 + 1);
324 // Get the target symbol before the current colon.
325 Target = Rest.substr(Start: 0, N: n3);
326 // Check if the word after the current colon is an integer.
327 n4 = AfterColon.find_first_of(C: ' ');
328 n4 = (n4 != StringRef::npos) ? n3 + n4 + 1 : Rest.size();
329 StringRef WordAfterColon = Rest.substr(Start: n3 + 1, N: n4 - n3 - 1);
330 if (!WordAfterColon.getAsInteger(Radix: 10, Result&: count))
331 break;
332
333 // Try to find the next colon.
334 uint64_t n5 = AfterColon.find_first_of(C: ':');
335 if (n5 == StringRef::npos)
336 return false;
337 n3 += n5 + 1;
338 }
339
340 // An anchor point is found. Save the {target, count} pair
341 TargetCountMap[Target] = count;
342 if (n4 == Rest.size())
343 break;
344 // Change n3 to the next blank space after colon + integer pair.
345 n3 = n4;
346 }
347 } else if (Rest.starts_with(Prefix: kVTableProfPrefix)) {
348 LineTy = LineType::VirtualCallTypeProfile;
349 return parseTypeCountMap(Input: Rest.substr(Start: strlen(s: kVTableProfPrefix)),
350 TypeCountMap);
351 } else {
352 LineTy = LineType::CallSiteProfile;
353 size_t n3 = Rest.find_last_of(C: ':');
354 CalleeName = Rest.substr(Start: 0, N: n3);
355 if (Rest.substr(Start: n3 + 1).getAsInteger(Radix: 10, Result&: NumSamples))
356 return false;
357 }
358 return true;
359}
360
361/// Load samples from a text file.
362///
363/// See the documentation at the top of the file for an explanation of
364/// the expected format.
365///
366/// \returns true if the file was loaded successfully, false otherwise.
367std::error_code SampleProfileReaderText::readImpl() {
368 line_iterator LineIt(*Buffer, /*SkipBlanks=*/true, '#');
369 sampleprof_error Result = sampleprof_error::success;
370
371 InlineCallStack InlineStack;
372 uint32_t TopLevelProbeProfileCount = 0;
373
374 // DepthMetadata tracks whether we have processed metadata for the current
375 // top-level or nested function profile.
376 uint32_t DepthMetadata = 0;
377
378 std::vector<SampleContext *> FlatSamples;
379
380 ProfileIsFS = ProfileIsFSDisciminator;
381 FunctionSamples::ProfileIsFS = ProfileIsFS;
382 for (; !LineIt.is_at_eof(); ++LineIt) {
383 size_t pos = LineIt->find_first_not_of(C: ' ');
384 if (pos == LineIt->npos || (*LineIt)[pos] == '#')
385 continue;
386 // Read the header of each function.
387 //
388 // Note that for function identifiers we are actually expecting
389 // mangled names, but we may not always get them. This happens when
390 // the compiler decides not to emit the function (e.g., it was inlined
391 // and removed). In this case, the binary will not have the linkage
392 // name for the function, so the profiler will emit the function's
393 // unmangled name, which may contain characters like ':' and '>' in its
394 // name (member functions, templates, etc).
395 //
396 // The only requirement we place on the identifier, then, is that it
397 // should not begin with a number.
398 if ((*LineIt)[0] != ' ') {
399 uint64_t NumSamples, NumHeadSamples;
400 StringRef FName;
401 if (!ParseHead(Input: *LineIt, FName, NumSamples, NumHeadSamples)) {
402 reportError(LineNumber: LineIt.line_number(),
403 Msg: "Expected 'mangled_name:NUM:NUM', found " + *LineIt);
404 return sampleprof_error::malformed;
405 }
406 DepthMetadata = 0;
407 SampleContext FContext(FName, CSNameTable);
408 if (FContext.hasContext())
409 ++CSProfileCount;
410 FunctionSamples &FProfile = Profiles.create(Ctx: FContext);
411 mergeSampleProfErrors(Accumulator&: Result, Result: FProfile.addTotalSamples(Num: NumSamples));
412 mergeSampleProfErrors(Accumulator&: Result, Result: FProfile.addHeadSamples(Num: NumHeadSamples));
413 InlineStack.clear();
414 InlineStack.push_back(Elt: &FProfile);
415 } else {
416 uint64_t NumSamples;
417 StringRef FName;
418 DenseMap<StringRef, uint64_t> TargetCountMap;
419 DenseMap<StringRef, uint64_t> TypeCountMap;
420 uint32_t Depth, LineOffset, Discriminator;
421 LineType LineTy = LineType::BodyProfile;
422 uint64_t FunctionHash = 0;
423 uint32_t Attributes = 0;
424 bool IsFlat = false;
425 // TODO: Update ParseLine to return an error code instead of a bool and
426 // report it.
427 if (!ParseLine(Input: *LineIt, LineTy, Depth, NumSamples, LineOffset,
428 Discriminator, CalleeName&: FName, TargetCountMap, TypeCountMap,
429 FunctionHash, Attributes, IsFlat)) {
430 switch (LineTy) {
431 case LineType::Metadata:
432 reportError(LineNumber: LineIt.line_number(),
433 Msg: "Cannot parse metadata: " + *LineIt);
434 break;
435 case LineType::VirtualCallTypeProfile:
436 reportError(LineNumber: LineIt.line_number(),
437 Msg: "Expected 'vtables [mangled_vtable:NUM]+', found " +
438 *LineIt);
439 break;
440 default:
441 reportError(LineNumber: LineIt.line_number(),
442 Msg: "Expected 'NUM[.NUM]: NUM[ mangled_name:NUM]*', found " +
443 *LineIt);
444 }
445 return sampleprof_error::malformed;
446 }
447 if (LineTy != LineType::Metadata && Depth == DepthMetadata) {
448 // Metadata must be put at the end of a function profile.
449 reportError(LineNumber: LineIt.line_number(),
450 Msg: "Found non-metadata after metadata: " + *LineIt);
451 return sampleprof_error::malformed;
452 }
453
454 // Here we handle FS discriminators.
455 Discriminator &= getDiscriminatorMask();
456
457 while (InlineStack.size() > Depth) {
458 InlineStack.pop_back();
459 }
460 switch (LineTy) {
461 case LineType::CallSiteProfile: {
462 FunctionSamples &FSamples = InlineStack.back()->functionSamplesAt(
463 Loc: LineLocation(LineOffset, Discriminator))[FunctionId(FName)];
464 FSamples.setFunction(FunctionId(FName));
465 mergeSampleProfErrors(Accumulator&: Result, Result: FSamples.addTotalSamples(Num: NumSamples));
466 InlineStack.push_back(Elt: &FSamples);
467 DepthMetadata = 0;
468 break;
469 }
470
471 case LineType::VirtualCallTypeProfile: {
472 mergeSampleProfErrors(
473 Accumulator&: Result, Result: InlineStack.back()->addCallsiteVTableTypeProfAt(
474 Loc: LineLocation(LineOffset, Discriminator), Other: TypeCountMap));
475 break;
476 }
477
478 case LineType::BodyProfile: {
479 FunctionSamples &FProfile = *InlineStack.back();
480 for (const auto &name_count : TargetCountMap) {
481 mergeSampleProfErrors(Accumulator&: Result, Result: FProfile.addCalledTargetSamples(
482 LineOffset, Discriminator,
483 Func: FunctionId(name_count.first),
484 Num: name_count.second));
485 }
486 mergeSampleProfErrors(
487 Accumulator&: Result,
488 Result: FProfile.addBodySamples(LineOffset, Discriminator, Num: NumSamples));
489 break;
490 }
491 case LineType::Metadata: {
492 FunctionSamples &FProfile = *InlineStack.back();
493 if (FunctionHash) {
494 FProfile.setFunctionHash(FunctionHash);
495 if (Depth == 1)
496 ++TopLevelProbeProfileCount;
497 }
498 FProfile.getContext().setAllAttributes(Attributes);
499 if (Attributes & (uint32_t)ContextShouldBeInlined)
500 ProfileIsPreInlined = true;
501 DepthMetadata = Depth;
502 if (IsFlat) {
503 if (Depth == 1)
504 FlatSamples.push_back(x: &FProfile.getContext());
505 else
506 Ctx.diagnose(DI: DiagnosticInfoSampleProfile(
507 Buffer->getBufferIdentifier(), LineIt.line_number(),
508 "!Flat may only be used at top level function.", DS_Warning));
509 }
510 break;
511 }
512 }
513 }
514 }
515
516 // Honor the option to skip flat functions. Since they are already added to
517 // the profile map, remove them all here.
518 if (SkipFlatProf)
519 for (SampleContext *FlatSample : FlatSamples)
520 Profiles.erase(Ctx: *FlatSample);
521
522 assert((CSProfileCount == 0 || CSProfileCount == Profiles.size()) &&
523 "Cannot have both context-sensitive and regular profile");
524 ProfileIsCS = (CSProfileCount > 0);
525 assert((TopLevelProbeProfileCount == 0 ||
526 TopLevelProbeProfileCount == Profiles.size()) &&
527 "Cannot have both probe-based profiles and regular profiles");
528 ProfileIsProbeBased = (TopLevelProbeProfileCount > 0);
529 FunctionSamples::ProfileIsProbeBased = ProfileIsProbeBased;
530 FunctionSamples::ProfileIsCS = ProfileIsCS;
531 FunctionSamples::ProfileIsPreInlined = ProfileIsPreInlined;
532
533 if (Result == sampleprof_error::success)
534 computeSummary();
535
536 return Result;
537}
538
539bool SampleProfileReaderText::hasFormat(const MemoryBuffer &Buffer) {
540 bool result = false;
541
542 // Check that the first non-comment line is a valid function header.
543 line_iterator LineIt(Buffer, /*SkipBlanks=*/true, '#');
544 if (!LineIt.is_at_eof()) {
545 if ((*LineIt)[0] != ' ') {
546 uint64_t NumSamples, NumHeadSamples;
547 StringRef FName;
548 result = ParseHead(Input: *LineIt, FName, NumSamples, NumHeadSamples);
549 }
550 }
551
552 return result;
553}
554
555/// Emit a reader diagnostic for \p ProfError and return its error code.
556static std::error_code diagnoseReaderError(const SampleProfileReader &Reader,
557 sampleprof_error ProfError) {
558 std::error_code EC = ProfError;
559 Reader.reportError(LineNumber: 0, Msg: EC.message());
560 return EC;
561}
562
563template <typename T> ErrorOr<T> SampleProfileReaderBinary::readNumber() {
564 if (Data >= End)
565 return diagnoseReaderError(Reader: *this, ProfError: sampleprof_error::truncated);
566
567 unsigned NumBytesRead = 0;
568 ULEB128DecodeError DecodeError = ULEB128DecodeError::None;
569 uint64_t Val = decodeULEB128(p: Data, n: &NumBytesRead, end: End, error: nullptr, errorCode: &DecodeError);
570
571 // Preserve the distinction between incomplete input and an invalid value.
572 switch (DecodeError) {
573 case ULEB128DecodeError::None:
574 break;
575 case ULEB128DecodeError::UnexpectedEnd:
576 return diagnoseReaderError(Reader: *this, ProfError: sampleprof_error::truncated);
577 case ULEB128DecodeError::TooBig:
578 return diagnoseReaderError(Reader: *this, ProfError: sampleprof_error::malformed);
579 }
580
581 if (Val > std::numeric_limits<T>::max())
582 return diagnoseReaderError(Reader: *this, ProfError: sampleprof_error::malformed);
583
584 Data += NumBytesRead;
585 return static_cast<T>(Val);
586}
587
588ErrorOr<StringRef> SampleProfileReaderBinary::readString() {
589 if (Data >= End)
590 return diagnoseReaderError(Reader: *this, ProfError: sampleprof_error::truncated);
591
592 const auto *Terminator = static_cast<const uint8_t *>(
593 std::memchr(s: Data, c: 0, n: static_cast<size_t>(End - Data)));
594 if (!Terminator)
595 return diagnoseReaderError(Reader: *this, ProfError: sampleprof_error::truncated);
596
597 StringRef Str(reinterpret_cast<const char *>(Data), Terminator - Data);
598 Data = Terminator + 1;
599 return Str;
600}
601
602template <typename T>
603ErrorOr<T> SampleProfileReaderBinary::readUnencodedNumber() {
604 if (Data > End || static_cast<size_t>(End - Data) < sizeof(T))
605 return diagnoseReaderError(Reader: *this, ProfError: sampleprof_error::truncated);
606
607 using namespace support;
608 T Val = endian::readNext<T, llvm::endianness::little>(Data);
609 return Val;
610}
611
612template <typename T>
613inline ErrorOr<size_t> SampleProfileReaderBinary::readStringIndex(T &Table) {
614 auto Idx = readNumber<size_t>();
615 if (std::error_code EC = Idx.getError())
616 return EC;
617 if (*Idx >= Table.size())
618 return sampleprof_error::truncated_name_table;
619 return *Idx;
620}
621
622ErrorOr<FunctionId>
623SampleProfileReaderBinary::readStringFromTable(size_t *RetIdx) {
624 if (!NameTable)
625 return sampleprof_error::truncated_name_table;
626 auto Idx = readStringIndex(Table&: *NameTable);
627 if (std::error_code EC = Idx.getError())
628 return EC;
629 if (RetIdx)
630 *RetIdx = *Idx;
631 return (*NameTable)[*Idx];
632}
633
634ErrorOr<SampleContextFrames>
635SampleProfileReaderBinary::readContextFromTable(size_t *RetIdx) {
636 auto ContextIdx = readNumber<size_t>();
637 if (std::error_code EC = ContextIdx.getError())
638 return EC;
639 if (*ContextIdx >= CSNameTable.size())
640 return sampleprof_error::truncated_name_table;
641 if (RetIdx)
642 *RetIdx = *ContextIdx;
643 return CSNameTable[*ContextIdx];
644}
645
646ErrorOr<std::pair<SampleContext, uint64_t>>
647SampleProfileReaderBinary::readSampleContextFromTable() {
648 SampleContext Context;
649 size_t Idx;
650 if (ProfileIsCS) {
651 auto FContext(readContextFromTable(RetIdx: &Idx));
652 if (std::error_code EC = FContext.getError())
653 return EC;
654 Context = SampleContext(*FContext);
655 } else {
656 auto FName(readStringFromTable(RetIdx: &Idx));
657 if (std::error_code EC = FName.getError())
658 return EC;
659 Context = SampleContext(*FName);
660 }
661 // Since MD5SampleContextStart may point to the profile's file data, need to
662 // make sure it is reading the same value on big endian CPU.
663 uint64_t Hash = support::endian::read64le(P: MD5SampleContextStart + Idx);
664 // Lazy computing of hash value, write back to the table to cache it. Only
665 // compute the context's hash value if it is being referenced for the first
666 // time.
667 if (Hash == 0) {
668 assert(MD5SampleContextStart == MD5SampleContextTable.data());
669 Hash = Context.getHashCode();
670 support::endian::write64le(P: &MD5SampleContextTable[Idx], V: Hash);
671 }
672 return std::make_pair(x&: Context, y&: Hash);
673}
674
675std::error_code
676SampleProfileReaderBinary::readVTableTypeCountMap(TypeCountMap &M) {
677 auto NumVTableTypes = readNumber<uint32_t>();
678 if (std::error_code EC = NumVTableTypes.getError())
679 return EC;
680 // With FS discriminators, M may already hold the types of another raw
681 // location that maps to the same masked location (see below), so reserve
682 // for both the existing and the incoming vtables.
683 M.reserve(Cap: M.size() + *NumVTableTypes);
684
685 for (uint32_t I = 0; I < *NumVTableTypes; ++I) {
686 auto VTableType(readStringFromTable());
687 if (std::error_code EC = VTableType.getError())
688 return EC;
689
690 auto VTableSamples = readNumber<uint64_t>();
691 if (std::error_code EC = VTableSamples.getError())
692 return EC;
693
694 auto [It, Inserted] = M.try_emplace(Key: *VTableType, Args&: *VTableSamples);
695 if (Inserted)
696 continue;
697
698 // Without FS discriminators, the source profile should not have duplicate
699 // vtable records at the same location. Emit a warning about duplicate
700 // vtables and continue processing the profile.
701 if (!ProfileIsFS) {
702 Ctx.diagnose(DI: DiagnosticInfoSampleProfile(
703 Buffer->getBufferIdentifier(), 0,
704 "Duplicate vtable type " + VTableType->str() +
705 " at the same location. Additional counters will be ignored.",
706 DS_Warning));
707 continue;
708 }
709
710 // With flow-sensitive discriminators enabled in the profiled binary,
711 // several raw locations of the profiled instruction may map to the same
712 // masked location. Accumulate vtable counts for the same masked location,
713 // like body and callsite samples are accumulated.
714 It->second = SaturatingAdd(X: It->second, Y: *VTableSamples);
715 }
716 return sampleprof_error::success;
717}
718
719std::error_code
720SampleProfileReaderBinary::readCallsiteVTableProf(FunctionSamples &FProfile) {
721 assert(ReadVTableProf &&
722 "Cannot read vtable profiles if ReadVTableProf is false");
723
724 // Read the vtable type profile for the callsite.
725 auto NumCallsites = readNumber<uint32_t>();
726 if (std::error_code EC = NumCallsites.getError())
727 return EC;
728 FProfile.reserveCallsiteTypeCounts(NumEntries: *NumCallsites);
729
730 for (uint32_t I = 0; I < *NumCallsites; ++I) {
731 auto LineOffset = readNumber<uint64_t>();
732 if (std::error_code EC = LineOffset.getError())
733 return EC;
734
735 if (!isOffsetLegal(L: *LineOffset))
736 return sampleprof_error::illegal_line_offset;
737
738 auto Discriminator = readNumber<uint64_t>();
739 if (std::error_code EC = Discriminator.getError())
740 return EC;
741
742 // Here we handle FS discriminators:
743 const uint32_t DiscriminatorVal = (*Discriminator) & getDiscriminatorMask();
744
745 if (std::error_code EC = readVTableTypeCountMap(M&: FProfile.getTypeSamplesAt(
746 Loc: LineLocation(*LineOffset, DiscriminatorVal))))
747 return EC;
748 }
749 return sampleprof_error::success;
750}
751
752std::error_code
753SampleProfileReaderBinary::readLBRProfile(FunctionSamples &FProfile,
754 bool IsNested) {
755 if (ProfileSecRange.IsComposite && !IsNested) {
756 auto NumHeadSamples = readNumber<uint64_t>();
757 if (std::error_code EC = NumHeadSamples.getError())
758 return EC;
759 FProfile.addHeadSamples(Num: *NumHeadSamples);
760 }
761 auto NumSamples = readNumber<uint64_t>();
762 if (std::error_code EC = NumSamples.getError())
763 return EC;
764 FProfile.addTotalSamples(Num: *NumSamples);
765
766 // Read the samples in the body.
767 auto NumRecords = readNumber<uint32_t>();
768 if (std::error_code EC = NumRecords.getError())
769 return EC;
770 FProfile.reserveBodySamples(NumEntries: *NumRecords);
771
772 for (uint32_t I = 0; I < *NumRecords; ++I) {
773 auto LineOffset = readNumber<uint64_t>();
774 if (std::error_code EC = LineOffset.getError())
775 return EC;
776
777 if (!isOffsetLegal(L: *LineOffset)) {
778 return sampleprof_error::illegal_line_offset;
779 }
780
781 auto Discriminator = readNumber<uint64_t>();
782 if (std::error_code EC = Discriminator.getError())
783 return EC;
784
785 auto NumSamples = readNumber<uint64_t>();
786 if (std::error_code EC = NumSamples.getError())
787 return EC;
788
789 auto NumCalls = readNumber<uint32_t>();
790 if (std::error_code EC = NumCalls.getError())
791 return EC;
792
793 // Here we handle FS discriminators:
794 uint32_t DiscriminatorVal = (*Discriminator) & getDiscriminatorMask();
795
796 for (uint32_t J = 0; J < *NumCalls; ++J) {
797 auto CalledFunction(readStringFromTable());
798 if (std::error_code EC = CalledFunction.getError())
799 return EC;
800
801 auto CalledFunctionSamples = readNumber<uint64_t>();
802 if (std::error_code EC = CalledFunctionSamples.getError())
803 return EC;
804
805 FProfile.addCalledTargetSamples(LineOffset: *LineOffset, Discriminator: DiscriminatorVal,
806 Func: *CalledFunction, Num: *CalledFunctionSamples);
807 }
808
809 FProfile.addBodySamples(LineOffset: *LineOffset, Discriminator: DiscriminatorVal, Num: *NumSamples);
810 }
811
812 return sampleprof_error::success;
813}
814
815std::error_code
816SampleProfileReaderBinary::readCompositeProfile(FunctionSamples &FProfile,
817 bool IsNested) {
818 // Read the number of profile types.
819 auto ProfNum = readNumber<uint64_t>();
820 if (std::error_code EC = ProfNum.getError())
821 return EC;
822 if (ProfileTypeInfoOS)
823 *ProfileTypeInfoOS << (IsNested ? "Nested function: " : "Function: ")
824 << FProfile.getContext().toString()
825 << "\n Profile blocks: " << *ProfNum << "\n";
826
827 // Each type identifies one logical payload for the function. Decoding the
828 // same type twice would merge absolute counters from malformed input.
829 SmallSet<uint64_t, 4> SeenTypes;
830
831 // Read the specified number of composite profiles.
832 for (uint64_t I = 0; I < *ProfNum; ++I) {
833 auto Type = readNumber<uint64_t>();
834 if (std::error_code EC = Type.getError())
835 return EC;
836 // Report the conflicting ID so malformed profiles can be diagnosed
837 // without inspecting their binary encoding.
838 if (!SeenTypes.insert(V: *Type).second) {
839 reportError(LineNumber: 0, Msg: "Duplicate profile type ID: " + Twine(*Type));
840 return sampleprof_error::malformed;
841 }
842 auto Size = readNumber<uint64_t>();
843 if (std::error_code EC = Size.getError())
844 return EC;
845 if (ProfileTypeInfoOS)
846 *ProfileTypeInfoOS << " Type: " << *Type << " ("
847 << getProfTypeName(Type: *Type)
848 << "), Payload size: " << *Size << "\n";
849 const uint64_t RemainingSize = End - Data;
850 // Diagnose a size that would let the payload cross its containing section.
851 if (*Size > RemainingSize) {
852 reportError(LineNumber: 0, Msg: "Profile type ID " + Twine(*Type) +
853 " declares payload size " + Twine(*Size) +
854 ", but only " + Twine(RemainingSize) +
855 " bytes remain");
856 return sampleprof_error::truncated;
857 }
858
859 const uint8_t *PayloadEnd = Data + *Size;
860 std::error_code EC = sampleprof_error::success;
861 // Restrict field readers to the current payload so they reject fields that
862 // extend into the following payload.
863 SaveAndRestore<const uint8_t *> RestoreEnd(End, PayloadEnd);
864 switch (*Type) {
865 case ProfTypeLBR:
866 EC = readLBRProfile(FProfile, IsNested);
867 break;
868 default:
869 // Skip unknown profile types for forward compatibility.
870 HasUnknownProfileTypes = true;
871 Data = PayloadEnd;
872 break;
873 }
874
875 if (EC)
876 return EC;
877 // Reject trailing bytes because every known decoder must consume exactly
878 // the payload declared for its type.
879 if (Data != PayloadEnd) {
880 reportError(LineNumber: 0,
881 Msg: "Profile type ID " + Twine(*Type) +
882 " did not consume its complete payload; unread bytes: " +
883 Twine(PayloadEnd - Data));
884 return sampleprof_error::malformed;
885 }
886 }
887
888 return sampleprof_error::success;
889}
890
891std::error_code
892SampleProfileReaderBinary::readProfile(FunctionSamples &FProfile,
893 bool IsNested) {
894 if (ProfileSecRange.IsComposite) {
895 if (std::error_code EC = readCompositeProfile(FProfile, IsNested))
896 return EC;
897 } else {
898 if (std::error_code EC = readLBRProfile(FProfile, IsNested))
899 return EC;
900 }
901
902 // Read all the samples for inlined function calls.
903 auto NumCallsites = readNumber<uint32_t>();
904 if (std::error_code EC = NumCallsites.getError())
905 return EC;
906
907 for (uint32_t J = 0; J < *NumCallsites; ++J) {
908 auto LineOffset = readNumber<uint64_t>();
909 if (std::error_code EC = LineOffset.getError())
910 return EC;
911
912 auto Discriminator = readNumber<uint64_t>();
913 if (std::error_code EC = Discriminator.getError())
914 return EC;
915
916 auto FName(readStringFromTable());
917 if (std::error_code EC = FName.getError())
918 return EC;
919
920 // Here we handle FS discriminators:
921 uint32_t DiscriminatorVal = (*Discriminator) & getDiscriminatorMask();
922
923 FunctionSamples &CalleeProfile = FProfile.functionSamplesAt(
924 Loc: LineLocation(*LineOffset, DiscriminatorVal))[*FName];
925 CalleeProfile.setFunction(*FName);
926 if (std::error_code EC = readProfile(FProfile&: CalleeProfile, /*IsNested=*/true))
927 return EC;
928 }
929
930 if (ReadVTableProf)
931 return readCallsiteVTableProf(FProfile);
932
933 return sampleprof_error::success;
934}
935
936std::error_code
937SampleProfileReaderBinary::readFuncProfile(const uint8_t *Start,
938 SampleProfileMap &Profiles) {
939 Data = Start;
940 ErrorOr<uint64_t> NumHeadSamples = 0;
941 if (!ProfileSecRange.IsComposite) {
942 NumHeadSamples = readNumber<uint64_t>();
943 if (std::error_code EC = NumHeadSamples.getError())
944 return EC;
945 }
946 auto FContextHash(readSampleContextFromTable());
947 if (std::error_code EC = FContextHash.getError())
948 return EC;
949
950 auto &[FContext, Hash] = *FContextHash;
951 // Use the cached hash value for insertion instead of recalculating it.
952 auto Res = Profiles.try_emplace(Hash, Key: FContext, Args: FunctionSamples());
953 FunctionSamples &FProfile = Res.first->second;
954 FProfile.setContext(FContext);
955 if (!ProfileSecRange.IsComposite)
956 FProfile.addHeadSamples(Num: *NumHeadSamples);
957
958 if (FContext.hasContext())
959 CSProfileCount++;
960
961 if (std::error_code EC = readProfile(FProfile, /*IsNested=*/false))
962 return EC;
963 return sampleprof_error::success;
964}
965
966std::error_code
967SampleProfileReaderBinary::readFuncProfile(const uint8_t *Start) {
968 return readFuncProfile(Start, Profiles);
969}
970
971std::error_code SampleProfileReaderBinary::readImpl() {
972 ProfileIsFS = ProfileIsFSDisciminator;
973 FunctionSamples::ProfileIsFS = ProfileIsFS;
974 while (Data < End) {
975 if (std::error_code EC = readFuncProfile(Start: Data))
976 return EC;
977 }
978
979 return sampleprof_error::success;
980}
981
982std::error_code SampleProfileReaderExtBinaryBase::readOneSection(
983 const uint8_t *Start, uint64_t Size, const SecHdrTableEntry &Entry) {
984 Data = Start;
985 End = Start + Size;
986 switch (Entry.Type) {
987 case SecProfSummary:
988 if (std::error_code EC = readSummary())
989 return EC;
990 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagPartial))
991 Summary->setPartialProfile(true);
992 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagFullContext))
993 FunctionSamples::ProfileIsCS = ProfileIsCS = true;
994 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagIsPreInlined))
995 FunctionSamples::ProfileIsPreInlined = ProfileIsPreInlined = true;
996 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagFSDiscriminator))
997 FunctionSamples::ProfileIsFS = ProfileIsFS = true;
998 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagHasVTableTypeProf))
999 ReadVTableProf = true;
1000 break;
1001 case SecNameTable: {
1002 bool FixedLengthMD5 =
1003 hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagFixedLengthMD5);
1004 bool UseMD5 = hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagMD5Name);
1005 // UseMD5 means if THIS section uses MD5, ProfileIsMD5 means if the entire
1006 // profile uses MD5 for function name matching in IPO passes.
1007 ProfileIsMD5 = ProfileIsMD5 || UseMD5;
1008 FunctionSamples::HasUniqSuffix =
1009 hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagUniqSuffix);
1010 bool IsEytzinger = hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagEytzinger);
1011 if (std::error_code EC =
1012 readNameTableSec(IsMD5: UseMD5, FixedLengthMD5, IsEytzinger))
1013 return EC;
1014 break;
1015 }
1016 case SecCSNameTable: {
1017 if (std::error_code EC = readCSNameTableSec())
1018 return EC;
1019 break;
1020 }
1021 case SecLBRProfile:
1022 case SecCompositeProfile:
1023 // Retain the section and its encoding for subsequent on-demand reads.
1024 ProfileSecRange = {.Start: Data, .End: End, .IsComposite: Entry.Type == SecCompositeProfile};
1025 if (std::error_code EC = readFuncProfiles())
1026 return EC;
1027 break;
1028 case SecFuncOffsetTable:
1029 case SecCompositeFuncOffsetTable:
1030 // If module is absent, we are using LLVM tools, and need to read all
1031 // profiles, so skip reading the function offset table.
1032 if (!M) {
1033 Data = End;
1034 } else {
1035 bool IsEytzinger =
1036 hasSecFlag(Entry, Flag: SecFuncOffsetFlags::SecFlagEytzinger);
1037 bool IsFlat = hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagFlat);
1038 // An unflagged function offset table inherently indexes the primary
1039 // Nested symbol span.
1040 bool IsNested = !IsFlat;
1041 assert((!ProfileIsCS ||
1042 hasSecFlag(Entry, SecFuncOffsetFlags::SecFlagOrdered) ||
1043 IsEytzinger) &&
1044 "func offset table should always be sorted or in Eytzinger BFS "
1045 "order in CS profile");
1046 if (std::error_code EC = readFuncOffsetTable(IsEytzinger, IsNested))
1047 return EC;
1048 }
1049 break;
1050 case SecFuncMetadata: {
1051 ProfileIsProbeBased =
1052 hasSecFlag(Entry, Flag: SecFuncMetadataFlags::SecFlagIsProbeBased);
1053 FunctionSamples::ProfileIsProbeBased = ProfileIsProbeBased;
1054 ProfileHasAttribute =
1055 hasSecFlag(Entry, Flag: SecFuncMetadataFlags::SecFlagHasAttribute);
1056 if (std::error_code EC = readFuncMetadata())
1057 return EC;
1058 break;
1059 }
1060 case SecProfileSymbolList:
1061 if (std::error_code EC = readProfileSymbolList(
1062 IsMD5: hasSecFlag(Entry, Flag: SecProfileSymbolListFlags::SecFlagMD5)))
1063 return EC;
1064 break;
1065 default:
1066 if (std::error_code EC = readCustomSection(Entry))
1067 return EC;
1068 break;
1069 }
1070 return sampleprof_error::success;
1071}
1072
1073bool SampleProfileReaderExtBinaryBase::useFuncOffsetList() const {
1074 // If profile is CS, the function offset section is expected to consist of
1075 // sequences of contexts in pre-order layout
1076 // (e.g. [A, A:1 @ B, A:1 @ B:2.3 @ C] [D, D:1 @ E]), so that when a matched
1077 // context in the module is found, the profiles of all its callees are
1078 // recursively loaded. A list is needed since the order of profiles matters.
1079 if (ProfileIsCS)
1080 return true;
1081
1082 // If the profile is MD5, use the map container to lookup functions in
1083 // the module. A remapper has no use on MD5 names.
1084 if (useMD5())
1085 return false;
1086
1087 // Profile is not MD5 and if a remapper is present, the remapped name of
1088 // every function needed to be matched against the module, so use the list
1089 // container since each entry is accessed.
1090 if (Remapper)
1091 return true;
1092
1093 // Otherwise use the map container for faster lookup.
1094 // TODO: If the cardinality of the function offset section is much smaller
1095 // than the number of functions in the module, using the list container can
1096 // be always faster, but we need to figure out the constant factor to
1097 // determine the cutoff.
1098 return false;
1099}
1100
1101std::error_code
1102SampleProfileReaderExtBinaryBase::read(const DenseSet<StringRef> &FuncsToUse,
1103 SampleProfileMap &Profiles) {
1104 if (FuncsToUse.empty())
1105 return sampleprof_error::success;
1106
1107 Data = ProfileSecRange.Start;
1108 End = ProfileSecRange.End;
1109 if (std::error_code EC = readFuncProfiles(FuncsToUse, Profiles))
1110 return EC;
1111 End = Data;
1112 DenseSet<FunctionSamples *> ProfilesToReadMetadata;
1113 for (auto FName : FuncsToUse) {
1114 auto I = Profiles.find(Ctx: FName);
1115 if (I != Profiles.end())
1116 ProfilesToReadMetadata.insert(V: &I->second);
1117 }
1118
1119 if (std::error_code EC = readFuncMetadata(Profiles&: ProfilesToReadMetadata))
1120 return EC;
1121 return sampleprof_error::success;
1122}
1123
1124bool SampleProfileReaderExtBinaryBase::collectFuncsFromModule() {
1125 if (!M)
1126 return false;
1127 FuncsToUse.clear();
1128 for (auto &F : *M)
1129 FuncsToUse.insert(V: FunctionSamples::getCanonicalFnName(F));
1130 return true;
1131}
1132
1133std::error_code
1134SampleProfileReaderExtBinaryBase::readFuncOffsetTable(bool IsEytzinger,
1135 bool IsNested) {
1136 if (IsEytzinger)
1137 return readEytzingerFuncOffsetTable(IsNested);
1138 return readLegacyFuncOffsetTable();
1139}
1140
1141std::error_code
1142SampleProfileReaderExtBinaryBase::readEytzingerFuncOffsetTable(bool IsNested) {
1143 // If there are more than one function offset section, the profile associated
1144 // with the previous section has to be done reading before next one is read.
1145 FuncOffsetTable.reset();
1146
1147 size_t Size = End - Data;
1148 size_t SpanSize = NameTable->getEytzingerSpan(IsNested).size();
1149 if (Size != SpanSize * sizeof(uint32_t))
1150 return sampleprof_error::malformed;
1151
1152 auto *Array = reinterpret_cast<const support::ulittle32_t *>(Data);
1153 ArrayRef<support::ulittle32_t> Offsets(Array, SpanSize);
1154
1155 FuncOffsetTable.emplace(args: EytzingerMode, args: NameTable->getEytzingerSpan(IsNested),
1156 args&: Offsets);
1157
1158 Data = End;
1159 return sampleprof_error::success;
1160}
1161
1162std::error_code SampleProfileReaderExtBinaryBase::readLegacyFuncOffsetTable() {
1163 // If there are more than one function offset section, the profile associated
1164 // with the previous section has to be done reading before next one is read.
1165 FuncOffsetTable.reset();
1166 FuncOffsetList.clear();
1167
1168 auto Size = readNumber<uint64_t>();
1169 if (std::error_code EC = Size.getError())
1170 return EC;
1171
1172 bool UseFuncOffsetList = useFuncOffsetList();
1173 if (UseFuncOffsetList)
1174 FuncOffsetList.reserve(n: *Size);
1175 else
1176 FuncOffsetTable.emplace(args: InMemoryMode, args&: *Size);
1177
1178 for (uint64_t I = 0; I < *Size; ++I) {
1179 auto FContextHash(readSampleContextFromTable());
1180 if (std::error_code EC = FContextHash.getError())
1181 return EC;
1182
1183 auto &[FContext, Hash] = *FContextHash;
1184 auto Offset = readNumber<uint64_t>();
1185 if (std::error_code EC = Offset.getError())
1186 return EC;
1187
1188 if (UseFuncOffsetList)
1189 FuncOffsetList.emplace_back(args&: FContext, args&: *Offset);
1190 else
1191 // Because Porfiles replace existing value with new value if collision
1192 // happens, we also use the latest offset so that they are consistent.
1193 FuncOffsetTable->insert(GUID: Hash, Offset: *Offset);
1194 }
1195
1196 return sampleprof_error::success;
1197}
1198
1199std::error_code SampleProfileReaderExtBinaryBase::readFuncProfiles(
1200 const DenseSet<StringRef> &FuncsToUse, SampleProfileMap &Profiles) {
1201 const uint8_t *Start = Data;
1202
1203 if (Remapper) {
1204 for (auto Name : FuncsToUse) {
1205 Remapper->insert(FunctionName: Name);
1206 }
1207 }
1208
1209 if (FuncOffsetTable && FuncOffsetTable->isEytzinger() &&
1210 useFuncOffsetList()) {
1211 ArrayRef<support::ulittle32_t> Offsets = FuncOffsetTable->getFuncOffsets();
1212 if (Offsets.size() != FuncOffsetTable->getExpectedSize())
1213 return sampleprof_error::malformed;
1214 for (const auto &[LocalIdx, RelOffset] : llvm::enumerate(First&: Offsets)) {
1215 if (RelOffset == UINT32_MAX)
1216 continue;
1217 const uint8_t *FuncProfileAddr = Start + RelOffset;
1218 if (std::error_code EC = readFuncProfile(Start: FuncProfileAddr, Profiles))
1219 return EC;
1220 }
1221 return sampleprof_error::success;
1222 }
1223
1224 if (ProfileIsCS) {
1225 assert(useFuncOffsetList());
1226 DenseSet<uint64_t> FuncGuidsToUse;
1227 if (useMD5()) {
1228 for (auto Name : FuncsToUse)
1229 FuncGuidsToUse.insert(V: Function::getGUIDAssumingExternalLinkage(GlobalName: Name));
1230 }
1231
1232 // For each function in current module, load all context profiles for
1233 // the function as well as their callee contexts which can help profile
1234 // guided importing for ThinLTO. This can be achieved by walking
1235 // through an ordered context container, where contexts are laid out
1236 // as if they were walked in preorder of a context trie. While
1237 // traversing the trie, a link to the highest common ancestor node is
1238 // kept so that all of its decendants will be loaded.
1239 const SampleContext *CommonContext = nullptr;
1240 for (const auto &NameOffset : FuncOffsetList) {
1241 const auto &FContext = NameOffset.first;
1242 FunctionId FName = FContext.getFunction();
1243 StringRef FNameString;
1244 if (!useMD5())
1245 FNameString = FName.stringRef();
1246
1247 // For function in the current module, keep its farthest ancestor
1248 // context. This can be used to load itself and its child and
1249 // sibling contexts.
1250 if ((useMD5() && FuncGuidsToUse.count(V: FName.getHashCode())) ||
1251 (!useMD5() && (FuncsToUse.count(V: FNameString) ||
1252 (Remapper && Remapper->exist(FunctionName: FNameString))))) {
1253 if (!CommonContext || !CommonContext->isPrefixOf(That: FContext))
1254 CommonContext = &FContext;
1255 }
1256
1257 if (CommonContext == &FContext ||
1258 (CommonContext && CommonContext->isPrefixOf(That: FContext))) {
1259 // Load profile for the current context which originated from
1260 // the common ancestor.
1261 const uint8_t *FuncProfileAddr = Start + NameOffset.second;
1262 if (std::error_code EC = readFuncProfile(Start: FuncProfileAddr))
1263 return EC;
1264 }
1265 }
1266 } else if (useMD5()) {
1267 assert(!useFuncOffsetList());
1268 for (auto Name : FuncsToUse) {
1269 auto GUID = MD5Hash(Str: Name);
1270 if (auto Offset = FuncOffsetTable->lookup(GUID)) {
1271 const uint8_t *FuncProfileAddr = Start + *Offset;
1272 if (std::error_code EC = readFuncProfile(Start: FuncProfileAddr, Profiles))
1273 return EC;
1274 }
1275 }
1276 } else if (Remapper) {
1277 assert(useFuncOffsetList());
1278 for (auto NameOffset : FuncOffsetList) {
1279 SampleContext FContext(NameOffset.first);
1280 auto FuncName = FContext.getFunction();
1281 StringRef FuncNameStr = FuncName.stringRef();
1282 if (!FuncsToUse.count(V: FuncNameStr) && !Remapper->exist(FunctionName: FuncNameStr))
1283 continue;
1284 // This function may be called more than once on the same reader: once
1285 // for the module's functions and then incrementally (e.g. by
1286 // SampleProfileMatcher) for additional candidates. The remapper is
1287 // cumulative, so names inserted by an earlier call still match here.
1288 // Skip profiles that are already loaded; otherwise they would be read
1289 // again and merged into themselves.
1290 if (Profiles.find(Ctx: FContext) != Profiles.end())
1291 continue;
1292 const uint8_t *FuncProfileAddr = Start + NameOffset.second;
1293 if (std::error_code EC = readFuncProfile(Start: FuncProfileAddr, Profiles))
1294 return EC;
1295 }
1296 } else {
1297 assert(!useFuncOffsetList());
1298 for (auto Name : FuncsToUse) {
1299 if (auto Offset = FuncOffsetTable->lookup(GUID: MD5Hash(Str: Name))) {
1300 const uint8_t *FuncProfileAddr = Start + *Offset;
1301 if (std::error_code EC = readFuncProfile(Start: FuncProfileAddr, Profiles))
1302 return EC;
1303 }
1304 }
1305 }
1306
1307 return sampleprof_error::success;
1308}
1309
1310std::error_code SampleProfileReaderExtBinaryBase::readFuncProfiles() {
1311 // Collect functions used by current module if the Reader has been
1312 // given a module.
1313 // collectFuncsFromModule uses FunctionSamples::getCanonicalFnName
1314 // which will query FunctionSamples::HasUniqSuffix, so it has to be
1315 // called after FunctionSamples::HasUniqSuffix is set, i.e. after
1316 // NameTable section is read.
1317 bool LoadFuncsToBeUsed = collectFuncsFromModule();
1318
1319 // When LoadFuncsToBeUsed is false, we are using LLVM tool, need to read all
1320 // profiles.
1321 if (!LoadFuncsToBeUsed) {
1322 while (Data < End) {
1323 if (std::error_code EC = readFuncProfile(Start: Data))
1324 return EC;
1325 }
1326 assert(Data == End && "More data is read than expected");
1327 } else {
1328 // Load function profiles on demand.
1329 if (std::error_code EC = readFuncProfiles(FuncsToUse, Profiles))
1330 return EC;
1331 Data = End;
1332 }
1333 assert((CSProfileCount == 0 || CSProfileCount == Profiles.size()) &&
1334 "Cannot have both context-sensitive and regular profile");
1335 assert((!CSProfileCount || ProfileIsCS) &&
1336 "Section flag should be consistent with actual profile");
1337 return sampleprof_error::success;
1338}
1339
1340std::error_code
1341SampleProfileReaderExtBinaryBase::readProfileSymbolList(bool IsMD5) {
1342 if (IsMD5)
1343 return readMD5ProfileSymbolList();
1344 return readStringBasedProfileSymbolList();
1345}
1346
1347std::error_code SampleProfileReaderExtBinaryBase::readMD5ProfileSymbolList() {
1348 size_t Size = End - Data;
1349 if (Size % sizeof(uint64_t) != 0)
1350 return sampleprof_error::truncated;
1351 const auto *Table = reinterpret_cast<const support::ulittle64_t *>(Data);
1352 size_t NumEntries = Size / sizeof(uint64_t);
1353 if (!ProfSymList)
1354 ProfSymList = std::make_unique<ProfileSymbolList>();
1355 ProfSymList->setColdGUIDTable(
1356 EytzingerTableSpan<support::ulittle64_t>(Table, NumEntries));
1357 Data = End;
1358 return sampleprof_error::success;
1359}
1360
1361std::error_code
1362SampleProfileReaderExtBinaryBase::readStringBasedProfileSymbolList() {
1363 if (!ProfSymList)
1364 ProfSymList = std::make_unique<ProfileSymbolList>();
1365
1366 if (std::error_code EC = ProfSymList->read(Data, ListSize: End - Data))
1367 return EC;
1368
1369 Data = End;
1370 return sampleprof_error::success;
1371}
1372
1373std::error_code SampleProfileReaderExtBinaryBase::decompressSection(
1374 const uint8_t *SecStart, const uint64_t SecSize,
1375 const uint8_t *&DecompressBuf, uint64_t &DecompressBufSize) {
1376 Data = SecStart;
1377 End = SecStart + SecSize;
1378 auto DecompressSize = readNumber<uint64_t>();
1379 if (std::error_code EC = DecompressSize.getError())
1380 return EC;
1381 DecompressBufSize = *DecompressSize;
1382
1383 auto CompressSize = readNumber<uint64_t>();
1384 if (std::error_code EC = CompressSize.getError())
1385 return EC;
1386
1387 if (!llvm::compression::zlib::isAvailable())
1388 return sampleprof_error::zlib_unavailable;
1389
1390 uint8_t *Buffer = Allocator.Allocate<uint8_t>(Num: DecompressBufSize);
1391 size_t UCSize = DecompressBufSize;
1392 llvm::Error E = compression::zlib::decompress(Input: ArrayRef(Data, *CompressSize),
1393 Output: Buffer, UncompressedSize&: UCSize);
1394 if (E)
1395 return sampleprof_error::uncompress_failed;
1396 DecompressBuf = reinterpret_cast<const uint8_t *>(Buffer);
1397 return sampleprof_error::success;
1398}
1399
1400std::error_code SampleProfileReaderExtBinaryBase::readImpl() {
1401 const uint8_t *BufStart =
1402 reinterpret_cast<const uint8_t *>(Buffer->getBufferStart());
1403
1404 for (auto &Entry : SecHdrTable) {
1405 // Skip empty section.
1406 if (!Entry.Size)
1407 continue;
1408
1409 // Skip sections without inlined functions when SkipFlatProf is true.
1410 if (SkipFlatProf && hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagFlat))
1411 continue;
1412
1413 const uint8_t *SecStart = BufStart + Entry.Offset;
1414 uint64_t SecSize = Entry.Size;
1415
1416 // If the section is compressed, decompress it into a buffer
1417 // DecompressBuf before reading the actual data. The pointee of
1418 // 'Data' will be changed to buffer hold by DecompressBuf
1419 // temporarily when reading the actual data.
1420 bool isCompressed = hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagCompress);
1421 if (isCompressed) {
1422 const uint8_t *DecompressBuf;
1423 uint64_t DecompressBufSize;
1424 if (std::error_code EC = decompressSection(
1425 SecStart, SecSize, DecompressBuf, DecompressBufSize))
1426 return EC;
1427 SecStart = DecompressBuf;
1428 SecSize = DecompressBufSize;
1429 }
1430
1431 if (std::error_code EC = readOneSection(Start: SecStart, Size: SecSize, Entry))
1432 return EC;
1433 if (Data != SecStart + SecSize)
1434 return sampleprof_error::malformed;
1435
1436 // Change the pointee of 'Data' from DecompressBuf to original Buffer.
1437 if (isCompressed) {
1438 Data = BufStart + Entry.Offset;
1439 End = BufStart + Buffer->getBufferSize();
1440 }
1441 }
1442
1443 return sampleprof_error::success;
1444}
1445
1446std::error_code SampleProfileReaderRawBinary::verifySPMagic(uint64_t Magic) {
1447 if (Magic == SPMagic())
1448 return sampleprof_error::success;
1449 return sampleprof_error::bad_magic;
1450}
1451
1452std::error_code SampleProfileReaderExtBinary::verifySPMagic(uint64_t Magic) {
1453 if (Magic == SPMagic(Format: SPF_Ext_Binary))
1454 return sampleprof_error::success;
1455 return sampleprof_error::bad_magic;
1456}
1457
1458std::error_code SampleProfileReaderBinary::readNameTable() {
1459 auto Size = readNumber<size_t>();
1460 if (std::error_code EC = Size.getError())
1461 return EC;
1462
1463 // Normally if useMD5 is true, the name table should have MD5 values, not
1464 // strings, however in the case that ExtBinary profile has multiple name
1465 // tables mixing string and MD5, all of them have to be normalized to use MD5,
1466 // because optimization passes can only handle either type.
1467 bool UseMD5 = useMD5();
1468
1469 std::vector<FunctionId> TableVec;
1470 TableVec.reserve(n: *Size);
1471 if (!ProfileIsCS) {
1472 MD5SampleContextTable.clear();
1473 if (UseMD5)
1474 MD5SampleContextTable.reserve(n: *Size);
1475 else
1476 // If we are using strings, delay MD5 computation since only a portion of
1477 // names are used by top level functions. Use 0 to indicate MD5 value is
1478 // to be calculated as no known string has a MD5 value of 0.
1479 MD5SampleContextTable.resize(new_size: *Size);
1480 }
1481 for (size_t I = 0; I < *Size; ++I) {
1482 auto Name(readString());
1483 if (std::error_code EC = Name.getError())
1484 return EC;
1485 if (UseMD5) {
1486 FunctionId FID(*Name);
1487 if (!ProfileIsCS)
1488 MD5SampleContextTable.emplace_back(args: FID.getHashCode());
1489 TableVec.emplace_back(args&: FID);
1490 } else
1491 TableVec.push_back(x: FunctionId(*Name));
1492 }
1493 if (!ProfileIsCS)
1494 MD5SampleContextStart = MD5SampleContextTable.data();
1495 if (UseMD5)
1496 NameTable =
1497 std::make_unique<MD5SampleProfileNameTable>(args: std::move(TableVec));
1498 else
1499 NameTable =
1500 std::make_unique<StringSampleProfileNameTable>(args: std::move(TableVec));
1501 return sampleprof_error::success;
1502}
1503
1504std::error_code SampleProfileReaderExtBinaryBase::readNameTableSec(
1505 bool IsMD5, bool FixedLengthMD5, bool IsEytzinger) {
1506 if (IsEytzinger)
1507 return readNameTableSecEytzinger(IsMD5, FixedLengthMD5);
1508 return readNameTableSecLegacy(IsMD5, FixedLengthMD5);
1509}
1510
1511// Read the Eytzinger layout for SecNameTable from an ExtBinary MD5 profile.
1512//
1513// The section consists of three sequential ULEB128 symbol counts (Nested, Flat,
1514// and Inlinees) followed by their corresponding arrays of 64-bit MD5 hash keys
1515// laid out in Eytzinger order.
1516std::error_code SampleProfileReaderExtBinaryBase::readNameTableSecEytzinger(
1517 bool IsMD5, bool FixedLengthMD5) {
1518 assert(IsMD5 && "Eytzinger name tables require MD5 representation");
1519 if (!IsMD5)
1520 return sampleprof_error::malformed;
1521
1522 // Read the table sizes for Nested, flat, and inlinee symbols.
1523 std::array<uint64_t, static_cast<size_t>(EytzingerSpan::NumSpans)> Counts;
1524 for (uint64_t &Count : Counts) {
1525 auto ValOrErr = readNumber<uint64_t>();
1526 if (std::error_code EC = ValOrErr.getError())
1527 return EC;
1528 Count = *ValOrErr;
1529 }
1530 auto [NumNested, NumFlat, NumInlinees] = Counts;
1531
1532 // Guard against unsigned overflow in total entry computation.
1533 if (NumNested > std::numeric_limits<uint32_t>::max() ||
1534 NumFlat > std::numeric_limits<uint32_t>::max() ||
1535 NumInlinees > std::numeric_limits<uint32_t>::max())
1536 return sampleprof_error::malformed;
1537
1538 uint64_t TotalEntries = NumNested + NumFlat + NumInlinees;
1539 if (static_cast<size_t>(End - Data) < TotalEntries * sizeof(uint64_t))
1540 return sampleprof_error::truncated;
1541
1542 NameTable = std::make_unique<EytzingerSampleProfileNameTable>(
1543 args: reinterpret_cast<const support::ulittle64_t *>(Data), args&: NumNested, args&: NumFlat,
1544 args&: NumInlinees);
1545
1546 if (!ProfileIsCS)
1547 MD5SampleContextStart = reinterpret_cast<const uint64_t *>(Data);
1548 Data = Data + TotalEntries * sizeof(uint64_t);
1549 return sampleprof_error::success;
1550}
1551
1552std::error_code
1553SampleProfileReaderExtBinaryBase::readNameTableSecLegacy(bool IsMD5,
1554 bool FixedLengthMD5) {
1555 if (FixedLengthMD5) {
1556 if (!IsMD5)
1557 errs() << "If FixedLengthMD5 is true, UseMD5 has to be true";
1558 auto Size = readNumber<size_t>();
1559 if (std::error_code EC = Size.getError())
1560 return EC;
1561
1562 assert(Data + (*Size) * sizeof(uint64_t) == End &&
1563 "Fixed length MD5 name table does not contain specified number of "
1564 "entries");
1565 if (Data + (*Size) * sizeof(uint64_t) > End)
1566 return sampleprof_error::truncated;
1567
1568 if (LazyLoadNameTable) {
1569 NameTable = std::make_unique<LazySampleProfileNameTable>(args&: Data, args&: *Size);
1570 } else {
1571 std::vector<FunctionId> TableVec;
1572 TableVec.reserve(n: *Size);
1573 for (size_t I = 0; I < *Size; ++I) {
1574 using namespace support;
1575 uint64_t FID = endian::read<uint64_t, unaligned>(
1576 memory: Data + I * sizeof(uint64_t), endian: endianness::little);
1577 TableVec.emplace_back(args: FunctionId(FID));
1578 }
1579 NameTable =
1580 std::make_unique<MD5SampleProfileNameTable>(args: std::move(TableVec));
1581 }
1582 if (!ProfileIsCS)
1583 MD5SampleContextStart = reinterpret_cast<const uint64_t *>(Data);
1584 Data = Data + (*Size) * sizeof(uint64_t);
1585 return sampleprof_error::success;
1586 }
1587
1588 if (IsMD5) {
1589 assert(!FixedLengthMD5 && "FixedLengthMD5 should be unreachable here");
1590 auto Size = readNumber<size_t>();
1591 if (std::error_code EC = Size.getError())
1592 return EC;
1593
1594 std::vector<FunctionId> TableVec;
1595 TableVec.reserve(n: *Size);
1596 if (!ProfileIsCS)
1597 MD5SampleContextTable.resize(new_size: *Size);
1598 for (size_t I = 0; I < *Size; ++I) {
1599 auto FID = readNumber<uint64_t>();
1600 if (std::error_code EC = FID.getError())
1601 return EC;
1602 if (!ProfileIsCS)
1603 support::endian::write64le(P: &MD5SampleContextTable[I], V: *FID);
1604 TableVec.emplace_back(args: FunctionId(*FID));
1605 }
1606 if (!ProfileIsCS)
1607 MD5SampleContextStart = MD5SampleContextTable.data();
1608 NameTable =
1609 std::make_unique<MD5SampleProfileNameTable>(args: std::move(TableVec));
1610 return sampleprof_error::success;
1611 }
1612
1613 return SampleProfileReaderBinary::readNameTable();
1614}
1615
1616// Read in the CS name table section, which basically contains a list of context
1617// vectors. Each element of a context vector, aka a frame, refers to the
1618// underlying raw function names that are stored in the name table, as well as
1619// a callsite identifier that only makes sense for non-leaf frames.
1620std::error_code SampleProfileReaderExtBinaryBase::readCSNameTableSec() {
1621 auto Size = readNumber<size_t>();
1622 if (std::error_code EC = Size.getError())
1623 return EC;
1624
1625 CSNameTable.clear();
1626 CSNameTable.reserve(n: *Size);
1627 if (ProfileIsCS) {
1628 // Delay MD5 computation of CS context until they are needed. Use 0 to
1629 // indicate MD5 value is to be calculated as no known string has a MD5
1630 // value of 0.
1631 MD5SampleContextTable.clear();
1632 MD5SampleContextTable.resize(new_size: *Size);
1633 MD5SampleContextStart = MD5SampleContextTable.data();
1634 }
1635 for (size_t I = 0; I < *Size; ++I) {
1636 CSNameTable.emplace_back(args: SampleContextFrameVector());
1637 auto ContextSize = readNumber<uint32_t>();
1638 if (std::error_code EC = ContextSize.getError())
1639 return EC;
1640 for (uint32_t J = 0; J < *ContextSize; ++J) {
1641 auto FName(readStringFromTable());
1642 if (std::error_code EC = FName.getError())
1643 return EC;
1644 auto LineOffset = readNumber<uint64_t>();
1645 if (std::error_code EC = LineOffset.getError())
1646 return EC;
1647
1648 if (!isOffsetLegal(L: *LineOffset))
1649 return sampleprof_error::illegal_line_offset;
1650
1651 auto Discriminator = readNumber<uint64_t>();
1652 if (std::error_code EC = Discriminator.getError())
1653 return EC;
1654
1655 CSNameTable.back().emplace_back(
1656 Args&: FName.get(), Args: LineLocation(LineOffset.get(), Discriminator.get()));
1657 }
1658 }
1659
1660 return sampleprof_error::success;
1661}
1662
1663std::error_code
1664SampleProfileReaderExtBinaryBase::readFuncMetadata(FunctionSamples *FProfile) {
1665 if (Data < End) {
1666 if (ProfileIsProbeBased) {
1667 auto Checksum = readNumber<uint64_t>();
1668 if (std::error_code EC = Checksum.getError())
1669 return EC;
1670 if (FProfile)
1671 FProfile->setFunctionHash(*Checksum);
1672 }
1673
1674 if (ProfileHasAttribute) {
1675 auto Attributes = readNumber<uint32_t>();
1676 if (std::error_code EC = Attributes.getError())
1677 return EC;
1678 if (FProfile)
1679 FProfile->getContext().setAllAttributes(*Attributes);
1680 }
1681
1682 if (!ProfileIsCS) {
1683 // Read all the attributes for inlined function calls.
1684 auto NumCallsites = readNumber<uint32_t>();
1685 if (std::error_code EC = NumCallsites.getError())
1686 return EC;
1687
1688 for (uint32_t J = 0; J < *NumCallsites; ++J) {
1689 auto LineOffset = readNumber<uint64_t>();
1690 if (std::error_code EC = LineOffset.getError())
1691 return EC;
1692
1693 auto Discriminator = readNumber<uint64_t>();
1694 if (std::error_code EC = Discriminator.getError())
1695 return EC;
1696
1697 auto FContextHash(readSampleContextFromTable());
1698 if (std::error_code EC = FContextHash.getError())
1699 return EC;
1700
1701 auto &[FContext, Hash] = *FContextHash;
1702 FunctionSamples *CalleeProfile = nullptr;
1703 if (FProfile) {
1704 CalleeProfile = const_cast<FunctionSamples *>(
1705 &FProfile->functionSamplesAt(Loc: LineLocation(
1706 *LineOffset, *Discriminator))[FContext.getFunction()]);
1707 }
1708 if (std::error_code EC = readFuncMetadata(FProfile: CalleeProfile))
1709 return EC;
1710 }
1711 }
1712 }
1713
1714 return sampleprof_error::success;
1715}
1716
1717std::error_code SampleProfileReaderExtBinaryBase::readFuncMetadata(
1718 DenseSet<FunctionSamples *> &Profiles) {
1719 if (FuncMetadataIndex.empty())
1720 return sampleprof_error::success;
1721
1722 for (auto *FProfile : Profiles) {
1723 auto R = FuncMetadataIndex.find(Val: FProfile->getContext().getHashCode());
1724 if (R == FuncMetadataIndex.end())
1725 continue;
1726
1727 Data = R->second.first;
1728 End = R->second.second;
1729 if (std::error_code EC = readFuncMetadata(FProfile))
1730 return EC;
1731 assert(Data == End && "More data is read than expected");
1732 }
1733 return sampleprof_error::success;
1734}
1735
1736std::error_code SampleProfileReaderExtBinaryBase::readFuncMetadata() {
1737 while (Data < End) {
1738 auto FContextHash(readSampleContextFromTable());
1739 if (std::error_code EC = FContextHash.getError())
1740 return EC;
1741 auto &[FContext, Hash] = *FContextHash;
1742 FunctionSamples *FProfile = nullptr;
1743 auto It = Profiles.find(Ctx: FContext);
1744 if (It != Profiles.end())
1745 FProfile = &It->second;
1746
1747 const uint8_t *Start = Data;
1748 if (std::error_code EC = readFuncMetadata(FProfile))
1749 return EC;
1750
1751 FuncMetadataIndex[FContext.getHashCode()] = {Start, Data};
1752 }
1753
1754 assert(Data == End && "More data is read than expected");
1755 return sampleprof_error::success;
1756}
1757
1758std::error_code
1759SampleProfileReaderExtBinaryBase::readSecHdrTableEntry(uint64_t Idx) {
1760 SecHdrTableEntry Entry;
1761 auto Type = readUnencodedNumber<uint64_t>();
1762 if (std::error_code EC = Type.getError())
1763 return EC;
1764 Entry.Type = static_cast<SecType>(*Type);
1765
1766 // Reject a section whose encoding is newer than the declared file version.
1767 if ((Entry.Type == SecCompositeProfile ||
1768 Entry.Type == SecCompositeFuncOffsetTable) &&
1769 FormatVersion < CompositeProfileVersion)
1770 return sampleprof_error::unsupported_version;
1771
1772 auto Flags = readUnencodedNumber<uint64_t>();
1773 if (std::error_code EC = Flags.getError())
1774 return EC;
1775 Entry.Flags = *Flags;
1776
1777 auto Offset = readUnencodedNumber<uint64_t>();
1778 if (std::error_code EC = Offset.getError())
1779 return EC;
1780 Entry.Offset = *Offset;
1781
1782 auto Size = readUnencodedNumber<uint64_t>();
1783 if (std::error_code EC = Size.getError())
1784 return EC;
1785 Entry.Size = *Size;
1786
1787 Entry.LayoutIndex = Idx;
1788 SecHdrTable.push_back(x: std::move(Entry));
1789 return sampleprof_error::success;
1790}
1791
1792std::error_code SampleProfileReaderExtBinaryBase::readSecHdrTable() {
1793 auto EntryNum = readUnencodedNumber<uint64_t>();
1794 if (std::error_code EC = EntryNum.getError())
1795 return EC;
1796
1797 for (uint64_t i = 0; i < (*EntryNum); i++)
1798 if (std::error_code EC = readSecHdrTableEntry(Idx: i))
1799 return EC;
1800
1801 return sampleprof_error::success;
1802}
1803
1804std::error_code SampleProfileReaderExtBinaryBase::readHeader() {
1805 const uint8_t *BufStart =
1806 reinterpret_cast<const uint8_t *>(Buffer->getBufferStart());
1807 Data = BufStart;
1808 End = BufStart + Buffer->getBufferSize();
1809
1810 if (std::error_code EC = readMagicIdent())
1811 return EC;
1812
1813 if (std::error_code EC = readSecHdrTable())
1814 return EC;
1815
1816 return sampleprof_error::success;
1817}
1818
1819uint64_t SampleProfileReaderExtBinaryBase::getSectionSize(SecType Type) {
1820 uint64_t Size = 0;
1821 for (auto &Entry : SecHdrTable) {
1822 if (Entry.Type == Type)
1823 Size += Entry.Size;
1824 }
1825 return Size;
1826}
1827
1828uint64_t SampleProfileReaderExtBinaryBase::getFileSize() {
1829 // Sections in SecHdrTable is not necessarily in the same order as
1830 // sections in the profile because section like FuncOffsetTable needs
1831 // to be written after section LBRProfile but needs to be read before
1832 // section LBRProfile, so we cannot simply use the last entry in
1833 // SecHdrTable to calculate the file size.
1834 uint64_t FileSize = 0;
1835 for (auto &Entry : SecHdrTable) {
1836 FileSize = std::max(a: Entry.Offset + Entry.Size, b: FileSize);
1837 }
1838 return FileSize;
1839}
1840
1841static std::string getSecFlagsStr(const SecHdrTableEntry &Entry) {
1842 std::string Flags;
1843 if (hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagCompress))
1844 Flags.append(s: "{compressed,");
1845 else
1846 Flags.append(s: "{");
1847
1848 if (hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagFlat))
1849 Flags.append(s: "flat,");
1850
1851 switch (Entry.Type) {
1852 case SecNameTable:
1853 if (hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagEytzinger))
1854 Flags.append(s: "eytzinger,");
1855 if (hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagFixedLengthMD5))
1856 Flags.append(s: "fixlenmd5,");
1857 else if (hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagMD5Name))
1858 Flags.append(s: "md5,");
1859 if (hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagUniqSuffix))
1860 Flags.append(s: "uniq,");
1861 break;
1862 case SecProfSummary:
1863 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagPartial))
1864 Flags.append(s: "partial,");
1865 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagFullContext))
1866 Flags.append(s: "context,");
1867 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagIsPreInlined))
1868 Flags.append(s: "preInlined,");
1869 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagFSDiscriminator))
1870 Flags.append(s: "fs-discriminator,");
1871 break;
1872 case SecFuncOffsetTable:
1873 case SecCompositeFuncOffsetTable:
1874 if (hasSecFlag(Entry, Flag: SecFuncOffsetFlags::SecFlagOrdered))
1875 Flags.append(s: "ordered,");
1876 if (hasSecFlag(Entry, Flag: SecFuncOffsetFlags::SecFlagEytzinger))
1877 Flags.append(s: "eytzinger,");
1878 break;
1879 case SecFuncMetadata:
1880 if (hasSecFlag(Entry, Flag: SecFuncMetadataFlags::SecFlagIsProbeBased))
1881 Flags.append(s: "probe,");
1882 if (hasSecFlag(Entry, Flag: SecFuncMetadataFlags::SecFlagHasAttribute))
1883 Flags.append(s: "attr,");
1884 break;
1885 case SecProfileSymbolList:
1886 if (hasSecFlag(Entry, Flag: SecProfileSymbolListFlags::SecFlagMD5))
1887 Flags.append(s: "md5,");
1888 break;
1889 default:
1890 break;
1891 }
1892 char &last = Flags.back();
1893 if (last == ',')
1894 last = '}';
1895 else
1896 Flags.append(s: "}");
1897 return Flags;
1898}
1899
1900bool SampleProfileReaderExtBinaryBase::dumpSectionInfo(raw_ostream &OS) {
1901 uint64_t TotalSecsSize = 0;
1902 for (auto &Entry : SecHdrTable) {
1903 OS << getSecName(Type: Entry.Type) << " - Offset: " << Entry.Offset
1904 << ", Size: " << Entry.Size << ", Flags: " << getSecFlagsStr(Entry)
1905 << "\n";
1906 ;
1907 TotalSecsSize += Entry.Size;
1908 }
1909 uint64_t HeaderSize = SecHdrTable.front().Offset;
1910 assert(HeaderSize + TotalSecsSize == getFileSize() &&
1911 "Size of 'header + sections' doesn't match the total size of profile");
1912
1913 OS << "Header Size: " << HeaderSize << "\n";
1914 OS << "Total Sections Size: " << TotalSecsSize << "\n";
1915 OS << "File Size: " << getFileSize() << "\n";
1916 return true;
1917}
1918
1919std::error_code SampleProfileReaderBinary::readMagicIdent() {
1920 // Read and check the magic identifier.
1921 auto Magic = readNumber<uint64_t>();
1922 if (std::error_code EC = Magic.getError())
1923 return EC;
1924 else if (std::error_code EC = verifySPMagic(Magic: *Magic))
1925 return EC;
1926
1927 // Read the version number.
1928 auto Version = readNumber<uint64_t>();
1929 if (std::error_code EC = Version.getError())
1930 return EC;
1931 else if (!formatVersionIsSupported(Version: *Version))
1932 return sampleprof_error::unsupported_version;
1933 FormatVersion = *Version;
1934
1935 return sampleprof_error::success;
1936}
1937
1938std::error_code SampleProfileReaderBinary::readHeader() {
1939 Data = reinterpret_cast<const uint8_t *>(Buffer->getBufferStart());
1940 End = Data + Buffer->getBufferSize();
1941
1942 if (std::error_code EC = readMagicIdent())
1943 return EC;
1944
1945 if (std::error_code EC = readSummary())
1946 return EC;
1947
1948 if (std::error_code EC = readNameTable())
1949 return EC;
1950 return sampleprof_error::success;
1951}
1952
1953std::error_code SampleProfileReaderBinary::readSummaryEntry(
1954 std::vector<ProfileSummaryEntry> &Entries) {
1955 auto Cutoff = readNumber<uint64_t>();
1956 if (std::error_code EC = Cutoff.getError())
1957 return EC;
1958
1959 auto MinBlockCount = readNumber<uint64_t>();
1960 if (std::error_code EC = MinBlockCount.getError())
1961 return EC;
1962
1963 auto NumBlocks = readNumber<uint64_t>();
1964 if (std::error_code EC = NumBlocks.getError())
1965 return EC;
1966
1967 Entries.emplace_back(args&: *Cutoff, args&: *MinBlockCount, args&: *NumBlocks);
1968 return sampleprof_error::success;
1969}
1970
1971std::error_code SampleProfileReaderBinary::readSummary() {
1972 auto TotalCount = readNumber<uint64_t>();
1973 if (std::error_code EC = TotalCount.getError())
1974 return EC;
1975
1976 auto MaxBlockCount = readNumber<uint64_t>();
1977 if (std::error_code EC = MaxBlockCount.getError())
1978 return EC;
1979
1980 auto MaxFunctionCount = readNumber<uint64_t>();
1981 if (std::error_code EC = MaxFunctionCount.getError())
1982 return EC;
1983
1984 auto NumBlocks = readNumber<uint64_t>();
1985 if (std::error_code EC = NumBlocks.getError())
1986 return EC;
1987
1988 auto NumFunctions = readNumber<uint64_t>();
1989 if (std::error_code EC = NumFunctions.getError())
1990 return EC;
1991
1992 auto NumSummaryEntries = readNumber<uint64_t>();
1993 if (std::error_code EC = NumSummaryEntries.getError())
1994 return EC;
1995
1996 std::vector<ProfileSummaryEntry> Entries;
1997 for (unsigned i = 0; i < *NumSummaryEntries; i++) {
1998 std::error_code EC = readSummaryEntry(Entries);
1999 if (EC != sampleprof_error::success)
2000 return EC;
2001 }
2002 Summary = std::make_unique<ProfileSummary>(
2003 args: ProfileSummary::PSK_Sample, args&: Entries, args&: *TotalCount, args&: *MaxBlockCount, args: 0,
2004 args&: *MaxFunctionCount, args&: *NumBlocks, args&: *NumFunctions);
2005
2006 return sampleprof_error::success;
2007}
2008
2009/// Return whether Buffer starts with ExpectedMagic without reading beyond it.
2010static bool hasBinaryFormat(const MemoryBuffer &Buffer,
2011 uint64_t ExpectedMagic) {
2012 const uint8_t *Data =
2013 reinterpret_cast<const uint8_t *>(Buffer.getBufferStart());
2014 const uint8_t *End = reinterpret_cast<const uint8_t *>(Buffer.getBufferEnd());
2015 ULEB128DecodeError DecodeError = ULEB128DecodeError::None;
2016 uint64_t Magic = decodeULEB128(p: Data, n: nullptr, end: End, error: nullptr, errorCode: &DecodeError);
2017 return DecodeError == ULEB128DecodeError::None && Magic == ExpectedMagic;
2018}
2019
2020bool SampleProfileReaderRawBinary::hasFormat(const MemoryBuffer &Buffer) {
2021 return hasBinaryFormat(Buffer, ExpectedMagic: SPMagic());
2022}
2023
2024bool SampleProfileReaderExtBinary::hasFormat(const MemoryBuffer &Buffer) {
2025 return hasBinaryFormat(Buffer, ExpectedMagic: SPMagic(Format: SPF_Ext_Binary));
2026}
2027
2028std::error_code SampleProfileReaderGCC::skipNextWord() {
2029 uint32_t dummy;
2030 if (!GcovBuffer.readInt(Val&: dummy))
2031 return sampleprof_error::truncated;
2032 return sampleprof_error::success;
2033}
2034
2035template <typename T> ErrorOr<T> SampleProfileReaderGCC::readNumber() {
2036 if (sizeof(T) <= sizeof(uint32_t)) {
2037 uint32_t Val;
2038 if (GcovBuffer.readInt(Val) && Val <= std::numeric_limits<T>::max())
2039 return static_cast<T>(Val);
2040 } else if (sizeof(T) <= sizeof(uint64_t)) {
2041 uint64_t Val;
2042 if (GcovBuffer.readInt64(Val) && Val <= std::numeric_limits<T>::max())
2043 return static_cast<T>(Val);
2044 }
2045
2046 std::error_code EC = sampleprof_error::malformed;
2047 reportError(LineNumber: 0, Msg: EC.message());
2048 return EC;
2049}
2050
2051ErrorOr<StringRef> SampleProfileReaderGCC::readString() {
2052 StringRef Str;
2053 if (!GcovBuffer.readString(str&: Str))
2054 return sampleprof_error::truncated;
2055 return Str;
2056}
2057
2058std::error_code SampleProfileReaderGCC::readHeader() {
2059 // Read the magic identifier.
2060 if (!GcovBuffer.readGCDAFormat())
2061 return sampleprof_error::unrecognized_format;
2062
2063 // Read the version number. Note - the GCC reader does not validate this
2064 // version, but the profile creator generates v704.
2065 GCOV::GCOVVersion version;
2066 if (!GcovBuffer.readGCOVVersion(version))
2067 return sampleprof_error::unrecognized_format;
2068
2069 if (version != GCOV::V407)
2070 return sampleprof_error::unsupported_version;
2071
2072 // Skip the empty integer.
2073 if (std::error_code EC = skipNextWord())
2074 return EC;
2075
2076 return sampleprof_error::success;
2077}
2078
2079std::error_code SampleProfileReaderGCC::readSectionTag(uint32_t Expected) {
2080 uint32_t Tag;
2081 if (!GcovBuffer.readInt(Val&: Tag))
2082 return sampleprof_error::truncated;
2083
2084 if (Tag != Expected)
2085 return sampleprof_error::malformed;
2086
2087 if (std::error_code EC = skipNextWord())
2088 return EC;
2089
2090 return sampleprof_error::success;
2091}
2092
2093std::error_code SampleProfileReaderGCC::readNameTable() {
2094 if (std::error_code EC = readSectionTag(Expected: GCOVTagAFDOFileNames))
2095 return EC;
2096
2097 uint32_t Size;
2098 if (!GcovBuffer.readInt(Val&: Size))
2099 return sampleprof_error::truncated;
2100
2101 for (uint32_t I = 0; I < Size; ++I) {
2102 StringRef Str;
2103 if (!GcovBuffer.readString(str&: Str))
2104 return sampleprof_error::truncated;
2105 Names.push_back(x: std::string(Str));
2106 }
2107
2108 return sampleprof_error::success;
2109}
2110
2111std::error_code SampleProfileReaderGCC::readFunctionProfiles() {
2112 if (std::error_code EC = readSectionTag(Expected: GCOVTagAFDOFunction))
2113 return EC;
2114
2115 uint32_t NumFunctions;
2116 if (!GcovBuffer.readInt(Val&: NumFunctions))
2117 return sampleprof_error::truncated;
2118
2119 InlineCallStack Stack;
2120 for (uint32_t I = 0; I < NumFunctions; ++I)
2121 if (std::error_code EC = readOneFunctionProfile(InlineStack: Stack, Update: true, Offset: 0))
2122 return EC;
2123
2124 computeSummary();
2125 return sampleprof_error::success;
2126}
2127
2128std::error_code SampleProfileReaderGCC::readOneFunctionProfile(
2129 const InlineCallStack &InlineStack, bool Update, uint32_t Offset) {
2130 uint64_t HeadCount = 0;
2131 if (InlineStack.size() == 0)
2132 if (!GcovBuffer.readInt64(Val&: HeadCount))
2133 return sampleprof_error::truncated;
2134
2135 uint32_t NameIdx;
2136 if (!GcovBuffer.readInt(Val&: NameIdx))
2137 return sampleprof_error::truncated;
2138
2139 StringRef Name(Names[NameIdx]);
2140
2141 uint32_t NumPosCounts;
2142 if (!GcovBuffer.readInt(Val&: NumPosCounts))
2143 return sampleprof_error::truncated;
2144
2145 uint32_t NumCallsites;
2146 if (!GcovBuffer.readInt(Val&: NumCallsites))
2147 return sampleprof_error::truncated;
2148
2149 FunctionSamples *FProfile = nullptr;
2150 if (InlineStack.size() == 0) {
2151 // If this is a top function that we have already processed, do not
2152 // update its profile again. This happens in the presence of
2153 // function aliases. Since these aliases share the same function
2154 // body, there will be identical replicated profiles for the
2155 // original function. In this case, we simply not bother updating
2156 // the profile of the original function.
2157 FProfile = &Profiles[FunctionId(Name)];
2158 FProfile->addHeadSamples(Num: HeadCount);
2159 if (FProfile->getTotalSamples() > 0)
2160 Update = false;
2161 } else {
2162 // Otherwise, we are reading an inlined instance. The top of the
2163 // inline stack contains the profile of the caller. Insert this
2164 // callee in the caller's CallsiteMap.
2165 FunctionSamples *CallerProfile = InlineStack.front();
2166 uint32_t LineOffset = Offset >> 16;
2167 uint32_t Discriminator = Offset & 0xffff;
2168 FProfile = &CallerProfile->functionSamplesAt(
2169 Loc: LineLocation(LineOffset, Discriminator))[FunctionId(Name)];
2170 }
2171 FProfile->setFunction(FunctionId(Name));
2172 FProfile->reserveBodySamples(NumEntries: NumPosCounts);
2173
2174 for (uint32_t I = 0; I < NumPosCounts; ++I) {
2175 uint32_t Offset;
2176 if (!GcovBuffer.readInt(Val&: Offset))
2177 return sampleprof_error::truncated;
2178
2179 uint32_t NumTargets;
2180 if (!GcovBuffer.readInt(Val&: NumTargets))
2181 return sampleprof_error::truncated;
2182
2183 uint64_t Count;
2184 if (!GcovBuffer.readInt64(Val&: Count))
2185 return sampleprof_error::truncated;
2186
2187 // The line location is encoded in the offset as:
2188 // high 16 bits: line offset to the start of the function.
2189 // low 16 bits: discriminator.
2190 uint32_t LineOffset = Offset >> 16;
2191 uint32_t Discriminator = Offset & 0xffff;
2192
2193 InlineCallStack NewStack;
2194 NewStack.push_back(Elt: FProfile);
2195 llvm::append_range(C&: NewStack, R: InlineStack);
2196 if (Update) {
2197 // Walk up the inline stack, adding the samples on this line to
2198 // the total sample count of the callers in the chain.
2199 for (auto *CallerProfile : NewStack)
2200 CallerProfile->addTotalSamples(Num: Count);
2201
2202 // Update the body samples for the current profile.
2203 FProfile->addBodySamples(LineOffset, Discriminator, Num: Count);
2204 }
2205
2206 // Process the list of functions called at an indirect call site.
2207 // These are all the targets that a function pointer (or virtual
2208 // function) resolved at runtime.
2209 for (uint32_t J = 0; J < NumTargets; J++) {
2210 uint32_t HistVal;
2211 if (!GcovBuffer.readInt(Val&: HistVal))
2212 return sampleprof_error::truncated;
2213
2214 if (HistVal != HIST_TYPE_INDIR_CALL_TOPN)
2215 return sampleprof_error::malformed;
2216
2217 uint64_t TargetIdx;
2218 if (!GcovBuffer.readInt64(Val&: TargetIdx))
2219 return sampleprof_error::truncated;
2220 StringRef TargetName(Names[TargetIdx]);
2221
2222 uint64_t TargetCount;
2223 if (!GcovBuffer.readInt64(Val&: TargetCount))
2224 return sampleprof_error::truncated;
2225
2226 if (Update)
2227 FProfile->addCalledTargetSamples(LineOffset, Discriminator,
2228 Func: FunctionId(TargetName), Num: TargetCount);
2229 }
2230 }
2231
2232 // Process all the inlined callers into the current function. These
2233 // are all the callsites that were inlined into this function.
2234 for (uint32_t I = 0; I < NumCallsites; I++) {
2235 // The offset is encoded as:
2236 // high 16 bits: line offset to the start of the function.
2237 // low 16 bits: discriminator.
2238 uint32_t Offset;
2239 if (!GcovBuffer.readInt(Val&: Offset))
2240 return sampleprof_error::truncated;
2241 InlineCallStack NewStack;
2242 NewStack.push_back(Elt: FProfile);
2243 llvm::append_range(C&: NewStack, R: InlineStack);
2244 if (std::error_code EC = readOneFunctionProfile(InlineStack: NewStack, Update, Offset))
2245 return EC;
2246 }
2247
2248 return sampleprof_error::success;
2249}
2250
2251/// Read a GCC AutoFDO profile.
2252///
2253/// This format is generated by the Linux Perf conversion tool at
2254/// https://github.com/google/autofdo.
2255std::error_code SampleProfileReaderGCC::readImpl() {
2256 assert(!ProfileIsFSDisciminator && "Gcc profiles not support FSDisciminator");
2257 // Read the string table.
2258 if (std::error_code EC = readNameTable())
2259 return EC;
2260
2261 // Read the source profile.
2262 if (std::error_code EC = readFunctionProfiles())
2263 return EC;
2264
2265 return sampleprof_error::success;
2266}
2267
2268bool SampleProfileReaderGCC::hasFormat(const MemoryBuffer &Buffer) {
2269 StringRef Contents = Buffer.getBuffer();
2270 // Preserve exact magic matching, including a magic-only eight-byte buffer.
2271 return Contents.starts_with(Prefix: "adcg*704") &&
2272 (Contents.size() == 8 || Contents[8] == '\0');
2273}
2274
2275void SampleProfileReaderItaniumRemapper::applyRemapping(LLVMContext &Ctx) {
2276 // If the reader uses MD5 to represent string, we can't remap it because
2277 // we don't know what the original function names were.
2278 if (Reader.useMD5()) {
2279 Ctx.diagnose(DI: DiagnosticInfoSampleProfile(
2280 Reader.getBuffer()->getBufferIdentifier(),
2281 "Profile data remapping cannot be applied to profile data "
2282 "using MD5 names (original mangled names are not available).",
2283 DS_Warning));
2284 return;
2285 }
2286
2287 // CSSPGO-TODO: Remapper is not yet supported.
2288 // We will need to remap the entire context string.
2289 assert(Remappings && "should be initialized while creating remapper");
2290 for (auto &Sample : Reader.getProfiles()) {
2291 DenseSet<FunctionId> NamesInSample;
2292 Sample.second.findAllNames(NameSet&: NamesInSample);
2293 for (auto &Name : NamesInSample) {
2294 StringRef NameStr = Name.stringRef();
2295 if (auto Key = Remappings->insert(FunctionName: NameStr))
2296 NameMap.insert(KV: {Key, NameStr});
2297 }
2298 }
2299
2300 RemappingApplied = true;
2301}
2302
2303std::optional<StringRef>
2304SampleProfileReaderItaniumRemapper::lookUpNameInProfile(StringRef Fname) {
2305 if (auto Key = Remappings->lookup(FunctionName: Fname)) {
2306 StringRef Result = NameMap.lookup(Val: Key);
2307 if (!Result.empty())
2308 return Result;
2309 }
2310 return std::nullopt;
2311}
2312
2313/// Prepare a memory buffer for the contents of \p Filename.
2314///
2315/// \returns an error code indicating the status of the buffer.
2316static ErrorOr<std::unique_ptr<MemoryBuffer>>
2317setupMemoryBuffer(const Twine &Filename, vfs::FileSystem &FS) {
2318 auto BufferOrErr = Filename.str() == "-" ? MemoryBuffer::getSTDIN()
2319 : FS.getBufferForFile(Name: Filename);
2320 if (std::error_code EC = BufferOrErr.getError())
2321 return EC;
2322 auto Buffer = std::move(BufferOrErr.get());
2323
2324 return std::move(Buffer);
2325}
2326
2327/// Create a sample profile reader based on the format of the input file.
2328///
2329/// \param Filename The file to open.
2330///
2331/// \param C The LLVM context to use to emit diagnostics.
2332///
2333/// \param P The FSDiscriminatorPass.
2334///
2335/// \param RemapFilename The file used for profile remapping.
2336///
2337/// \returns an error code indicating the status of the created reader.
2338ErrorOr<std::unique_ptr<SampleProfileReader>>
2339SampleProfileReader::create(StringRef Filename, LLVMContext &C,
2340 vfs::FileSystem &FS, FSDiscriminatorPass P,
2341 StringRef RemapFilename) {
2342 auto BufferOrError = setupMemoryBuffer(Filename, FS);
2343 if (std::error_code EC = BufferOrError.getError())
2344 return EC;
2345 return create(B&: BufferOrError.get(), C, FS, P, RemapFilename);
2346}
2347
2348/// Create a sample profile remapper from the given input, to remap the
2349/// function names in the given profile data.
2350///
2351/// \param Filename The file to open.
2352///
2353/// \param Reader The profile reader the remapper is going to be applied to.
2354///
2355/// \param C The LLVM context to use to emit diagnostics.
2356///
2357/// \returns an error code indicating the status of the created reader.
2358ErrorOr<std::unique_ptr<SampleProfileReaderItaniumRemapper>>
2359SampleProfileReaderItaniumRemapper::create(StringRef Filename,
2360 vfs::FileSystem &FS,
2361 SampleProfileReader &Reader,
2362 LLVMContext &C) {
2363 auto BufferOrError = setupMemoryBuffer(Filename, FS);
2364 if (std::error_code EC = BufferOrError.getError())
2365 return EC;
2366 return create(B&: BufferOrError.get(), Reader, C);
2367}
2368
2369/// Create a sample profile remapper from the given input, to remap the
2370/// function names in the given profile data.
2371///
2372/// \param B The memory buffer to create the reader from (assumes ownership).
2373///
2374/// \param C The LLVM context to use to emit diagnostics.
2375///
2376/// \param Reader The profile reader the remapper is going to be applied to.
2377///
2378/// \returns an error code indicating the status of the created reader.
2379ErrorOr<std::unique_ptr<SampleProfileReaderItaniumRemapper>>
2380SampleProfileReaderItaniumRemapper::create(std::unique_ptr<MemoryBuffer> &B,
2381 SampleProfileReader &Reader,
2382 LLVMContext &C) {
2383 auto Remappings = std::make_unique<SymbolRemappingReader>();
2384 if (Error E = Remappings->read(B&: *B)) {
2385 handleAllErrors(
2386 E: std::move(E), Handlers: [&](const SymbolRemappingParseError &ParseError) {
2387 C.diagnose(DI: DiagnosticInfoSampleProfile(B->getBufferIdentifier(),
2388 ParseError.getLineNum(),
2389 ParseError.getMessage()));
2390 });
2391 return sampleprof_error::malformed;
2392 }
2393
2394 return std::make_unique<SampleProfileReaderItaniumRemapper>(
2395 args: std::move(B), args: std::move(Remappings), args&: Reader);
2396}
2397
2398/// Create a sample profile reader based on the format of the input data.
2399///
2400/// \param B The memory buffer to create the reader from (assumes ownership).
2401///
2402/// \param C The LLVM context to use to emit diagnostics.
2403///
2404/// \param P The FSDiscriminatorPass.
2405///
2406/// \param RemapFilename The file used for profile remapping.
2407///
2408/// \returns an error code indicating the status of the created reader.
2409ErrorOr<std::unique_ptr<SampleProfileReader>>
2410SampleProfileReader::create(std::unique_ptr<MemoryBuffer> &B, LLVMContext &C,
2411 vfs::FileSystem &FS, FSDiscriminatorPass P,
2412 StringRef RemapFilename) {
2413 std::unique_ptr<SampleProfileReader> Reader;
2414 if (SampleProfileReaderRawBinary::hasFormat(Buffer: *B))
2415 Reader.reset(p: new SampleProfileReaderRawBinary(std::move(B), C));
2416 else if (SampleProfileReaderExtBinary::hasFormat(Buffer: *B))
2417 Reader.reset(p: new SampleProfileReaderExtBinary(std::move(B), C));
2418 else if (SampleProfileReaderGCC::hasFormat(Buffer: *B))
2419 Reader.reset(p: new SampleProfileReaderGCC(std::move(B), C));
2420 else if (SampleProfileReaderText::hasFormat(Buffer: *B))
2421 Reader.reset(p: new SampleProfileReaderText(std::move(B), C));
2422 else
2423 return sampleprof_error::unrecognized_format;
2424
2425 if (!RemapFilename.empty()) {
2426 auto ReaderOrErr = SampleProfileReaderItaniumRemapper::create(
2427 Filename: RemapFilename, FS, Reader&: *Reader, C);
2428 if (std::error_code EC = ReaderOrErr.getError()) {
2429 std::string Msg = "Could not create remapper: " + EC.message();
2430 C.diagnose(DI: DiagnosticInfoSampleProfile(RemapFilename, Msg));
2431 return EC;
2432 }
2433 Reader->Remapper = std::move(ReaderOrErr.get());
2434 }
2435
2436 if (std::error_code EC = Reader->readHeader()) {
2437 return EC;
2438 }
2439
2440 Reader->setDiscriminatorMaskedBitFrom(P);
2441
2442 return std::move(Reader);
2443}
2444
2445// For text and GCC file formats, we compute the summary after reading the
2446// profile. Binary format has the profile summary in its header.
2447void SampleProfileReader::computeSummary() {
2448 SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs);
2449 Summary = Builder.computeSummaryForProfiles(Profiles);
2450}
2451