1//===- SampleProfReader.cpp - Read LLVM sample profile data ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the class that reads LLVM sample profiles. It
10// supports three file formats: text, binary and gcov.
11//
12// The textual representation is useful for debugging and testing purposes. The
13// binary representation is more compact, resulting in smaller file sizes.
14//
15// The gcov encoding is the one generated by GCC's AutoFDO profile creation
16// tool (https://github.com/google/autofdo)
17//
18// All three encodings can be used interchangeably as an input sample profile.
19//
20//===----------------------------------------------------------------------===//
21
22#include "llvm/ProfileData/SampleProfReader.h"
23#include "llvm/ADT/DenseMap.h"
24#include "llvm/ADT/STLExtras.h"
25#include "llvm/ADT/StringRef.h"
26#include "llvm/IR/Module.h"
27#include "llvm/IR/ProfileSummary.h"
28#include "llvm/ProfileData/ProfileCommon.h"
29#include "llvm/ProfileData/SampleProf.h"
30#include "llvm/Support/CommandLine.h"
31#include "llvm/Support/Compression.h"
32#include "llvm/Support/ErrorOr.h"
33#include "llvm/Support/JSON.h"
34#include "llvm/Support/LEB128.h"
35#include "llvm/Support/LineIterator.h"
36#include "llvm/Support/MD5.h"
37#include "llvm/Support/MemoryBuffer.h"
38#include "llvm/Support/VirtualFileSystem.h"
39#include "llvm/Support/raw_ostream.h"
40#include <algorithm>
41#include <cstddef>
42#include <cstdint>
43#include <limits>
44#include <memory>
45#include <system_error>
46#include <vector>
47
48using namespace llvm;
49using namespace sampleprof;
50
51#define DEBUG_TYPE "samplepgo-reader"
52
53// This internal option specifies if the profile uses FS discriminators.
54// It only applies to text, and binary format profiles.
55// For ext-binary format profiles, the flag is set in the summary.
56static cl::opt<bool> ProfileIsFSDisciminator(
57 "profile-isfs", cl::Hidden, cl::init(Val: false),
58 cl::desc("Profile uses flow sensitive discriminators"));
59
60/// Dump the function profile for \p FName.
61///
62/// \param FContext Name + context of the function to print.
63/// \param OS Stream to emit the output to.
64void SampleProfileReader::dumpFunctionProfile(const FunctionSamples &FS,
65 raw_ostream &OS) {
66 OS << "Function: " << FS.getContext().toString() << ": " << FS;
67}
68
69/// Dump all the function profiles found on stream \p OS.
70void SampleProfileReader::dump(raw_ostream &OS) {
71 std::vector<NameFunctionSamples> V;
72 sortFuncProfiles(ProfileMap: Profiles, SortedProfiles&: V);
73 for (const auto &I : V)
74 dumpFunctionProfile(FS: *I.second, OS);
75}
76
77static void dumpFunctionProfileJson(const FunctionSamples &S,
78 json::OStream &JOS, bool TopLevel = false) {
79 auto DumpBody = [&](const BodySampleMap &BodySamples) {
80 for (const auto &I : BodySamples) {
81 const LineLocation &Loc = I.first;
82 const SampleRecord &Sample = I.second;
83 JOS.object(Contents: [&] {
84 JOS.attribute(Key: "line", Contents: Loc.LineOffset);
85 if (Loc.Discriminator)
86 JOS.attribute(Key: "discriminator", Contents: Loc.Discriminator);
87 JOS.attribute(Key: "samples", Contents: Sample.getSamples());
88
89 auto CallTargets = Sample.getSortedCallTargets();
90 if (!CallTargets.empty()) {
91 JOS.attributeArray(Key: "calls", Contents: [&] {
92 for (const auto &J : CallTargets) {
93 JOS.object(Contents: [&] {
94 JOS.attribute(Key: "function", Contents: J.first.str());
95 JOS.attribute(Key: "samples", Contents: J.second);
96 });
97 }
98 });
99 }
100 });
101 }
102 };
103
104 auto DumpCallsiteSamples = [&](const CallsiteSampleMap &CallsiteSamples) {
105 for (const auto &I : CallsiteSamples)
106 for (const auto &FS : I.second) {
107 const LineLocation &Loc = I.first;
108 const FunctionSamples &CalleeSamples = FS.second;
109 JOS.object(Contents: [&] {
110 JOS.attribute(Key: "line", Contents: Loc.LineOffset);
111 if (Loc.Discriminator)
112 JOS.attribute(Key: "discriminator", Contents: Loc.Discriminator);
113 JOS.attributeArray(
114 Key: "samples", Contents: [&] { dumpFunctionProfileJson(S: CalleeSamples, JOS); });
115 });
116 }
117 };
118
119 JOS.object(Contents: [&] {
120 JOS.attribute(Key: "name", Contents: S.getFunction().str());
121 JOS.attribute(Key: "total", Contents: S.getTotalSamples());
122 if (TopLevel)
123 JOS.attribute(Key: "head", Contents: S.getHeadSamples());
124
125 const auto &BodySamples = S.getBodySamples();
126 if (!BodySamples.empty())
127 JOS.attributeArray(Key: "body", Contents: [&] { DumpBody(BodySamples); });
128
129 const auto &CallsiteSamples = S.getCallsiteSamples();
130 if (!CallsiteSamples.empty())
131 JOS.attributeArray(Key: "callsites",
132 Contents: [&] { DumpCallsiteSamples(CallsiteSamples); });
133 });
134}
135
136/// Dump all the function profiles found on stream \p OS in the JSON format.
137void SampleProfileReader::dumpJson(raw_ostream &OS) {
138 std::vector<NameFunctionSamples> V;
139 sortFuncProfiles(ProfileMap: Profiles, SortedProfiles&: V);
140 json::OStream JOS(OS, 2);
141 JOS.arrayBegin();
142 for (const auto &F : V)
143 dumpFunctionProfileJson(S: *F.second, JOS, TopLevel: true);
144 JOS.arrayEnd();
145
146 // Emit a newline character at the end as json::OStream doesn't emit one.
147 OS << "\n";
148}
149
150/// Parse \p Input as function head.
151///
152/// Parse one line of \p Input, and update function name in \p FName,
153/// function's total sample count in \p NumSamples, function's entry
154/// count in \p NumHeadSamples.
155///
156/// \returns true if parsing is successful.
157static bool ParseHead(const StringRef &Input, StringRef &FName,
158 uint64_t &NumSamples, uint64_t &NumHeadSamples) {
159 if (Input[0] == ' ')
160 return false;
161 size_t n2 = Input.rfind(C: ':');
162 size_t n1 = Input.rfind(C: ':', From: n2 - 1);
163 FName = Input.substr(Start: 0, N: n1);
164 if (Input.substr(Start: n1 + 1, N: n2 - n1 - 1).getAsInteger(Radix: 10, Result&: NumSamples))
165 return false;
166 if (Input.substr(Start: n2 + 1).getAsInteger(Radix: 10, Result&: NumHeadSamples))
167 return false;
168 return true;
169}
170
171/// Returns true if line offset \p L is legal (only has 16 bits).
172static bool isOffsetLegal(unsigned L) { return (L & 0xffff) == L; }
173
174/// Parse \p Input that contains metadata.
175/// Possible metadata:
176/// - CFG Checksum information:
177/// !CFGChecksum: 12345
178/// - CFG Checksum information:
179/// !Attributes: 1
180/// Stores the FunctionHash (a.k.a. CFG Checksum) into \p FunctionHash.
181static bool parseMetadata(const StringRef &Input, uint64_t &FunctionHash,
182 uint32_t &Attributes) {
183 if (Input.starts_with(Prefix: "!CFGChecksum:")) {
184 StringRef CFGInfo = Input.substr(Start: strlen(s: "!CFGChecksum:")).trim();
185 return !CFGInfo.getAsInteger(Radix: 10, Result&: FunctionHash);
186 }
187
188 if (Input.starts_with(Prefix: "!Attributes:")) {
189 StringRef Attrib = Input.substr(Start: strlen(s: "!Attributes:")).trim();
190 return !Attrib.getAsInteger(Radix: 10, Result&: Attributes);
191 }
192
193 return false;
194}
195
196enum class LineType {
197 CallSiteProfile,
198 BodyProfile,
199 Metadata,
200 VirtualCallTypeProfile,
201};
202
203// Parse `Input` as a white-space separated list of `vtable:count` pairs. An
204// example input line is `_ZTVbar:1471 _ZTVfoo:630`.
205static bool parseTypeCountMap(StringRef Input,
206 DenseMap<StringRef, uint64_t> &TypeCountMap) {
207 for (size_t Index = Input.find_first_not_of(C: ' '); Index != StringRef::npos;) {
208 size_t ColonIndex = Input.find(C: ':', From: Index);
209 if (ColonIndex == StringRef::npos)
210 return false; // No colon found, invalid format.
211 StringRef TypeName = Input.substr(Start: Index, N: ColonIndex - Index);
212 // CountIndex is the start index of count.
213 size_t CountStartIndex = ColonIndex + 1;
214 // NextIndex is the start index after the 'target:count' pair.
215 size_t NextIndex = Input.find_first_of(C: ' ', From: CountStartIndex);
216 uint64_t Count;
217 if (Input.substr(Start: CountStartIndex, N: NextIndex - CountStartIndex)
218 .getAsInteger(Radix: 10, Result&: Count))
219 return false; // Invalid count.
220 // Error on duplicated type names in one line of input.
221 auto [Iter, Inserted] = TypeCountMap.insert(KV: {TypeName, Count});
222 if (!Inserted)
223 return false;
224 Index = (NextIndex == StringRef::npos)
225 ? StringRef::npos
226 : Input.find_first_not_of(C: ' ', From: NextIndex);
227 }
228 return true;
229}
230
231/// Parse \p Input as line sample.
232///
233/// \param Input input line.
234/// \param LineTy Type of this line.
235/// \param Depth the depth of the inline stack.
236/// \param NumSamples total samples of the line/inlined callsite.
237/// \param LineOffset line offset to the start of the function.
238/// \param Discriminator discriminator of the line.
239/// \param TargetCountMap map from indirect call target to count.
240/// \param FunctionHash the function's CFG hash, used by pseudo probe.
241///
242/// returns true if parsing is successful.
243static bool ParseLine(const StringRef &Input, LineType &LineTy, uint32_t &Depth,
244 uint64_t &NumSamples, uint32_t &LineOffset,
245 uint32_t &Discriminator, StringRef &CalleeName,
246 DenseMap<StringRef, uint64_t> &TargetCountMap,
247 DenseMap<StringRef, uint64_t> &TypeCountMap,
248 uint64_t &FunctionHash, uint32_t &Attributes,
249 bool &IsFlat) {
250 for (Depth = 0; Input[Depth] == ' '; Depth++)
251 ;
252 if (Depth == 0)
253 return false;
254
255 if (Input[Depth] == '!') {
256 LineTy = LineType::Metadata;
257 // This metadata is only for manual inspection only. We already created a
258 // FunctionSamples and put it in the profile map, so there is no point
259 // to skip profiles even they have no use for ThinLTO.
260 if (Input == StringRef(" !Flat")) {
261 IsFlat = true;
262 return true;
263 }
264 return parseMetadata(Input: Input.substr(Start: Depth), FunctionHash, Attributes);
265 }
266
267 size_t n1 = Input.find(C: ':');
268 StringRef Loc = Input.substr(Start: Depth, N: n1 - Depth);
269 size_t n2 = Loc.find(C: '.');
270 if (n2 == StringRef::npos) {
271 if (Loc.getAsInteger(Radix: 10, Result&: LineOffset) || !isOffsetLegal(L: LineOffset))
272 return false;
273 Discriminator = 0;
274 } else {
275 if (Loc.substr(Start: 0, N: n2).getAsInteger(Radix: 10, Result&: LineOffset))
276 return false;
277 if (Loc.substr(Start: n2 + 1).getAsInteger(Radix: 10, Result&: Discriminator))
278 return false;
279 }
280
281 StringRef Rest = Input.substr(Start: n1 + 2);
282 if (isDigit(C: Rest[0])) {
283 LineTy = LineType::BodyProfile;
284 size_t n3 = Rest.find(C: ' ');
285 if (n3 == StringRef::npos) {
286 if (Rest.getAsInteger(Radix: 10, Result&: NumSamples))
287 return false;
288 } else {
289 if (Rest.substr(Start: 0, N: n3).getAsInteger(Radix: 10, Result&: NumSamples))
290 return false;
291 }
292 // Find call targets and their sample counts.
293 // Note: In some cases, there are symbols in the profile which are not
294 // mangled. To accommodate such cases, use colon + integer pairs as the
295 // anchor points.
296 // An example:
297 // _M_construct<char *>:1000 string_view<std::allocator<char> >:437
298 // ":1000" and ":437" are used as anchor points so the string above will
299 // be interpreted as
300 // target: _M_construct<char *>
301 // count: 1000
302 // target: string_view<std::allocator<char> >
303 // count: 437
304 while (n3 != StringRef::npos) {
305 n3 += Rest.substr(Start: n3).find_first_not_of(C: ' ');
306 Rest = Rest.substr(Start: n3);
307 n3 = Rest.find_first_of(C: ':');
308 if (n3 == StringRef::npos || n3 == 0)
309 return false;
310
311 StringRef Target;
312 uint64_t count, n4;
313 while (true) {
314 // Get the segment after the current colon.
315 StringRef AfterColon = Rest.substr(Start: n3 + 1);
316 // Get the target symbol before the current colon.
317 Target = Rest.substr(Start: 0, N: n3);
318 // Check if the word after the current colon is an integer.
319 n4 = AfterColon.find_first_of(C: ' ');
320 n4 = (n4 != StringRef::npos) ? n3 + n4 + 1 : Rest.size();
321 StringRef WordAfterColon = Rest.substr(Start: n3 + 1, N: n4 - n3 - 1);
322 if (!WordAfterColon.getAsInteger(Radix: 10, Result&: count))
323 break;
324
325 // Try to find the next colon.
326 uint64_t n5 = AfterColon.find_first_of(C: ':');
327 if (n5 == StringRef::npos)
328 return false;
329 n3 += n5 + 1;
330 }
331
332 // An anchor point is found. Save the {target, count} pair
333 TargetCountMap[Target] = count;
334 if (n4 == Rest.size())
335 break;
336 // Change n3 to the next blank space after colon + integer pair.
337 n3 = n4;
338 }
339 } else if (Rest.starts_with(Prefix: kVTableProfPrefix)) {
340 LineTy = LineType::VirtualCallTypeProfile;
341 return parseTypeCountMap(Input: Rest.substr(Start: strlen(s: kVTableProfPrefix)),
342 TypeCountMap);
343 } else {
344 LineTy = LineType::CallSiteProfile;
345 size_t n3 = Rest.find_last_of(C: ':');
346 CalleeName = Rest.substr(Start: 0, N: n3);
347 if (Rest.substr(Start: n3 + 1).getAsInteger(Radix: 10, Result&: NumSamples))
348 return false;
349 }
350 return true;
351}
352
353/// Load samples from a text file.
354///
355/// See the documentation at the top of the file for an explanation of
356/// the expected format.
357///
358/// \returns true if the file was loaded successfully, false otherwise.
359std::error_code SampleProfileReaderText::readImpl() {
360 line_iterator LineIt(*Buffer, /*SkipBlanks=*/true, '#');
361 sampleprof_error Result = sampleprof_error::success;
362
363 InlineCallStack InlineStack;
364 uint32_t TopLevelProbeProfileCount = 0;
365
366 // DepthMetadata tracks whether we have processed metadata for the current
367 // top-level or nested function profile.
368 uint32_t DepthMetadata = 0;
369
370 std::vector<SampleContext *> FlatSamples;
371
372 ProfileIsFS = ProfileIsFSDisciminator;
373 FunctionSamples::ProfileIsFS = ProfileIsFS;
374 for (; !LineIt.is_at_eof(); ++LineIt) {
375 size_t pos = LineIt->find_first_not_of(C: ' ');
376 if (pos == LineIt->npos || (*LineIt)[pos] == '#')
377 continue;
378 // Read the header of each function.
379 //
380 // Note that for function identifiers we are actually expecting
381 // mangled names, but we may not always get them. This happens when
382 // the compiler decides not to emit the function (e.g., it was inlined
383 // and removed). In this case, the binary will not have the linkage
384 // name for the function, so the profiler will emit the function's
385 // unmangled name, which may contain characters like ':' and '>' in its
386 // name (member functions, templates, etc).
387 //
388 // The only requirement we place on the identifier, then, is that it
389 // should not begin with a number.
390 if ((*LineIt)[0] != ' ') {
391 uint64_t NumSamples, NumHeadSamples;
392 StringRef FName;
393 if (!ParseHead(Input: *LineIt, FName, NumSamples, NumHeadSamples)) {
394 reportError(LineNumber: LineIt.line_number(),
395 Msg: "Expected 'mangled_name:NUM:NUM', found " + *LineIt);
396 return sampleprof_error::malformed;
397 }
398 DepthMetadata = 0;
399 SampleContext FContext(FName, CSNameTable);
400 if (FContext.hasContext())
401 ++CSProfileCount;
402 FunctionSamples &FProfile = Profiles.create(Ctx: FContext);
403 mergeSampleProfErrors(Accumulator&: Result, Result: FProfile.addTotalSamples(Num: NumSamples));
404 mergeSampleProfErrors(Accumulator&: Result, Result: FProfile.addHeadSamples(Num: NumHeadSamples));
405 InlineStack.clear();
406 InlineStack.push_back(Elt: &FProfile);
407 } else {
408 uint64_t NumSamples;
409 StringRef FName;
410 DenseMap<StringRef, uint64_t> TargetCountMap;
411 DenseMap<StringRef, uint64_t> TypeCountMap;
412 uint32_t Depth, LineOffset, Discriminator;
413 LineType LineTy = LineType::BodyProfile;
414 uint64_t FunctionHash = 0;
415 uint32_t Attributes = 0;
416 bool IsFlat = false;
417 // TODO: Update ParseLine to return an error code instead of a bool and
418 // report it.
419 if (!ParseLine(Input: *LineIt, LineTy, Depth, NumSamples, LineOffset,
420 Discriminator, CalleeName&: FName, TargetCountMap, TypeCountMap,
421 FunctionHash, Attributes, IsFlat)) {
422 switch (LineTy) {
423 case LineType::Metadata:
424 reportError(LineNumber: LineIt.line_number(),
425 Msg: "Cannot parse metadata: " + *LineIt);
426 break;
427 case LineType::VirtualCallTypeProfile:
428 reportError(LineNumber: LineIt.line_number(),
429 Msg: "Expected 'vtables [mangled_vtable:NUM]+', found " +
430 *LineIt);
431 break;
432 default:
433 reportError(LineNumber: LineIt.line_number(),
434 Msg: "Expected 'NUM[.NUM]: NUM[ mangled_name:NUM]*', found " +
435 *LineIt);
436 }
437 return sampleprof_error::malformed;
438 }
439 if (LineTy != LineType::Metadata && Depth == DepthMetadata) {
440 // Metadata must be put at the end of a function profile.
441 reportError(LineNumber: LineIt.line_number(),
442 Msg: "Found non-metadata after metadata: " + *LineIt);
443 return sampleprof_error::malformed;
444 }
445
446 // Here we handle FS discriminators.
447 Discriminator &= getDiscriminatorMask();
448
449 while (InlineStack.size() > Depth) {
450 InlineStack.pop_back();
451 }
452 switch (LineTy) {
453 case LineType::CallSiteProfile: {
454 FunctionSamples &FSamples = InlineStack.back()->functionSamplesAt(
455 Loc: LineLocation(LineOffset, Discriminator))[FunctionId(FName)];
456 FSamples.setFunction(FunctionId(FName));
457 mergeSampleProfErrors(Accumulator&: Result, Result: FSamples.addTotalSamples(Num: NumSamples));
458 InlineStack.push_back(Elt: &FSamples);
459 DepthMetadata = 0;
460 break;
461 }
462
463 case LineType::VirtualCallTypeProfile: {
464 mergeSampleProfErrors(
465 Accumulator&: Result, Result: InlineStack.back()->addCallsiteVTableTypeProfAt(
466 Loc: LineLocation(LineOffset, Discriminator), Other: TypeCountMap));
467 break;
468 }
469
470 case LineType::BodyProfile: {
471 FunctionSamples &FProfile = *InlineStack.back();
472 for (const auto &name_count : TargetCountMap) {
473 mergeSampleProfErrors(Accumulator&: Result, Result: FProfile.addCalledTargetSamples(
474 LineOffset, Discriminator,
475 Func: FunctionId(name_count.first),
476 Num: name_count.second));
477 }
478 mergeSampleProfErrors(
479 Accumulator&: Result,
480 Result: FProfile.addBodySamples(LineOffset, Discriminator, Num: NumSamples));
481 break;
482 }
483 case LineType::Metadata: {
484 FunctionSamples &FProfile = *InlineStack.back();
485 if (FunctionHash) {
486 FProfile.setFunctionHash(FunctionHash);
487 if (Depth == 1)
488 ++TopLevelProbeProfileCount;
489 }
490 FProfile.getContext().setAllAttributes(Attributes);
491 if (Attributes & (uint32_t)ContextShouldBeInlined)
492 ProfileIsPreInlined = true;
493 DepthMetadata = Depth;
494 if (IsFlat) {
495 if (Depth == 1)
496 FlatSamples.push_back(x: &FProfile.getContext());
497 else
498 Ctx.diagnose(DI: DiagnosticInfoSampleProfile(
499 Buffer->getBufferIdentifier(), LineIt.line_number(),
500 "!Flat may only be used at top level function.", DS_Warning));
501 }
502 break;
503 }
504 }
505 }
506 }
507
508 // Honor the option to skip flat functions. Since they are already added to
509 // the profile map, remove them all here.
510 if (SkipFlatProf)
511 for (SampleContext *FlatSample : FlatSamples)
512 Profiles.erase(Ctx: *FlatSample);
513
514 assert((CSProfileCount == 0 || CSProfileCount == Profiles.size()) &&
515 "Cannot have both context-sensitive and regular profile");
516 ProfileIsCS = (CSProfileCount > 0);
517 assert((TopLevelProbeProfileCount == 0 ||
518 TopLevelProbeProfileCount == Profiles.size()) &&
519 "Cannot have both probe-based profiles and regular profiles");
520 ProfileIsProbeBased = (TopLevelProbeProfileCount > 0);
521 FunctionSamples::ProfileIsProbeBased = ProfileIsProbeBased;
522 FunctionSamples::ProfileIsCS = ProfileIsCS;
523 FunctionSamples::ProfileIsPreInlined = ProfileIsPreInlined;
524
525 if (Result == sampleprof_error::success)
526 computeSummary();
527
528 return Result;
529}
530
531bool SampleProfileReaderText::hasFormat(const MemoryBuffer &Buffer) {
532 bool result = false;
533
534 // Check that the first non-comment line is a valid function header.
535 line_iterator LineIt(Buffer, /*SkipBlanks=*/true, '#');
536 if (!LineIt.is_at_eof()) {
537 if ((*LineIt)[0] != ' ') {
538 uint64_t NumSamples, NumHeadSamples;
539 StringRef FName;
540 result = ParseHead(Input: *LineIt, FName, NumSamples, NumHeadSamples);
541 }
542 }
543
544 return result;
545}
546
547template <typename T> ErrorOr<T> SampleProfileReaderBinary::readNumber() {
548 unsigned NumBytesRead = 0;
549 uint64_t Val = decodeULEB128(p: Data, n: &NumBytesRead);
550
551 if (Val > std::numeric_limits<T>::max()) {
552 std::error_code EC = sampleprof_error::malformed;
553 reportError(LineNumber: 0, Msg: EC.message());
554 return EC;
555 } else if (Data + NumBytesRead > End) {
556 std::error_code EC = sampleprof_error::truncated;
557 reportError(LineNumber: 0, Msg: EC.message());
558 return EC;
559 }
560
561 Data += NumBytesRead;
562 return static_cast<T>(Val);
563}
564
565ErrorOr<StringRef> SampleProfileReaderBinary::readString() {
566 StringRef Str(reinterpret_cast<const char *>(Data));
567 if (Data + Str.size() + 1 > End) {
568 std::error_code EC = sampleprof_error::truncated;
569 reportError(LineNumber: 0, Msg: EC.message());
570 return EC;
571 }
572
573 Data += Str.size() + 1;
574 return Str;
575}
576
577template <typename T>
578ErrorOr<T> SampleProfileReaderBinary::readUnencodedNumber() {
579 if (Data + sizeof(T) > End) {
580 std::error_code EC = sampleprof_error::truncated;
581 reportError(LineNumber: 0, Msg: EC.message());
582 return EC;
583 }
584
585 using namespace support;
586 T Val = endian::readNext<T, llvm::endianness::little>(Data);
587 return Val;
588}
589
590template <typename T>
591inline ErrorOr<size_t> SampleProfileReaderBinary::readStringIndex(T &Table) {
592 auto Idx = readNumber<size_t>();
593 if (std::error_code EC = Idx.getError())
594 return EC;
595 if (*Idx >= Table.size())
596 return sampleprof_error::truncated_name_table;
597 return *Idx;
598}
599
600ErrorOr<FunctionId>
601SampleProfileReaderBinary::readStringFromTable(size_t *RetIdx) {
602 auto Idx = readStringIndex(Table&: NameTable);
603 if (std::error_code EC = Idx.getError())
604 return EC;
605 if (RetIdx)
606 *RetIdx = *Idx;
607 return NameTable[*Idx];
608}
609
610ErrorOr<SampleContextFrames>
611SampleProfileReaderBinary::readContextFromTable(size_t *RetIdx) {
612 auto ContextIdx = readNumber<size_t>();
613 if (std::error_code EC = ContextIdx.getError())
614 return EC;
615 if (*ContextIdx >= CSNameTable.size())
616 return sampleprof_error::truncated_name_table;
617 if (RetIdx)
618 *RetIdx = *ContextIdx;
619 return CSNameTable[*ContextIdx];
620}
621
622ErrorOr<std::pair<SampleContext, uint64_t>>
623SampleProfileReaderBinary::readSampleContextFromTable() {
624 SampleContext Context;
625 size_t Idx;
626 if (ProfileIsCS) {
627 auto FContext(readContextFromTable(RetIdx: &Idx));
628 if (std::error_code EC = FContext.getError())
629 return EC;
630 Context = SampleContext(*FContext);
631 } else {
632 auto FName(readStringFromTable(RetIdx: &Idx));
633 if (std::error_code EC = FName.getError())
634 return EC;
635 Context = SampleContext(*FName);
636 }
637 // Since MD5SampleContextStart may point to the profile's file data, need to
638 // make sure it is reading the same value on big endian CPU.
639 uint64_t Hash = support::endian::read64le(P: MD5SampleContextStart + Idx);
640 // Lazy computing of hash value, write back to the table to cache it. Only
641 // compute the context's hash value if it is being referenced for the first
642 // time.
643 if (Hash == 0) {
644 assert(MD5SampleContextStart == MD5SampleContextTable.data());
645 Hash = Context.getHashCode();
646 support::endian::write64le(P: &MD5SampleContextTable[Idx], V: Hash);
647 }
648 return std::make_pair(x&: Context, y&: Hash);
649}
650
651std::error_code
652SampleProfileReaderBinary::readVTableTypeCountMap(TypeCountMap &M) {
653 auto NumVTableTypes = readNumber<uint32_t>();
654 if (std::error_code EC = NumVTableTypes.getError())
655 return EC;
656
657 for (uint32_t I = 0; I < *NumVTableTypes; ++I) {
658 auto VTableType(readStringFromTable());
659 if (std::error_code EC = VTableType.getError())
660 return EC;
661
662 auto VTableSamples = readNumber<uint64_t>();
663 if (std::error_code EC = VTableSamples.getError())
664 return EC;
665 // The source profile should not have duplicate vtable records at the same
666 // location. In case duplicate vtables are found, reader can emit a warning
667 // but continue processing the profile.
668 if (!M.insert(x: std::make_pair(x&: *VTableType, y&: *VTableSamples)).second) {
669 Ctx.diagnose(DI: DiagnosticInfoSampleProfile(
670 Buffer->getBufferIdentifier(), 0,
671 "Duplicate vtable type " + VTableType->str() +
672 " at the same location. Additional counters will be ignored.",
673 DS_Warning));
674 continue;
675 }
676 }
677 return sampleprof_error::success;
678}
679
680std::error_code
681SampleProfileReaderBinary::readCallsiteVTableProf(FunctionSamples &FProfile) {
682 assert(ReadVTableProf &&
683 "Cannot read vtable profiles if ReadVTableProf is false");
684
685 // Read the vtable type profile for the callsite.
686 auto NumCallsites = readNumber<uint32_t>();
687 if (std::error_code EC = NumCallsites.getError())
688 return EC;
689
690 for (uint32_t I = 0; I < *NumCallsites; ++I) {
691 auto LineOffset = readNumber<uint64_t>();
692 if (std::error_code EC = LineOffset.getError())
693 return EC;
694
695 if (!isOffsetLegal(L: *LineOffset))
696 return sampleprof_error::illegal_line_offset;
697
698 auto Discriminator = readNumber<uint64_t>();
699 if (std::error_code EC = Discriminator.getError())
700 return EC;
701
702 // Here we handle FS discriminators:
703 const uint32_t DiscriminatorVal = (*Discriminator) & getDiscriminatorMask();
704
705 if (std::error_code EC = readVTableTypeCountMap(M&: FProfile.getTypeSamplesAt(
706 Loc: LineLocation(*LineOffset, DiscriminatorVal))))
707 return EC;
708 }
709 return sampleprof_error::success;
710}
711
712std::error_code
713SampleProfileReaderBinary::readProfile(FunctionSamples &FProfile) {
714 auto NumSamples = readNumber<uint64_t>();
715 if (std::error_code EC = NumSamples.getError())
716 return EC;
717 FProfile.addTotalSamples(Num: *NumSamples);
718
719 // Read the samples in the body.
720 auto NumRecords = readNumber<uint32_t>();
721 if (std::error_code EC = NumRecords.getError())
722 return EC;
723
724 for (uint32_t I = 0; I < *NumRecords; ++I) {
725 auto LineOffset = readNumber<uint64_t>();
726 if (std::error_code EC = LineOffset.getError())
727 return EC;
728
729 if (!isOffsetLegal(L: *LineOffset)) {
730 return sampleprof_error::illegal_line_offset;
731 }
732
733 auto Discriminator = readNumber<uint64_t>();
734 if (std::error_code EC = Discriminator.getError())
735 return EC;
736
737 auto NumSamples = readNumber<uint64_t>();
738 if (std::error_code EC = NumSamples.getError())
739 return EC;
740
741 auto NumCalls = readNumber<uint32_t>();
742 if (std::error_code EC = NumCalls.getError())
743 return EC;
744
745 // Here we handle FS discriminators:
746 uint32_t DiscriminatorVal = (*Discriminator) & getDiscriminatorMask();
747
748 for (uint32_t J = 0; J < *NumCalls; ++J) {
749 auto CalledFunction(readStringFromTable());
750 if (std::error_code EC = CalledFunction.getError())
751 return EC;
752
753 auto CalledFunctionSamples = readNumber<uint64_t>();
754 if (std::error_code EC = CalledFunctionSamples.getError())
755 return EC;
756
757 FProfile.addCalledTargetSamples(LineOffset: *LineOffset, Discriminator: DiscriminatorVal,
758 Func: *CalledFunction, Num: *CalledFunctionSamples);
759 }
760
761 FProfile.addBodySamples(LineOffset: *LineOffset, Discriminator: DiscriminatorVal, Num: *NumSamples);
762 }
763
764 // Read all the samples for inlined function calls.
765 auto NumCallsites = readNumber<uint32_t>();
766 if (std::error_code EC = NumCallsites.getError())
767 return EC;
768
769 for (uint32_t J = 0; J < *NumCallsites; ++J) {
770 auto LineOffset = readNumber<uint64_t>();
771 if (std::error_code EC = LineOffset.getError())
772 return EC;
773
774 auto Discriminator = readNumber<uint64_t>();
775 if (std::error_code EC = Discriminator.getError())
776 return EC;
777
778 auto FName(readStringFromTable());
779 if (std::error_code EC = FName.getError())
780 return EC;
781
782 // Here we handle FS discriminators:
783 uint32_t DiscriminatorVal = (*Discriminator) & getDiscriminatorMask();
784
785 FunctionSamples &CalleeProfile = FProfile.functionSamplesAt(
786 Loc: LineLocation(*LineOffset, DiscriminatorVal))[*FName];
787 CalleeProfile.setFunction(*FName);
788 if (std::error_code EC = readProfile(FProfile&: CalleeProfile))
789 return EC;
790 }
791
792 if (ReadVTableProf)
793 return readCallsiteVTableProf(FProfile);
794
795 return sampleprof_error::success;
796}
797
798std::error_code
799SampleProfileReaderBinary::readFuncProfile(const uint8_t *Start,
800 SampleProfileMap &Profiles) {
801 Data = Start;
802 auto NumHeadSamples = readNumber<uint64_t>();
803 if (std::error_code EC = NumHeadSamples.getError())
804 return EC;
805
806 auto FContextHash(readSampleContextFromTable());
807 if (std::error_code EC = FContextHash.getError())
808 return EC;
809
810 auto &[FContext, Hash] = *FContextHash;
811 // Use the cached hash value for insertion instead of recalculating it.
812 auto Res = Profiles.try_emplace(Hash, Key: FContext, Args: FunctionSamples());
813 FunctionSamples &FProfile = Res.first->second;
814 FProfile.setContext(FContext);
815 FProfile.addHeadSamples(Num: *NumHeadSamples);
816
817 if (FContext.hasContext())
818 CSProfileCount++;
819
820 if (std::error_code EC = readProfile(FProfile))
821 return EC;
822 return sampleprof_error::success;
823}
824
825std::error_code
826SampleProfileReaderBinary::readFuncProfile(const uint8_t *Start) {
827 return readFuncProfile(Start, Profiles);
828}
829
830std::error_code SampleProfileReaderBinary::readImpl() {
831 ProfileIsFS = ProfileIsFSDisciminator;
832 FunctionSamples::ProfileIsFS = ProfileIsFS;
833 while (Data < End) {
834 if (std::error_code EC = readFuncProfile(Start: Data))
835 return EC;
836 }
837
838 return sampleprof_error::success;
839}
840
841std::error_code SampleProfileReaderExtBinaryBase::readOneSection(
842 const uint8_t *Start, uint64_t Size, const SecHdrTableEntry &Entry) {
843 Data = Start;
844 End = Start + Size;
845 switch (Entry.Type) {
846 case SecProfSummary:
847 if (std::error_code EC = readSummary())
848 return EC;
849 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagPartial))
850 Summary->setPartialProfile(true);
851 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagFullContext))
852 FunctionSamples::ProfileIsCS = ProfileIsCS = true;
853 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagIsPreInlined))
854 FunctionSamples::ProfileIsPreInlined = ProfileIsPreInlined = true;
855 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagFSDiscriminator))
856 FunctionSamples::ProfileIsFS = ProfileIsFS = true;
857 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagHasVTableTypeProf))
858 ReadVTableProf = true;
859 break;
860 case SecNameTable: {
861 bool FixedLengthMD5 =
862 hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagFixedLengthMD5);
863 bool UseMD5 = hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagMD5Name);
864 // UseMD5 means if THIS section uses MD5, ProfileIsMD5 means if the entire
865 // profile uses MD5 for function name matching in IPO passes.
866 ProfileIsMD5 = ProfileIsMD5 || UseMD5;
867 FunctionSamples::HasUniqSuffix =
868 hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagUniqSuffix);
869 if (std::error_code EC = readNameTableSec(IsMD5: UseMD5, FixedLengthMD5))
870 return EC;
871 break;
872 }
873 case SecCSNameTable: {
874 if (std::error_code EC = readCSNameTableSec())
875 return EC;
876 break;
877 }
878 case SecLBRProfile:
879 ProfileSecRange = std::make_pair(x&: Data, y&: End);
880 if (std::error_code EC = readFuncProfiles())
881 return EC;
882 break;
883 case SecFuncOffsetTable:
884 // If module is absent, we are using LLVM tools, and need to read all
885 // profiles, so skip reading the function offset table.
886 if (!M) {
887 Data = End;
888 } else {
889 assert((!ProfileIsCS ||
890 hasSecFlag(Entry, SecFuncOffsetFlags::SecFlagOrdered)) &&
891 "func offset table should always be sorted in CS profile");
892 if (std::error_code EC = readFuncOffsetTable())
893 return EC;
894 }
895 break;
896 case SecFuncMetadata: {
897 ProfileIsProbeBased =
898 hasSecFlag(Entry, Flag: SecFuncMetadataFlags::SecFlagIsProbeBased);
899 FunctionSamples::ProfileIsProbeBased = ProfileIsProbeBased;
900 ProfileHasAttribute =
901 hasSecFlag(Entry, Flag: SecFuncMetadataFlags::SecFlagHasAttribute);
902 if (std::error_code EC = readFuncMetadata(ProfileHasAttribute))
903 return EC;
904 break;
905 }
906 case SecProfileSymbolList:
907 if (std::error_code EC = readProfileSymbolList())
908 return EC;
909 break;
910 default:
911 if (std::error_code EC = readCustomSection(Entry))
912 return EC;
913 break;
914 }
915 return sampleprof_error::success;
916}
917
918bool SampleProfileReaderExtBinaryBase::useFuncOffsetList() const {
919 // If profile is CS, the function offset section is expected to consist of
920 // sequences of contexts in pre-order layout
921 // (e.g. [A, A:1 @ B, A:1 @ B:2.3 @ C] [D, D:1 @ E]), so that when a matched
922 // context in the module is found, the profiles of all its callees are
923 // recursively loaded. A list is needed since the order of profiles matters.
924 if (ProfileIsCS)
925 return true;
926
927 // If the profile is MD5, use the map container to lookup functions in
928 // the module. A remapper has no use on MD5 names.
929 if (useMD5())
930 return false;
931
932 // Profile is not MD5 and if a remapper is present, the remapped name of
933 // every function needed to be matched against the module, so use the list
934 // container since each entry is accessed.
935 if (Remapper)
936 return true;
937
938 // Otherwise use the map container for faster lookup.
939 // TODO: If the cardinality of the function offset section is much smaller
940 // than the number of functions in the module, using the list container can
941 // be always faster, but we need to figure out the constant factor to
942 // determine the cutoff.
943 return false;
944}
945
946std::error_code
947SampleProfileReaderExtBinaryBase::read(const DenseSet<StringRef> &FuncsToUse,
948 SampleProfileMap &Profiles) {
949 if (FuncsToUse.empty())
950 return sampleprof_error::success;
951
952 Data = ProfileSecRange.first;
953 End = ProfileSecRange.second;
954 if (std::error_code EC = readFuncProfiles(FuncsToUse, Profiles))
955 return EC;
956 End = Data;
957 DenseSet<FunctionSamples *> ProfilesToReadMetadata;
958 for (auto FName : FuncsToUse) {
959 auto I = Profiles.find(Ctx: FName);
960 if (I != Profiles.end())
961 ProfilesToReadMetadata.insert(V: &I->second);
962 }
963
964 if (std::error_code EC =
965 readFuncMetadata(ProfileHasAttribute, Profiles&: ProfilesToReadMetadata))
966 return EC;
967 return sampleprof_error::success;
968}
969
970bool SampleProfileReaderExtBinaryBase::collectFuncsFromModule() {
971 if (!M)
972 return false;
973 FuncsToUse.clear();
974 for (auto &F : *M)
975 FuncsToUse.insert(V: FunctionSamples::getCanonicalFnName(F));
976 return true;
977}
978
979std::error_code SampleProfileReaderExtBinaryBase::readFuncOffsetTable() {
980 // If there are more than one function offset section, the profile associated
981 // with the previous section has to be done reading before next one is read.
982 FuncOffsetTable.clear();
983 FuncOffsetList.clear();
984
985 auto Size = readNumber<uint64_t>();
986 if (std::error_code EC = Size.getError())
987 return EC;
988
989 bool UseFuncOffsetList = useFuncOffsetList();
990 if (UseFuncOffsetList)
991 FuncOffsetList.reserve(n: *Size);
992 else
993 FuncOffsetTable.reserve(NumEntries: *Size);
994
995 for (uint64_t I = 0; I < *Size; ++I) {
996 auto FContextHash(readSampleContextFromTable());
997 if (std::error_code EC = FContextHash.getError())
998 return EC;
999
1000 auto &[FContext, Hash] = *FContextHash;
1001 auto Offset = readNumber<uint64_t>();
1002 if (std::error_code EC = Offset.getError())
1003 return EC;
1004
1005 if (UseFuncOffsetList)
1006 FuncOffsetList.emplace_back(args&: FContext, args&: *Offset);
1007 else
1008 // Because Porfiles replace existing value with new value if collision
1009 // happens, we also use the latest offset so that they are consistent.
1010 FuncOffsetTable[Hash] = *Offset;
1011 }
1012
1013 return sampleprof_error::success;
1014}
1015
1016std::error_code SampleProfileReaderExtBinaryBase::readFuncProfiles(
1017 const DenseSet<StringRef> &FuncsToUse, SampleProfileMap &Profiles) {
1018 const uint8_t *Start = Data;
1019
1020 if (Remapper) {
1021 for (auto Name : FuncsToUse) {
1022 Remapper->insert(FunctionName: Name);
1023 }
1024 }
1025
1026 if (ProfileIsCS) {
1027 assert(useFuncOffsetList());
1028 DenseSet<uint64_t> FuncGuidsToUse;
1029 if (useMD5()) {
1030 for (auto Name : FuncsToUse)
1031 FuncGuidsToUse.insert(V: Function::getGUIDAssumingExternalLinkage(GlobalName: Name));
1032 }
1033
1034 // For each function in current module, load all context profiles for
1035 // the function as well as their callee contexts which can help profile
1036 // guided importing for ThinLTO. This can be achieved by walking
1037 // through an ordered context container, where contexts are laid out
1038 // as if they were walked in preorder of a context trie. While
1039 // traversing the trie, a link to the highest common ancestor node is
1040 // kept so that all of its decendants will be loaded.
1041 const SampleContext *CommonContext = nullptr;
1042 for (const auto &NameOffset : FuncOffsetList) {
1043 const auto &FContext = NameOffset.first;
1044 FunctionId FName = FContext.getFunction();
1045 StringRef FNameString;
1046 if (!useMD5())
1047 FNameString = FName.stringRef();
1048
1049 // For function in the current module, keep its farthest ancestor
1050 // context. This can be used to load itself and its child and
1051 // sibling contexts.
1052 if ((useMD5() && FuncGuidsToUse.count(V: FName.getHashCode())) ||
1053 (!useMD5() && (FuncsToUse.count(V: FNameString) ||
1054 (Remapper && Remapper->exist(FunctionName: FNameString))))) {
1055 if (!CommonContext || !CommonContext->isPrefixOf(That: FContext))
1056 CommonContext = &FContext;
1057 }
1058
1059 if (CommonContext == &FContext ||
1060 (CommonContext && CommonContext->isPrefixOf(That: FContext))) {
1061 // Load profile for the current context which originated from
1062 // the common ancestor.
1063 const uint8_t *FuncProfileAddr = Start + NameOffset.second;
1064 if (std::error_code EC = readFuncProfile(Start: FuncProfileAddr))
1065 return EC;
1066 }
1067 }
1068 } else if (useMD5()) {
1069 assert(!useFuncOffsetList());
1070 for (auto Name : FuncsToUse) {
1071 auto GUID = MD5Hash(Str: Name);
1072 auto iter = FuncOffsetTable.find(Val: GUID);
1073 if (iter == FuncOffsetTable.end())
1074 continue;
1075 const uint8_t *FuncProfileAddr = Start + iter->second;
1076 if (std::error_code EC = readFuncProfile(Start: FuncProfileAddr, Profiles))
1077 return EC;
1078 }
1079 } else if (Remapper) {
1080 assert(useFuncOffsetList());
1081 for (auto NameOffset : FuncOffsetList) {
1082 SampleContext FContext(NameOffset.first);
1083 auto FuncName = FContext.getFunction();
1084 StringRef FuncNameStr = FuncName.stringRef();
1085 if (!FuncsToUse.count(V: FuncNameStr) && !Remapper->exist(FunctionName: FuncNameStr))
1086 continue;
1087 const uint8_t *FuncProfileAddr = Start + NameOffset.second;
1088 if (std::error_code EC = readFuncProfile(Start: FuncProfileAddr, Profiles))
1089 return EC;
1090 }
1091 } else {
1092 assert(!useFuncOffsetList());
1093 for (auto Name : FuncsToUse) {
1094
1095 auto iter = FuncOffsetTable.find(Val: MD5Hash(Str: Name));
1096 if (iter == FuncOffsetTable.end())
1097 continue;
1098 const uint8_t *FuncProfileAddr = Start + iter->second;
1099 if (std::error_code EC = readFuncProfile(Start: FuncProfileAddr, Profiles))
1100 return EC;
1101 }
1102 }
1103
1104 return sampleprof_error::success;
1105}
1106
1107std::error_code SampleProfileReaderExtBinaryBase::readFuncProfiles() {
1108 // Collect functions used by current module if the Reader has been
1109 // given a module.
1110 // collectFuncsFromModule uses FunctionSamples::getCanonicalFnName
1111 // which will query FunctionSamples::HasUniqSuffix, so it has to be
1112 // called after FunctionSamples::HasUniqSuffix is set, i.e. after
1113 // NameTable section is read.
1114 bool LoadFuncsToBeUsed = collectFuncsFromModule();
1115
1116 // When LoadFuncsToBeUsed is false, we are using LLVM tool, need to read all
1117 // profiles.
1118 if (!LoadFuncsToBeUsed) {
1119 while (Data < End) {
1120 if (std::error_code EC = readFuncProfile(Start: Data))
1121 return EC;
1122 }
1123 assert(Data == End && "More data is read than expected");
1124 } else {
1125 // Load function profiles on demand.
1126 if (std::error_code EC = readFuncProfiles(FuncsToUse, Profiles))
1127 return EC;
1128 Data = End;
1129 }
1130 assert((CSProfileCount == 0 || CSProfileCount == Profiles.size()) &&
1131 "Cannot have both context-sensitive and regular profile");
1132 assert((!CSProfileCount || ProfileIsCS) &&
1133 "Section flag should be consistent with actual profile");
1134 return sampleprof_error::success;
1135}
1136
1137std::error_code SampleProfileReaderExtBinaryBase::readProfileSymbolList() {
1138 if (!ProfSymList)
1139 ProfSymList = std::make_unique<ProfileSymbolList>();
1140
1141 if (std::error_code EC = ProfSymList->read(Data, ListSize: End - Data))
1142 return EC;
1143
1144 Data = End;
1145 return sampleprof_error::success;
1146}
1147
1148std::error_code SampleProfileReaderExtBinaryBase::decompressSection(
1149 const uint8_t *SecStart, const uint64_t SecSize,
1150 const uint8_t *&DecompressBuf, uint64_t &DecompressBufSize) {
1151 Data = SecStart;
1152 End = SecStart + SecSize;
1153 auto DecompressSize = readNumber<uint64_t>();
1154 if (std::error_code EC = DecompressSize.getError())
1155 return EC;
1156 DecompressBufSize = *DecompressSize;
1157
1158 auto CompressSize = readNumber<uint64_t>();
1159 if (std::error_code EC = CompressSize.getError())
1160 return EC;
1161
1162 if (!llvm::compression::zlib::isAvailable())
1163 return sampleprof_error::zlib_unavailable;
1164
1165 uint8_t *Buffer = Allocator.Allocate<uint8_t>(Num: DecompressBufSize);
1166 size_t UCSize = DecompressBufSize;
1167 llvm::Error E = compression::zlib::decompress(Input: ArrayRef(Data, *CompressSize),
1168 Output: Buffer, UncompressedSize&: UCSize);
1169 if (E)
1170 return sampleprof_error::uncompress_failed;
1171 DecompressBuf = reinterpret_cast<const uint8_t *>(Buffer);
1172 return sampleprof_error::success;
1173}
1174
1175std::error_code SampleProfileReaderExtBinaryBase::readImpl() {
1176 const uint8_t *BufStart =
1177 reinterpret_cast<const uint8_t *>(Buffer->getBufferStart());
1178
1179 for (auto &Entry : SecHdrTable) {
1180 // Skip empty section.
1181 if (!Entry.Size)
1182 continue;
1183
1184 // Skip sections without inlined functions when SkipFlatProf is true.
1185 if (SkipFlatProf && hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagFlat))
1186 continue;
1187
1188 const uint8_t *SecStart = BufStart + Entry.Offset;
1189 uint64_t SecSize = Entry.Size;
1190
1191 // If the section is compressed, decompress it into a buffer
1192 // DecompressBuf before reading the actual data. The pointee of
1193 // 'Data' will be changed to buffer hold by DecompressBuf
1194 // temporarily when reading the actual data.
1195 bool isCompressed = hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagCompress);
1196 if (isCompressed) {
1197 const uint8_t *DecompressBuf;
1198 uint64_t DecompressBufSize;
1199 if (std::error_code EC = decompressSection(
1200 SecStart, SecSize, DecompressBuf, DecompressBufSize))
1201 return EC;
1202 SecStart = DecompressBuf;
1203 SecSize = DecompressBufSize;
1204 }
1205
1206 if (std::error_code EC = readOneSection(Start: SecStart, Size: SecSize, Entry))
1207 return EC;
1208 if (Data != SecStart + SecSize)
1209 return sampleprof_error::malformed;
1210
1211 // Change the pointee of 'Data' from DecompressBuf to original Buffer.
1212 if (isCompressed) {
1213 Data = BufStart + Entry.Offset;
1214 End = BufStart + Buffer->getBufferSize();
1215 }
1216 }
1217
1218 return sampleprof_error::success;
1219}
1220
1221std::error_code SampleProfileReaderRawBinary::verifySPMagic(uint64_t Magic) {
1222 if (Magic == SPMagic())
1223 return sampleprof_error::success;
1224 return sampleprof_error::bad_magic;
1225}
1226
1227std::error_code SampleProfileReaderExtBinary::verifySPMagic(uint64_t Magic) {
1228 if (Magic == SPMagic(Format: SPF_Ext_Binary))
1229 return sampleprof_error::success;
1230 return sampleprof_error::bad_magic;
1231}
1232
1233std::error_code SampleProfileReaderBinary::readNameTable() {
1234 auto Size = readNumber<size_t>();
1235 if (std::error_code EC = Size.getError())
1236 return EC;
1237
1238 // Normally if useMD5 is true, the name table should have MD5 values, not
1239 // strings, however in the case that ExtBinary profile has multiple name
1240 // tables mixing string and MD5, all of them have to be normalized to use MD5,
1241 // because optimization passes can only handle either type.
1242 bool UseMD5 = useMD5();
1243
1244 NameTable.clear();
1245 NameTable.reserve(n: *Size);
1246 if (!ProfileIsCS) {
1247 MD5SampleContextTable.clear();
1248 if (UseMD5)
1249 MD5SampleContextTable.reserve(n: *Size);
1250 else
1251 // If we are using strings, delay MD5 computation since only a portion of
1252 // names are used by top level functions. Use 0 to indicate MD5 value is
1253 // to be calculated as no known string has a MD5 value of 0.
1254 MD5SampleContextTable.resize(new_size: *Size);
1255 }
1256 for (size_t I = 0; I < *Size; ++I) {
1257 auto Name(readString());
1258 if (std::error_code EC = Name.getError())
1259 return EC;
1260 if (UseMD5) {
1261 FunctionId FID(*Name);
1262 if (!ProfileIsCS)
1263 MD5SampleContextTable.emplace_back(args: FID.getHashCode());
1264 NameTable.emplace_back(args&: FID);
1265 } else
1266 NameTable.push_back(x: FunctionId(*Name));
1267 }
1268 if (!ProfileIsCS)
1269 MD5SampleContextStart = MD5SampleContextTable.data();
1270 return sampleprof_error::success;
1271}
1272
1273std::error_code
1274SampleProfileReaderExtBinaryBase::readNameTableSec(bool IsMD5,
1275 bool FixedLengthMD5) {
1276 if (FixedLengthMD5) {
1277 if (!IsMD5)
1278 errs() << "If FixedLengthMD5 is true, UseMD5 has to be true";
1279 auto Size = readNumber<size_t>();
1280 if (std::error_code EC = Size.getError())
1281 return EC;
1282
1283 assert(Data + (*Size) * sizeof(uint64_t) == End &&
1284 "Fixed length MD5 name table does not contain specified number of "
1285 "entries");
1286 if (Data + (*Size) * sizeof(uint64_t) > End)
1287 return sampleprof_error::truncated;
1288
1289 NameTable.clear();
1290 NameTable.reserve(n: *Size);
1291 for (size_t I = 0; I < *Size; ++I) {
1292 using namespace support;
1293 uint64_t FID = endian::read<uint64_t, unaligned>(
1294 memory: Data + I * sizeof(uint64_t), endian: endianness::little);
1295 NameTable.emplace_back(args: FunctionId(FID));
1296 }
1297 if (!ProfileIsCS)
1298 MD5SampleContextStart = reinterpret_cast<const uint64_t *>(Data);
1299 Data = Data + (*Size) * sizeof(uint64_t);
1300 return sampleprof_error::success;
1301 }
1302
1303 if (IsMD5) {
1304 assert(!FixedLengthMD5 && "FixedLengthMD5 should be unreachable here");
1305 auto Size = readNumber<size_t>();
1306 if (std::error_code EC = Size.getError())
1307 return EC;
1308
1309 NameTable.clear();
1310 NameTable.reserve(n: *Size);
1311 if (!ProfileIsCS)
1312 MD5SampleContextTable.resize(new_size: *Size);
1313 for (size_t I = 0; I < *Size; ++I) {
1314 auto FID = readNumber<uint64_t>();
1315 if (std::error_code EC = FID.getError())
1316 return EC;
1317 if (!ProfileIsCS)
1318 support::endian::write64le(P: &MD5SampleContextTable[I], V: *FID);
1319 NameTable.emplace_back(args: FunctionId(*FID));
1320 }
1321 if (!ProfileIsCS)
1322 MD5SampleContextStart = MD5SampleContextTable.data();
1323 return sampleprof_error::success;
1324 }
1325
1326 return SampleProfileReaderBinary::readNameTable();
1327}
1328
1329// Read in the CS name table section, which basically contains a list of context
1330// vectors. Each element of a context vector, aka a frame, refers to the
1331// underlying raw function names that are stored in the name table, as well as
1332// a callsite identifier that only makes sense for non-leaf frames.
1333std::error_code SampleProfileReaderExtBinaryBase::readCSNameTableSec() {
1334 auto Size = readNumber<size_t>();
1335 if (std::error_code EC = Size.getError())
1336 return EC;
1337
1338 CSNameTable.clear();
1339 CSNameTable.reserve(n: *Size);
1340 if (ProfileIsCS) {
1341 // Delay MD5 computation of CS context until they are needed. Use 0 to
1342 // indicate MD5 value is to be calculated as no known string has a MD5
1343 // value of 0.
1344 MD5SampleContextTable.clear();
1345 MD5SampleContextTable.resize(new_size: *Size);
1346 MD5SampleContextStart = MD5SampleContextTable.data();
1347 }
1348 for (size_t I = 0; I < *Size; ++I) {
1349 CSNameTable.emplace_back(args: SampleContextFrameVector());
1350 auto ContextSize = readNumber<uint32_t>();
1351 if (std::error_code EC = ContextSize.getError())
1352 return EC;
1353 for (uint32_t J = 0; J < *ContextSize; ++J) {
1354 auto FName(readStringFromTable());
1355 if (std::error_code EC = FName.getError())
1356 return EC;
1357 auto LineOffset = readNumber<uint64_t>();
1358 if (std::error_code EC = LineOffset.getError())
1359 return EC;
1360
1361 if (!isOffsetLegal(L: *LineOffset))
1362 return sampleprof_error::illegal_line_offset;
1363
1364 auto Discriminator = readNumber<uint64_t>();
1365 if (std::error_code EC = Discriminator.getError())
1366 return EC;
1367
1368 CSNameTable.back().emplace_back(
1369 Args&: FName.get(), Args: LineLocation(LineOffset.get(), Discriminator.get()));
1370 }
1371 }
1372
1373 return sampleprof_error::success;
1374}
1375
1376std::error_code
1377SampleProfileReaderExtBinaryBase::readFuncMetadata(bool ProfileHasAttribute,
1378 FunctionSamples *FProfile) {
1379 if (Data < End) {
1380 if (ProfileIsProbeBased) {
1381 auto Checksum = readNumber<uint64_t>();
1382 if (std::error_code EC = Checksum.getError())
1383 return EC;
1384 if (FProfile)
1385 FProfile->setFunctionHash(*Checksum);
1386 }
1387
1388 if (ProfileHasAttribute) {
1389 auto Attributes = readNumber<uint32_t>();
1390 if (std::error_code EC = Attributes.getError())
1391 return EC;
1392 if (FProfile)
1393 FProfile->getContext().setAllAttributes(*Attributes);
1394 }
1395
1396 if (!ProfileIsCS) {
1397 // Read all the attributes for inlined function calls.
1398 auto NumCallsites = readNumber<uint32_t>();
1399 if (std::error_code EC = NumCallsites.getError())
1400 return EC;
1401
1402 for (uint32_t J = 0; J < *NumCallsites; ++J) {
1403 auto LineOffset = readNumber<uint64_t>();
1404 if (std::error_code EC = LineOffset.getError())
1405 return EC;
1406
1407 auto Discriminator = readNumber<uint64_t>();
1408 if (std::error_code EC = Discriminator.getError())
1409 return EC;
1410
1411 auto FContextHash(readSampleContextFromTable());
1412 if (std::error_code EC = FContextHash.getError())
1413 return EC;
1414
1415 auto &[FContext, Hash] = *FContextHash;
1416 FunctionSamples *CalleeProfile = nullptr;
1417 if (FProfile) {
1418 CalleeProfile = const_cast<FunctionSamples *>(
1419 &FProfile->functionSamplesAt(Loc: LineLocation(
1420 *LineOffset,
1421 *Discriminator))[FContext.getFunction()]);
1422 }
1423 if (std::error_code EC =
1424 readFuncMetadata(ProfileHasAttribute, FProfile: CalleeProfile))
1425 return EC;
1426 }
1427 }
1428 }
1429
1430 return sampleprof_error::success;
1431}
1432
1433std::error_code SampleProfileReaderExtBinaryBase::readFuncMetadata(
1434 bool ProfileHasAttribute, DenseSet<FunctionSamples *> &Profiles) {
1435 if (FuncMetadataIndex.empty())
1436 return sampleprof_error::success;
1437
1438 for (auto *FProfile : Profiles) {
1439 auto R = FuncMetadataIndex.find(x: FProfile->getContext().getHashCode());
1440 if (R == FuncMetadataIndex.end())
1441 continue;
1442
1443 Data = R->second.first;
1444 End = R->second.second;
1445 if (std::error_code EC = readFuncMetadata(ProfileHasAttribute, FProfile))
1446 return EC;
1447 assert(Data == End && "More data is read than expected");
1448 }
1449 return sampleprof_error::success;
1450}
1451
1452std::error_code
1453SampleProfileReaderExtBinaryBase::readFuncMetadata(bool ProfileHasAttribute) {
1454 while (Data < End) {
1455 auto FContextHash(readSampleContextFromTable());
1456 if (std::error_code EC = FContextHash.getError())
1457 return EC;
1458 auto &[FContext, Hash] = *FContextHash;
1459 FunctionSamples *FProfile = nullptr;
1460 auto It = Profiles.find(Ctx: FContext);
1461 if (It != Profiles.end())
1462 FProfile = &It->second;
1463
1464 const uint8_t *Start = Data;
1465 if (std::error_code EC = readFuncMetadata(ProfileHasAttribute, FProfile))
1466 return EC;
1467
1468 FuncMetadataIndex[FContext.getHashCode()] = {Start, Data};
1469 }
1470
1471 assert(Data == End && "More data is read than expected");
1472 return sampleprof_error::success;
1473}
1474
1475std::error_code
1476SampleProfileReaderExtBinaryBase::readSecHdrTableEntry(uint64_t Idx) {
1477 SecHdrTableEntry Entry;
1478 auto Type = readUnencodedNumber<uint64_t>();
1479 if (std::error_code EC = Type.getError())
1480 return EC;
1481 Entry.Type = static_cast<SecType>(*Type);
1482
1483 auto Flags = readUnencodedNumber<uint64_t>();
1484 if (std::error_code EC = Flags.getError())
1485 return EC;
1486 Entry.Flags = *Flags;
1487
1488 auto Offset = readUnencodedNumber<uint64_t>();
1489 if (std::error_code EC = Offset.getError())
1490 return EC;
1491 Entry.Offset = *Offset;
1492
1493 auto Size = readUnencodedNumber<uint64_t>();
1494 if (std::error_code EC = Size.getError())
1495 return EC;
1496 Entry.Size = *Size;
1497
1498 Entry.LayoutIndex = Idx;
1499 SecHdrTable.push_back(x: std::move(Entry));
1500 return sampleprof_error::success;
1501}
1502
1503std::error_code SampleProfileReaderExtBinaryBase::readSecHdrTable() {
1504 auto EntryNum = readUnencodedNumber<uint64_t>();
1505 if (std::error_code EC = EntryNum.getError())
1506 return EC;
1507
1508 for (uint64_t i = 0; i < (*EntryNum); i++)
1509 if (std::error_code EC = readSecHdrTableEntry(Idx: i))
1510 return EC;
1511
1512 return sampleprof_error::success;
1513}
1514
1515std::error_code SampleProfileReaderExtBinaryBase::readHeader() {
1516 const uint8_t *BufStart =
1517 reinterpret_cast<const uint8_t *>(Buffer->getBufferStart());
1518 Data = BufStart;
1519 End = BufStart + Buffer->getBufferSize();
1520
1521 if (std::error_code EC = readMagicIdent())
1522 return EC;
1523
1524 if (std::error_code EC = readSecHdrTable())
1525 return EC;
1526
1527 return sampleprof_error::success;
1528}
1529
1530uint64_t SampleProfileReaderExtBinaryBase::getSectionSize(SecType Type) {
1531 uint64_t Size = 0;
1532 for (auto &Entry : SecHdrTable) {
1533 if (Entry.Type == Type)
1534 Size += Entry.Size;
1535 }
1536 return Size;
1537}
1538
1539uint64_t SampleProfileReaderExtBinaryBase::getFileSize() {
1540 // Sections in SecHdrTable is not necessarily in the same order as
1541 // sections in the profile because section like FuncOffsetTable needs
1542 // to be written after section LBRProfile but needs to be read before
1543 // section LBRProfile, so we cannot simply use the last entry in
1544 // SecHdrTable to calculate the file size.
1545 uint64_t FileSize = 0;
1546 for (auto &Entry : SecHdrTable) {
1547 FileSize = std::max(a: Entry.Offset + Entry.Size, b: FileSize);
1548 }
1549 return FileSize;
1550}
1551
1552static std::string getSecFlagsStr(const SecHdrTableEntry &Entry) {
1553 std::string Flags;
1554 if (hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagCompress))
1555 Flags.append(s: "{compressed,");
1556 else
1557 Flags.append(s: "{");
1558
1559 if (hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagFlat))
1560 Flags.append(s: "flat,");
1561
1562 switch (Entry.Type) {
1563 case SecNameTable:
1564 if (hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagFixedLengthMD5))
1565 Flags.append(s: "fixlenmd5,");
1566 else if (hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagMD5Name))
1567 Flags.append(s: "md5,");
1568 if (hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagUniqSuffix))
1569 Flags.append(s: "uniq,");
1570 break;
1571 case SecProfSummary:
1572 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagPartial))
1573 Flags.append(s: "partial,");
1574 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagFullContext))
1575 Flags.append(s: "context,");
1576 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagIsPreInlined))
1577 Flags.append(s: "preInlined,");
1578 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagFSDiscriminator))
1579 Flags.append(s: "fs-discriminator,");
1580 break;
1581 case SecFuncOffsetTable:
1582 if (hasSecFlag(Entry, Flag: SecFuncOffsetFlags::SecFlagOrdered))
1583 Flags.append(s: "ordered,");
1584 break;
1585 case SecFuncMetadata:
1586 if (hasSecFlag(Entry, Flag: SecFuncMetadataFlags::SecFlagIsProbeBased))
1587 Flags.append(s: "probe,");
1588 if (hasSecFlag(Entry, Flag: SecFuncMetadataFlags::SecFlagHasAttribute))
1589 Flags.append(s: "attr,");
1590 break;
1591 default:
1592 break;
1593 }
1594 char &last = Flags.back();
1595 if (last == ',')
1596 last = '}';
1597 else
1598 Flags.append(s: "}");
1599 return Flags;
1600}
1601
1602bool SampleProfileReaderExtBinaryBase::dumpSectionInfo(raw_ostream &OS) {
1603 uint64_t TotalSecsSize = 0;
1604 for (auto &Entry : SecHdrTable) {
1605 OS << getSecName(Type: Entry.Type) << " - Offset: " << Entry.Offset
1606 << ", Size: " << Entry.Size << ", Flags: " << getSecFlagsStr(Entry)
1607 << "\n";
1608 ;
1609 TotalSecsSize += Entry.Size;
1610 }
1611 uint64_t HeaderSize = SecHdrTable.front().Offset;
1612 assert(HeaderSize + TotalSecsSize == getFileSize() &&
1613 "Size of 'header + sections' doesn't match the total size of profile");
1614
1615 OS << "Header Size: " << HeaderSize << "\n";
1616 OS << "Total Sections Size: " << TotalSecsSize << "\n";
1617 OS << "File Size: " << getFileSize() << "\n";
1618 return true;
1619}
1620
1621std::error_code SampleProfileReaderBinary::readMagicIdent() {
1622 // Read and check the magic identifier.
1623 auto Magic = readNumber<uint64_t>();
1624 if (std::error_code EC = Magic.getError())
1625 return EC;
1626 else if (std::error_code EC = verifySPMagic(Magic: *Magic))
1627 return EC;
1628
1629 // Read the version number.
1630 auto Version = readNumber<uint64_t>();
1631 if (std::error_code EC = Version.getError())
1632 return EC;
1633 else if (*Version != SPVersion())
1634 return sampleprof_error::unsupported_version;
1635
1636 return sampleprof_error::success;
1637}
1638
1639std::error_code SampleProfileReaderBinary::readHeader() {
1640 Data = reinterpret_cast<const uint8_t *>(Buffer->getBufferStart());
1641 End = Data + Buffer->getBufferSize();
1642
1643 if (std::error_code EC = readMagicIdent())
1644 return EC;
1645
1646 if (std::error_code EC = readSummary())
1647 return EC;
1648
1649 if (std::error_code EC = readNameTable())
1650 return EC;
1651 return sampleprof_error::success;
1652}
1653
1654std::error_code SampleProfileReaderBinary::readSummaryEntry(
1655 std::vector<ProfileSummaryEntry> &Entries) {
1656 auto Cutoff = readNumber<uint64_t>();
1657 if (std::error_code EC = Cutoff.getError())
1658 return EC;
1659
1660 auto MinBlockCount = readNumber<uint64_t>();
1661 if (std::error_code EC = MinBlockCount.getError())
1662 return EC;
1663
1664 auto NumBlocks = readNumber<uint64_t>();
1665 if (std::error_code EC = NumBlocks.getError())
1666 return EC;
1667
1668 Entries.emplace_back(args&: *Cutoff, args&: *MinBlockCount, args&: *NumBlocks);
1669 return sampleprof_error::success;
1670}
1671
1672std::error_code SampleProfileReaderBinary::readSummary() {
1673 auto TotalCount = readNumber<uint64_t>();
1674 if (std::error_code EC = TotalCount.getError())
1675 return EC;
1676
1677 auto MaxBlockCount = readNumber<uint64_t>();
1678 if (std::error_code EC = MaxBlockCount.getError())
1679 return EC;
1680
1681 auto MaxFunctionCount = readNumber<uint64_t>();
1682 if (std::error_code EC = MaxFunctionCount.getError())
1683 return EC;
1684
1685 auto NumBlocks = readNumber<uint64_t>();
1686 if (std::error_code EC = NumBlocks.getError())
1687 return EC;
1688
1689 auto NumFunctions = readNumber<uint64_t>();
1690 if (std::error_code EC = NumFunctions.getError())
1691 return EC;
1692
1693 auto NumSummaryEntries = readNumber<uint64_t>();
1694 if (std::error_code EC = NumSummaryEntries.getError())
1695 return EC;
1696
1697 std::vector<ProfileSummaryEntry> Entries;
1698 for (unsigned i = 0; i < *NumSummaryEntries; i++) {
1699 std::error_code EC = readSummaryEntry(Entries);
1700 if (EC != sampleprof_error::success)
1701 return EC;
1702 }
1703 Summary = std::make_unique<ProfileSummary>(
1704 args: ProfileSummary::PSK_Sample, args&: Entries, args&: *TotalCount, args&: *MaxBlockCount, args: 0,
1705 args&: *MaxFunctionCount, args&: *NumBlocks, args&: *NumFunctions);
1706
1707 return sampleprof_error::success;
1708}
1709
1710bool SampleProfileReaderRawBinary::hasFormat(const MemoryBuffer &Buffer) {
1711 const uint8_t *Data =
1712 reinterpret_cast<const uint8_t *>(Buffer.getBufferStart());
1713 uint64_t Magic = decodeULEB128(p: Data);
1714 return Magic == SPMagic();
1715}
1716
1717bool SampleProfileReaderExtBinary::hasFormat(const MemoryBuffer &Buffer) {
1718 const uint8_t *Data =
1719 reinterpret_cast<const uint8_t *>(Buffer.getBufferStart());
1720 uint64_t Magic = decodeULEB128(p: Data);
1721 return Magic == SPMagic(Format: SPF_Ext_Binary);
1722}
1723
1724std::error_code SampleProfileReaderGCC::skipNextWord() {
1725 uint32_t dummy;
1726 if (!GcovBuffer.readInt(Val&: dummy))
1727 return sampleprof_error::truncated;
1728 return sampleprof_error::success;
1729}
1730
1731template <typename T> ErrorOr<T> SampleProfileReaderGCC::readNumber() {
1732 if (sizeof(T) <= sizeof(uint32_t)) {
1733 uint32_t Val;
1734 if (GcovBuffer.readInt(Val) && Val <= std::numeric_limits<T>::max())
1735 return static_cast<T>(Val);
1736 } else if (sizeof(T) <= sizeof(uint64_t)) {
1737 uint64_t Val;
1738 if (GcovBuffer.readInt64(Val) && Val <= std::numeric_limits<T>::max())
1739 return static_cast<T>(Val);
1740 }
1741
1742 std::error_code EC = sampleprof_error::malformed;
1743 reportError(LineNumber: 0, Msg: EC.message());
1744 return EC;
1745}
1746
1747ErrorOr<StringRef> SampleProfileReaderGCC::readString() {
1748 StringRef Str;
1749 if (!GcovBuffer.readString(str&: Str))
1750 return sampleprof_error::truncated;
1751 return Str;
1752}
1753
1754std::error_code SampleProfileReaderGCC::readHeader() {
1755 // Read the magic identifier.
1756 if (!GcovBuffer.readGCDAFormat())
1757 return sampleprof_error::unrecognized_format;
1758
1759 // Read the version number. Note - the GCC reader does not validate this
1760 // version, but the profile creator generates v704.
1761 GCOV::GCOVVersion version;
1762 if (!GcovBuffer.readGCOVVersion(version))
1763 return sampleprof_error::unrecognized_format;
1764
1765 if (version != GCOV::V407)
1766 return sampleprof_error::unsupported_version;
1767
1768 // Skip the empty integer.
1769 if (std::error_code EC = skipNextWord())
1770 return EC;
1771
1772 return sampleprof_error::success;
1773}
1774
1775std::error_code SampleProfileReaderGCC::readSectionTag(uint32_t Expected) {
1776 uint32_t Tag;
1777 if (!GcovBuffer.readInt(Val&: Tag))
1778 return sampleprof_error::truncated;
1779
1780 if (Tag != Expected)
1781 return sampleprof_error::malformed;
1782
1783 if (std::error_code EC = skipNextWord())
1784 return EC;
1785
1786 return sampleprof_error::success;
1787}
1788
1789std::error_code SampleProfileReaderGCC::readNameTable() {
1790 if (std::error_code EC = readSectionTag(Expected: GCOVTagAFDOFileNames))
1791 return EC;
1792
1793 uint32_t Size;
1794 if (!GcovBuffer.readInt(Val&: Size))
1795 return sampleprof_error::truncated;
1796
1797 for (uint32_t I = 0; I < Size; ++I) {
1798 StringRef Str;
1799 if (!GcovBuffer.readString(str&: Str))
1800 return sampleprof_error::truncated;
1801 Names.push_back(x: std::string(Str));
1802 }
1803
1804 return sampleprof_error::success;
1805}
1806
1807std::error_code SampleProfileReaderGCC::readFunctionProfiles() {
1808 if (std::error_code EC = readSectionTag(Expected: GCOVTagAFDOFunction))
1809 return EC;
1810
1811 uint32_t NumFunctions;
1812 if (!GcovBuffer.readInt(Val&: NumFunctions))
1813 return sampleprof_error::truncated;
1814
1815 InlineCallStack Stack;
1816 for (uint32_t I = 0; I < NumFunctions; ++I)
1817 if (std::error_code EC = readOneFunctionProfile(InlineStack: Stack, Update: true, Offset: 0))
1818 return EC;
1819
1820 computeSummary();
1821 return sampleprof_error::success;
1822}
1823
1824std::error_code SampleProfileReaderGCC::readOneFunctionProfile(
1825 const InlineCallStack &InlineStack, bool Update, uint32_t Offset) {
1826 uint64_t HeadCount = 0;
1827 if (InlineStack.size() == 0)
1828 if (!GcovBuffer.readInt64(Val&: HeadCount))
1829 return sampleprof_error::truncated;
1830
1831 uint32_t NameIdx;
1832 if (!GcovBuffer.readInt(Val&: NameIdx))
1833 return sampleprof_error::truncated;
1834
1835 StringRef Name(Names[NameIdx]);
1836
1837 uint32_t NumPosCounts;
1838 if (!GcovBuffer.readInt(Val&: NumPosCounts))
1839 return sampleprof_error::truncated;
1840
1841 uint32_t NumCallsites;
1842 if (!GcovBuffer.readInt(Val&: NumCallsites))
1843 return sampleprof_error::truncated;
1844
1845 FunctionSamples *FProfile = nullptr;
1846 if (InlineStack.size() == 0) {
1847 // If this is a top function that we have already processed, do not
1848 // update its profile again. This happens in the presence of
1849 // function aliases. Since these aliases share the same function
1850 // body, there will be identical replicated profiles for the
1851 // original function. In this case, we simply not bother updating
1852 // the profile of the original function.
1853 FProfile = &Profiles[FunctionId(Name)];
1854 FProfile->addHeadSamples(Num: HeadCount);
1855 if (FProfile->getTotalSamples() > 0)
1856 Update = false;
1857 } else {
1858 // Otherwise, we are reading an inlined instance. The top of the
1859 // inline stack contains the profile of the caller. Insert this
1860 // callee in the caller's CallsiteMap.
1861 FunctionSamples *CallerProfile = InlineStack.front();
1862 uint32_t LineOffset = Offset >> 16;
1863 uint32_t Discriminator = Offset & 0xffff;
1864 FProfile = &CallerProfile->functionSamplesAt(
1865 Loc: LineLocation(LineOffset, Discriminator))[FunctionId(Name)];
1866 }
1867 FProfile->setFunction(FunctionId(Name));
1868
1869 for (uint32_t I = 0; I < NumPosCounts; ++I) {
1870 uint32_t Offset;
1871 if (!GcovBuffer.readInt(Val&: Offset))
1872 return sampleprof_error::truncated;
1873
1874 uint32_t NumTargets;
1875 if (!GcovBuffer.readInt(Val&: NumTargets))
1876 return sampleprof_error::truncated;
1877
1878 uint64_t Count;
1879 if (!GcovBuffer.readInt64(Val&: Count))
1880 return sampleprof_error::truncated;
1881
1882 // The line location is encoded in the offset as:
1883 // high 16 bits: line offset to the start of the function.
1884 // low 16 bits: discriminator.
1885 uint32_t LineOffset = Offset >> 16;
1886 uint32_t Discriminator = Offset & 0xffff;
1887
1888 InlineCallStack NewStack;
1889 NewStack.push_back(Elt: FProfile);
1890 llvm::append_range(C&: NewStack, R: InlineStack);
1891 if (Update) {
1892 // Walk up the inline stack, adding the samples on this line to
1893 // the total sample count of the callers in the chain.
1894 for (auto *CallerProfile : NewStack)
1895 CallerProfile->addTotalSamples(Num: Count);
1896
1897 // Update the body samples for the current profile.
1898 FProfile->addBodySamples(LineOffset, Discriminator, Num: Count);
1899 }
1900
1901 // Process the list of functions called at an indirect call site.
1902 // These are all the targets that a function pointer (or virtual
1903 // function) resolved at runtime.
1904 for (uint32_t J = 0; J < NumTargets; J++) {
1905 uint32_t HistVal;
1906 if (!GcovBuffer.readInt(Val&: HistVal))
1907 return sampleprof_error::truncated;
1908
1909 if (HistVal != HIST_TYPE_INDIR_CALL_TOPN)
1910 return sampleprof_error::malformed;
1911
1912 uint64_t TargetIdx;
1913 if (!GcovBuffer.readInt64(Val&: TargetIdx))
1914 return sampleprof_error::truncated;
1915 StringRef TargetName(Names[TargetIdx]);
1916
1917 uint64_t TargetCount;
1918 if (!GcovBuffer.readInt64(Val&: TargetCount))
1919 return sampleprof_error::truncated;
1920
1921 if (Update)
1922 FProfile->addCalledTargetSamples(LineOffset, Discriminator,
1923 Func: FunctionId(TargetName),
1924 Num: TargetCount);
1925 }
1926 }
1927
1928 // Process all the inlined callers into the current function. These
1929 // are all the callsites that were inlined into this function.
1930 for (uint32_t I = 0; I < NumCallsites; I++) {
1931 // The offset is encoded as:
1932 // high 16 bits: line offset to the start of the function.
1933 // low 16 bits: discriminator.
1934 uint32_t Offset;
1935 if (!GcovBuffer.readInt(Val&: Offset))
1936 return sampleprof_error::truncated;
1937 InlineCallStack NewStack;
1938 NewStack.push_back(Elt: FProfile);
1939 llvm::append_range(C&: NewStack, R: InlineStack);
1940 if (std::error_code EC = readOneFunctionProfile(InlineStack: NewStack, Update, Offset))
1941 return EC;
1942 }
1943
1944 return sampleprof_error::success;
1945}
1946
1947/// Read a GCC AutoFDO profile.
1948///
1949/// This format is generated by the Linux Perf conversion tool at
1950/// https://github.com/google/autofdo.
1951std::error_code SampleProfileReaderGCC::readImpl() {
1952 assert(!ProfileIsFSDisciminator && "Gcc profiles not support FSDisciminator");
1953 // Read the string table.
1954 if (std::error_code EC = readNameTable())
1955 return EC;
1956
1957 // Read the source profile.
1958 if (std::error_code EC = readFunctionProfiles())
1959 return EC;
1960
1961 return sampleprof_error::success;
1962}
1963
1964bool SampleProfileReaderGCC::hasFormat(const MemoryBuffer &Buffer) {
1965 StringRef Magic(Buffer.getBufferStart());
1966 return Magic == "adcg*704";
1967}
1968
1969void SampleProfileReaderItaniumRemapper::applyRemapping(LLVMContext &Ctx) {
1970 // If the reader uses MD5 to represent string, we can't remap it because
1971 // we don't know what the original function names were.
1972 if (Reader.useMD5()) {
1973 Ctx.diagnose(DI: DiagnosticInfoSampleProfile(
1974 Reader.getBuffer()->getBufferIdentifier(),
1975 "Profile data remapping cannot be applied to profile data "
1976 "using MD5 names (original mangled names are not available).",
1977 DS_Warning));
1978 return;
1979 }
1980
1981 // CSSPGO-TODO: Remapper is not yet supported.
1982 // We will need to remap the entire context string.
1983 assert(Remappings && "should be initialized while creating remapper");
1984 for (auto &Sample : Reader.getProfiles()) {
1985 DenseSet<FunctionId> NamesInSample;
1986 Sample.second.findAllNames(NameSet&: NamesInSample);
1987 for (auto &Name : NamesInSample) {
1988 StringRef NameStr = Name.stringRef();
1989 if (auto Key = Remappings->insert(FunctionName: NameStr))
1990 NameMap.insert(KV: {Key, NameStr});
1991 }
1992 }
1993
1994 RemappingApplied = true;
1995}
1996
1997std::optional<StringRef>
1998SampleProfileReaderItaniumRemapper::lookUpNameInProfile(StringRef Fname) {
1999 if (auto Key = Remappings->lookup(FunctionName: Fname)) {
2000 StringRef Result = NameMap.lookup(Val: Key);
2001 if (!Result.empty())
2002 return Result;
2003 }
2004 return std::nullopt;
2005}
2006
2007/// Prepare a memory buffer for the contents of \p Filename.
2008///
2009/// \returns an error code indicating the status of the buffer.
2010static ErrorOr<std::unique_ptr<MemoryBuffer>>
2011setupMemoryBuffer(const Twine &Filename, vfs::FileSystem &FS) {
2012 auto BufferOrErr = Filename.str() == "-" ? MemoryBuffer::getSTDIN()
2013 : FS.getBufferForFile(Name: Filename);
2014 if (std::error_code EC = BufferOrErr.getError())
2015 return EC;
2016 auto Buffer = std::move(BufferOrErr.get());
2017
2018 return std::move(Buffer);
2019}
2020
2021/// Create a sample profile reader based on the format of the input file.
2022///
2023/// \param Filename The file to open.
2024///
2025/// \param C The LLVM context to use to emit diagnostics.
2026///
2027/// \param P The FSDiscriminatorPass.
2028///
2029/// \param RemapFilename The file used for profile remapping.
2030///
2031/// \returns an error code indicating the status of the created reader.
2032ErrorOr<std::unique_ptr<SampleProfileReader>>
2033SampleProfileReader::create(StringRef Filename, LLVMContext &C,
2034 vfs::FileSystem &FS, FSDiscriminatorPass P,
2035 StringRef RemapFilename) {
2036 auto BufferOrError = setupMemoryBuffer(Filename, FS);
2037 if (std::error_code EC = BufferOrError.getError())
2038 return EC;
2039 return create(B&: BufferOrError.get(), C, FS, P, RemapFilename);
2040}
2041
2042/// Create a sample profile remapper from the given input, to remap the
2043/// function names in the given profile data.
2044///
2045/// \param Filename The file to open.
2046///
2047/// \param Reader The profile reader the remapper is going to be applied to.
2048///
2049/// \param C The LLVM context to use to emit diagnostics.
2050///
2051/// \returns an error code indicating the status of the created reader.
2052ErrorOr<std::unique_ptr<SampleProfileReaderItaniumRemapper>>
2053SampleProfileReaderItaniumRemapper::create(StringRef Filename,
2054 vfs::FileSystem &FS,
2055 SampleProfileReader &Reader,
2056 LLVMContext &C) {
2057 auto BufferOrError = setupMemoryBuffer(Filename, FS);
2058 if (std::error_code EC = BufferOrError.getError())
2059 return EC;
2060 return create(B&: BufferOrError.get(), Reader, C);
2061}
2062
2063/// Create a sample profile remapper from the given input, to remap the
2064/// function names in the given profile data.
2065///
2066/// \param B The memory buffer to create the reader from (assumes ownership).
2067///
2068/// \param C The LLVM context to use to emit diagnostics.
2069///
2070/// \param Reader The profile reader the remapper is going to be applied to.
2071///
2072/// \returns an error code indicating the status of the created reader.
2073ErrorOr<std::unique_ptr<SampleProfileReaderItaniumRemapper>>
2074SampleProfileReaderItaniumRemapper::create(std::unique_ptr<MemoryBuffer> &B,
2075 SampleProfileReader &Reader,
2076 LLVMContext &C) {
2077 auto Remappings = std::make_unique<SymbolRemappingReader>();
2078 if (Error E = Remappings->read(B&: *B)) {
2079 handleAllErrors(
2080 E: std::move(E), Handlers: [&](const SymbolRemappingParseError &ParseError) {
2081 C.diagnose(DI: DiagnosticInfoSampleProfile(B->getBufferIdentifier(),
2082 ParseError.getLineNum(),
2083 ParseError.getMessage()));
2084 });
2085 return sampleprof_error::malformed;
2086 }
2087
2088 return std::make_unique<SampleProfileReaderItaniumRemapper>(
2089 args: std::move(B), args: std::move(Remappings), args&: Reader);
2090}
2091
2092/// Create a sample profile reader based on the format of the input data.
2093///
2094/// \param B The memory buffer to create the reader from (assumes ownership).
2095///
2096/// \param C The LLVM context to use to emit diagnostics.
2097///
2098/// \param P The FSDiscriminatorPass.
2099///
2100/// \param RemapFilename The file used for profile remapping.
2101///
2102/// \returns an error code indicating the status of the created reader.
2103ErrorOr<std::unique_ptr<SampleProfileReader>>
2104SampleProfileReader::create(std::unique_ptr<MemoryBuffer> &B, LLVMContext &C,
2105 vfs::FileSystem &FS, FSDiscriminatorPass P,
2106 StringRef RemapFilename) {
2107 std::unique_ptr<SampleProfileReader> Reader;
2108 if (SampleProfileReaderRawBinary::hasFormat(Buffer: *B))
2109 Reader.reset(p: new SampleProfileReaderRawBinary(std::move(B), C));
2110 else if (SampleProfileReaderExtBinary::hasFormat(Buffer: *B))
2111 Reader.reset(p: new SampleProfileReaderExtBinary(std::move(B), C));
2112 else if (SampleProfileReaderGCC::hasFormat(Buffer: *B))
2113 Reader.reset(p: new SampleProfileReaderGCC(std::move(B), C));
2114 else if (SampleProfileReaderText::hasFormat(Buffer: *B))
2115 Reader.reset(p: new SampleProfileReaderText(std::move(B), C));
2116 else
2117 return sampleprof_error::unrecognized_format;
2118
2119 if (!RemapFilename.empty()) {
2120 auto ReaderOrErr = SampleProfileReaderItaniumRemapper::create(
2121 Filename: RemapFilename, FS, Reader&: *Reader, C);
2122 if (std::error_code EC = ReaderOrErr.getError()) {
2123 std::string Msg = "Could not create remapper: " + EC.message();
2124 C.diagnose(DI: DiagnosticInfoSampleProfile(RemapFilename, Msg));
2125 return EC;
2126 }
2127 Reader->Remapper = std::move(ReaderOrErr.get());
2128 }
2129
2130 if (std::error_code EC = Reader->readHeader()) {
2131 return EC;
2132 }
2133
2134 Reader->setDiscriminatorMaskedBitFrom(P);
2135
2136 return std::move(Reader);
2137}
2138
2139// For text and GCC file formats, we compute the summary after reading the
2140// profile. Binary format has the profile summary in its header.
2141void SampleProfileReader::computeSummary() {
2142 SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs);
2143 Summary = Builder.computeSummaryForProfiles(Profiles);
2144}
2145