1//===- SampleProfReader.cpp - Read LLVM sample profile data ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the class that reads LLVM sample profiles. It
10// supports three file formats: text, binary and gcov.
11//
12// The textual representation is useful for debugging and testing purposes. The
13// binary representation is more compact, resulting in smaller file sizes.
14//
15// The gcov encoding is the one generated by GCC's AutoFDO profile creation
16// tool (https://github.com/google/autofdo)
17//
18// All three encodings can be used interchangeably as an input sample profile.
19//
20//===----------------------------------------------------------------------===//
21
22#include "llvm/ProfileData/SampleProfReader.h"
23#include "llvm/ADT/DenseMap.h"
24#include "llvm/ADT/STLExtras.h"
25#include "llvm/ADT/StringRef.h"
26#include "llvm/IR/Module.h"
27#include "llvm/IR/ProfileSummary.h"
28#include "llvm/ProfileData/ProfileCommon.h"
29#include "llvm/ProfileData/SampleProf.h"
30#include "llvm/Support/CommandLine.h"
31#include "llvm/Support/Compression.h"
32#include "llvm/Support/ErrorOr.h"
33#include "llvm/Support/JSON.h"
34#include "llvm/Support/LEB128.h"
35#include "llvm/Support/LineIterator.h"
36#include "llvm/Support/MD5.h"
37#include "llvm/Support/MemoryBuffer.h"
38#include "llvm/Support/VirtualFileSystem.h"
39#include "llvm/Support/raw_ostream.h"
40#include <algorithm>
41#include <cstddef>
42#include <cstdint>
43#include <limits>
44#include <memory>
45#include <system_error>
46#include <vector>
47
48using namespace llvm;
49using namespace sampleprof;
50
51#define DEBUG_TYPE "samplepgo-reader"
52
53// This internal option specifies if the profile uses FS discriminators.
54// It only applies to text, and binary format profiles.
55// For ext-binary format profiles, the flag is set in the summary.
56static cl::opt<bool> ProfileIsFSDisciminator(
57 "profile-isfs", cl::Hidden, cl::init(Val: false),
58 cl::desc("Profile uses flow sensitive discriminators"));
59
60static cl::opt<bool>
61 LazyLoadNameTable("sample-profile-lazy-load-name-table", cl::init(Val: true),
62 cl::Hidden,
63 cl::desc("Lazy load the name table from the profile."));
64
65/// Dump the function profile for \p FName.
66///
67/// \param FContext Name + context of the function to print.
68/// \param OS Stream to emit the output to.
69void SampleProfileReader::dumpFunctionProfile(const FunctionSamples &FS,
70 raw_ostream &OS) {
71 OS << "Function: " << FS.getContext().toString() << ": " << FS;
72}
73
74/// Dump all the function profiles found on stream \p OS.
75void SampleProfileReader::dump(raw_ostream &OS) {
76 std::vector<NameFunctionSamples> V;
77 sortFuncProfiles(ProfileMap: Profiles, SortedProfiles&: V);
78 for (const auto &I : V)
79 dumpFunctionProfile(FS: *I.second, OS);
80}
81
82static void dumpFunctionProfileJson(const FunctionSamples &S,
83 json::OStream &JOS, bool TopLevel = false) {
84 auto DumpBody = [&](const BodySampleMap &BodySamples) {
85 for (const auto &I : BodySamples) {
86 const LineLocation &Loc = I.first;
87 const SampleRecord &Sample = I.second;
88 JOS.object(Contents: [&] {
89 JOS.attribute(Key: "line", Contents: Loc.LineOffset);
90 if (Loc.Discriminator)
91 JOS.attribute(Key: "discriminator", Contents: Loc.Discriminator);
92 JOS.attribute(Key: "samples", Contents: Sample.getSamples());
93
94 auto CallTargets = Sample.getSortedCallTargets();
95 if (!CallTargets.empty()) {
96 JOS.attributeArray(Key: "calls", Contents: [&] {
97 for (const auto &J : CallTargets) {
98 JOS.object(Contents: [&] {
99 JOS.attribute(Key: "function", Contents: J.first.str());
100 JOS.attribute(Key: "samples", Contents: J.second);
101 });
102 }
103 });
104 }
105 });
106 }
107 };
108
109 auto DumpCallsiteSamples = [&](const CallsiteSampleMap &CallsiteSamples) {
110 for (const auto &I : CallsiteSamples)
111 for (const auto &FS : I.second) {
112 const LineLocation &Loc = I.first;
113 const FunctionSamples &CalleeSamples = FS.second;
114 JOS.object(Contents: [&] {
115 JOS.attribute(Key: "line", Contents: Loc.LineOffset);
116 if (Loc.Discriminator)
117 JOS.attribute(Key: "discriminator", Contents: Loc.Discriminator);
118 JOS.attributeArray(
119 Key: "samples", Contents: [&] { dumpFunctionProfileJson(S: CalleeSamples, JOS); });
120 });
121 }
122 };
123
124 JOS.object(Contents: [&] {
125 JOS.attribute(Key: "name", Contents: S.getFunction().str());
126 JOS.attribute(Key: "total", Contents: S.getTotalSamples());
127 if (TopLevel)
128 JOS.attribute(Key: "head", Contents: S.getHeadSamples());
129
130 const auto &BodySamples = S.getBodySamples();
131 if (!BodySamples.empty())
132 JOS.attributeArray(Key: "body", Contents: [&] { DumpBody(BodySamples); });
133
134 const auto &CallsiteSamples = S.getCallsiteSamples();
135 if (!CallsiteSamples.empty())
136 JOS.attributeArray(Key: "callsites",
137 Contents: [&] { DumpCallsiteSamples(CallsiteSamples); });
138 });
139}
140
141/// Dump all the function profiles found on stream \p OS in the JSON format.
142void SampleProfileReader::dumpJson(raw_ostream &OS) {
143 std::vector<NameFunctionSamples> V;
144 sortFuncProfiles(ProfileMap: Profiles, SortedProfiles&: V);
145 json::OStream JOS(OS, 2);
146 JOS.arrayBegin();
147 for (const auto &F : V)
148 dumpFunctionProfileJson(S: *F.second, JOS, TopLevel: true);
149 JOS.arrayEnd();
150
151 // Emit a newline character at the end as json::OStream doesn't emit one.
152 OS << "\n";
153}
154
155/// Parse \p Input as function head.
156///
157/// Parse one line of \p Input, and update function name in \p FName,
158/// function's total sample count in \p NumSamples, function's entry
159/// count in \p NumHeadSamples.
160///
161/// \returns true if parsing is successful.
162static bool ParseHead(const StringRef &Input, StringRef &FName,
163 uint64_t &NumSamples, uint64_t &NumHeadSamples) {
164 if (Input[0] == ' ')
165 return false;
166 size_t n2 = Input.rfind(C: ':');
167 size_t n1 = Input.rfind(C: ':', From: n2 - 1);
168 FName = Input.substr(Start: 0, N: n1);
169 if (Input.substr(Start: n1 + 1, N: n2 - n1 - 1).getAsInteger(Radix: 10, Result&: NumSamples))
170 return false;
171 if (Input.substr(Start: n2 + 1).getAsInteger(Radix: 10, Result&: NumHeadSamples))
172 return false;
173 return true;
174}
175
176/// Returns true if line offset \p L is legal (only has 16 bits).
177static bool isOffsetLegal(unsigned L) { return (L & 0xffff) == L; }
178
179/// Parse \p Input that contains metadata.
180/// Possible metadata:
181/// - CFG Checksum information:
182/// !CFGChecksum: 12345
183/// - CFG Checksum information:
184/// !Attributes: 1
185/// Stores the FunctionHash (a.k.a. CFG Checksum) into \p FunctionHash.
186static bool parseMetadata(const StringRef &Input, uint64_t &FunctionHash,
187 uint32_t &Attributes) {
188 if (Input.starts_with(Prefix: "!CFGChecksum:")) {
189 StringRef CFGInfo = Input.substr(Start: strlen(s: "!CFGChecksum:")).trim();
190 return !CFGInfo.getAsInteger(Radix: 10, Result&: FunctionHash);
191 }
192
193 if (Input.starts_with(Prefix: "!Attributes:")) {
194 StringRef Attrib = Input.substr(Start: strlen(s: "!Attributes:")).trim();
195 return !Attrib.getAsInteger(Radix: 10, Result&: Attributes);
196 }
197
198 return false;
199}
200
201enum class LineType {
202 CallSiteProfile,
203 BodyProfile,
204 Metadata,
205 VirtualCallTypeProfile,
206};
207
208// Parse `Input` as a white-space separated list of `vtable:count` pairs. An
209// example input line is `_ZTVbar:1471 _ZTVfoo:630`.
210static bool parseTypeCountMap(StringRef Input,
211 DenseMap<StringRef, uint64_t> &TypeCountMap) {
212 for (size_t Index = Input.find_first_not_of(C: ' '); Index != StringRef::npos;) {
213 size_t ColonIndex = Input.find(C: ':', From: Index);
214 if (ColonIndex == StringRef::npos)
215 return false; // No colon found, invalid format.
216 StringRef TypeName = Input.substr(Start: Index, N: ColonIndex - Index);
217 // CountIndex is the start index of count.
218 size_t CountStartIndex = ColonIndex + 1;
219 // NextIndex is the start index after the 'target:count' pair.
220 size_t NextIndex = Input.find_first_of(C: ' ', From: CountStartIndex);
221 uint64_t Count;
222 if (Input.substr(Start: CountStartIndex, N: NextIndex - CountStartIndex)
223 .getAsInteger(Radix: 10, Result&: Count))
224 return false; // Invalid count.
225 // Error on duplicated type names in one line of input.
226 auto [Iter, Inserted] = TypeCountMap.insert(KV: {TypeName, Count});
227 if (!Inserted)
228 return false;
229 Index = (NextIndex == StringRef::npos)
230 ? StringRef::npos
231 : Input.find_first_not_of(C: ' ', From: NextIndex);
232 }
233 return true;
234}
235
236/// Parse \p Input as line sample.
237///
238/// \param Input input line.
239/// \param LineTy Type of this line.
240/// \param Depth the depth of the inline stack.
241/// \param NumSamples total samples of the line/inlined callsite.
242/// \param LineOffset line offset to the start of the function.
243/// \param Discriminator discriminator of the line.
244/// \param TargetCountMap map from indirect call target to count.
245/// \param FunctionHash the function's CFG hash, used by pseudo probe.
246///
247/// returns true if parsing is successful.
248static bool ParseLine(const StringRef &Input, LineType &LineTy, uint32_t &Depth,
249 uint64_t &NumSamples, uint32_t &LineOffset,
250 uint32_t &Discriminator, StringRef &CalleeName,
251 DenseMap<StringRef, uint64_t> &TargetCountMap,
252 DenseMap<StringRef, uint64_t> &TypeCountMap,
253 uint64_t &FunctionHash, uint32_t &Attributes,
254 bool &IsFlat) {
255 for (Depth = 0; Input[Depth] == ' '; Depth++)
256 ;
257 if (Depth == 0)
258 return false;
259
260 if (Input[Depth] == '!') {
261 LineTy = LineType::Metadata;
262 // This metadata is only for manual inspection only. We already created a
263 // FunctionSamples and put it in the profile map, so there is no point
264 // to skip profiles even they have no use for ThinLTO.
265 if (Input == StringRef(" !Flat")) {
266 IsFlat = true;
267 return true;
268 }
269 return parseMetadata(Input: Input.substr(Start: Depth), FunctionHash, Attributes);
270 }
271
272 size_t n1 = Input.find(C: ':');
273 StringRef Loc = Input.substr(Start: Depth, N: n1 - Depth);
274 size_t n2 = Loc.find(C: '.');
275 if (n2 == StringRef::npos) {
276 if (Loc.getAsInteger(Radix: 10, Result&: LineOffset) || !isOffsetLegal(L: LineOffset))
277 return false;
278 Discriminator = 0;
279 } else {
280 if (Loc.substr(Start: 0, N: n2).getAsInteger(Radix: 10, Result&: LineOffset))
281 return false;
282 if (Loc.substr(Start: n2 + 1).getAsInteger(Radix: 10, Result&: Discriminator))
283 return false;
284 }
285
286 StringRef Rest = Input.substr(Start: n1 + 2);
287 if (isDigit(C: Rest[0])) {
288 LineTy = LineType::BodyProfile;
289 size_t n3 = Rest.find(C: ' ');
290 if (n3 == StringRef::npos) {
291 if (Rest.getAsInteger(Radix: 10, Result&: NumSamples))
292 return false;
293 } else {
294 if (Rest.substr(Start: 0, N: n3).getAsInteger(Radix: 10, Result&: NumSamples))
295 return false;
296 }
297 // Find call targets and their sample counts.
298 // Note: In some cases, there are symbols in the profile which are not
299 // mangled. To accommodate such cases, use colon + integer pairs as the
300 // anchor points.
301 // An example:
302 // _M_construct<char *>:1000 string_view<std::allocator<char> >:437
303 // ":1000" and ":437" are used as anchor points so the string above will
304 // be interpreted as
305 // target: _M_construct<char *>
306 // count: 1000
307 // target: string_view<std::allocator<char> >
308 // count: 437
309 while (n3 != StringRef::npos) {
310 n3 += Rest.substr(Start: n3).find_first_not_of(C: ' ');
311 Rest = Rest.substr(Start: n3);
312 n3 = Rest.find_first_of(C: ':');
313 if (n3 == StringRef::npos || n3 == 0)
314 return false;
315
316 StringRef Target;
317 uint64_t count, n4;
318 while (true) {
319 // Get the segment after the current colon.
320 StringRef AfterColon = Rest.substr(Start: n3 + 1);
321 // Get the target symbol before the current colon.
322 Target = Rest.substr(Start: 0, N: n3);
323 // Check if the word after the current colon is an integer.
324 n4 = AfterColon.find_first_of(C: ' ');
325 n4 = (n4 != StringRef::npos) ? n3 + n4 + 1 : Rest.size();
326 StringRef WordAfterColon = Rest.substr(Start: n3 + 1, N: n4 - n3 - 1);
327 if (!WordAfterColon.getAsInteger(Radix: 10, Result&: count))
328 break;
329
330 // Try to find the next colon.
331 uint64_t n5 = AfterColon.find_first_of(C: ':');
332 if (n5 == StringRef::npos)
333 return false;
334 n3 += n5 + 1;
335 }
336
337 // An anchor point is found. Save the {target, count} pair
338 TargetCountMap[Target] = count;
339 if (n4 == Rest.size())
340 break;
341 // Change n3 to the next blank space after colon + integer pair.
342 n3 = n4;
343 }
344 } else if (Rest.starts_with(Prefix: kVTableProfPrefix)) {
345 LineTy = LineType::VirtualCallTypeProfile;
346 return parseTypeCountMap(Input: Rest.substr(Start: strlen(s: kVTableProfPrefix)),
347 TypeCountMap);
348 } else {
349 LineTy = LineType::CallSiteProfile;
350 size_t n3 = Rest.find_last_of(C: ':');
351 CalleeName = Rest.substr(Start: 0, N: n3);
352 if (Rest.substr(Start: n3 + 1).getAsInteger(Radix: 10, Result&: NumSamples))
353 return false;
354 }
355 return true;
356}
357
358/// Load samples from a text file.
359///
360/// See the documentation at the top of the file for an explanation of
361/// the expected format.
362///
363/// \returns true if the file was loaded successfully, false otherwise.
364std::error_code SampleProfileReaderText::readImpl() {
365 line_iterator LineIt(*Buffer, /*SkipBlanks=*/true, '#');
366 sampleprof_error Result = sampleprof_error::success;
367
368 InlineCallStack InlineStack;
369 uint32_t TopLevelProbeProfileCount = 0;
370
371 // DepthMetadata tracks whether we have processed metadata for the current
372 // top-level or nested function profile.
373 uint32_t DepthMetadata = 0;
374
375 std::vector<SampleContext *> FlatSamples;
376
377 ProfileIsFS = ProfileIsFSDisciminator;
378 FunctionSamples::ProfileIsFS = ProfileIsFS;
379 for (; !LineIt.is_at_eof(); ++LineIt) {
380 size_t pos = LineIt->find_first_not_of(C: ' ');
381 if (pos == LineIt->npos || (*LineIt)[pos] == '#')
382 continue;
383 // Read the header of each function.
384 //
385 // Note that for function identifiers we are actually expecting
386 // mangled names, but we may not always get them. This happens when
387 // the compiler decides not to emit the function (e.g., it was inlined
388 // and removed). In this case, the binary will not have the linkage
389 // name for the function, so the profiler will emit the function's
390 // unmangled name, which may contain characters like ':' and '>' in its
391 // name (member functions, templates, etc).
392 //
393 // The only requirement we place on the identifier, then, is that it
394 // should not begin with a number.
395 if ((*LineIt)[0] != ' ') {
396 uint64_t NumSamples, NumHeadSamples;
397 StringRef FName;
398 if (!ParseHead(Input: *LineIt, FName, NumSamples, NumHeadSamples)) {
399 reportError(LineNumber: LineIt.line_number(),
400 Msg: "Expected 'mangled_name:NUM:NUM', found " + *LineIt);
401 return sampleprof_error::malformed;
402 }
403 DepthMetadata = 0;
404 SampleContext FContext(FName, CSNameTable);
405 if (FContext.hasContext())
406 ++CSProfileCount;
407 FunctionSamples &FProfile = Profiles.create(Ctx: FContext);
408 mergeSampleProfErrors(Accumulator&: Result, Result: FProfile.addTotalSamples(Num: NumSamples));
409 mergeSampleProfErrors(Accumulator&: Result, Result: FProfile.addHeadSamples(Num: NumHeadSamples));
410 InlineStack.clear();
411 InlineStack.push_back(Elt: &FProfile);
412 } else {
413 uint64_t NumSamples;
414 StringRef FName;
415 DenseMap<StringRef, uint64_t> TargetCountMap;
416 DenseMap<StringRef, uint64_t> TypeCountMap;
417 uint32_t Depth, LineOffset, Discriminator;
418 LineType LineTy = LineType::BodyProfile;
419 uint64_t FunctionHash = 0;
420 uint32_t Attributes = 0;
421 bool IsFlat = false;
422 // TODO: Update ParseLine to return an error code instead of a bool and
423 // report it.
424 if (!ParseLine(Input: *LineIt, LineTy, Depth, NumSamples, LineOffset,
425 Discriminator, CalleeName&: FName, TargetCountMap, TypeCountMap,
426 FunctionHash, Attributes, IsFlat)) {
427 switch (LineTy) {
428 case LineType::Metadata:
429 reportError(LineNumber: LineIt.line_number(),
430 Msg: "Cannot parse metadata: " + *LineIt);
431 break;
432 case LineType::VirtualCallTypeProfile:
433 reportError(LineNumber: LineIt.line_number(),
434 Msg: "Expected 'vtables [mangled_vtable:NUM]+', found " +
435 *LineIt);
436 break;
437 default:
438 reportError(LineNumber: LineIt.line_number(),
439 Msg: "Expected 'NUM[.NUM]: NUM[ mangled_name:NUM]*', found " +
440 *LineIt);
441 }
442 return sampleprof_error::malformed;
443 }
444 if (LineTy != LineType::Metadata && Depth == DepthMetadata) {
445 // Metadata must be put at the end of a function profile.
446 reportError(LineNumber: LineIt.line_number(),
447 Msg: "Found non-metadata after metadata: " + *LineIt);
448 return sampleprof_error::malformed;
449 }
450
451 // Here we handle FS discriminators.
452 Discriminator &= getDiscriminatorMask();
453
454 while (InlineStack.size() > Depth) {
455 InlineStack.pop_back();
456 }
457 switch (LineTy) {
458 case LineType::CallSiteProfile: {
459 FunctionSamples &FSamples = InlineStack.back()->functionSamplesAt(
460 Loc: LineLocation(LineOffset, Discriminator))[FunctionId(FName)];
461 FSamples.setFunction(FunctionId(FName));
462 mergeSampleProfErrors(Accumulator&: Result, Result: FSamples.addTotalSamples(Num: NumSamples));
463 InlineStack.push_back(Elt: &FSamples);
464 DepthMetadata = 0;
465 break;
466 }
467
468 case LineType::VirtualCallTypeProfile: {
469 mergeSampleProfErrors(
470 Accumulator&: Result, Result: InlineStack.back()->addCallsiteVTableTypeProfAt(
471 Loc: LineLocation(LineOffset, Discriminator), Other: TypeCountMap));
472 break;
473 }
474
475 case LineType::BodyProfile: {
476 FunctionSamples &FProfile = *InlineStack.back();
477 for (const auto &name_count : TargetCountMap) {
478 mergeSampleProfErrors(Accumulator&: Result, Result: FProfile.addCalledTargetSamples(
479 LineOffset, Discriminator,
480 Func: FunctionId(name_count.first),
481 Num: name_count.second));
482 }
483 mergeSampleProfErrors(
484 Accumulator&: Result,
485 Result: FProfile.addBodySamples(LineOffset, Discriminator, Num: NumSamples));
486 break;
487 }
488 case LineType::Metadata: {
489 FunctionSamples &FProfile = *InlineStack.back();
490 if (FunctionHash) {
491 FProfile.setFunctionHash(FunctionHash);
492 if (Depth == 1)
493 ++TopLevelProbeProfileCount;
494 }
495 FProfile.getContext().setAllAttributes(Attributes);
496 if (Attributes & (uint32_t)ContextShouldBeInlined)
497 ProfileIsPreInlined = true;
498 DepthMetadata = Depth;
499 if (IsFlat) {
500 if (Depth == 1)
501 FlatSamples.push_back(x: &FProfile.getContext());
502 else
503 Ctx.diagnose(DI: DiagnosticInfoSampleProfile(
504 Buffer->getBufferIdentifier(), LineIt.line_number(),
505 "!Flat may only be used at top level function.", DS_Warning));
506 }
507 break;
508 }
509 }
510 }
511 }
512
513 // Honor the option to skip flat functions. Since they are already added to
514 // the profile map, remove them all here.
515 if (SkipFlatProf)
516 for (SampleContext *FlatSample : FlatSamples)
517 Profiles.erase(Ctx: *FlatSample);
518
519 assert((CSProfileCount == 0 || CSProfileCount == Profiles.size()) &&
520 "Cannot have both context-sensitive and regular profile");
521 ProfileIsCS = (CSProfileCount > 0);
522 assert((TopLevelProbeProfileCount == 0 ||
523 TopLevelProbeProfileCount == Profiles.size()) &&
524 "Cannot have both probe-based profiles and regular profiles");
525 ProfileIsProbeBased = (TopLevelProbeProfileCount > 0);
526 FunctionSamples::ProfileIsProbeBased = ProfileIsProbeBased;
527 FunctionSamples::ProfileIsCS = ProfileIsCS;
528 FunctionSamples::ProfileIsPreInlined = ProfileIsPreInlined;
529
530 if (Result == sampleprof_error::success)
531 computeSummary();
532
533 return Result;
534}
535
536bool SampleProfileReaderText::hasFormat(const MemoryBuffer &Buffer) {
537 bool result = false;
538
539 // Check that the first non-comment line is a valid function header.
540 line_iterator LineIt(Buffer, /*SkipBlanks=*/true, '#');
541 if (!LineIt.is_at_eof()) {
542 if ((*LineIt)[0] != ' ') {
543 uint64_t NumSamples, NumHeadSamples;
544 StringRef FName;
545 result = ParseHead(Input: *LineIt, FName, NumSamples, NumHeadSamples);
546 }
547 }
548
549 return result;
550}
551
552template <typename T> ErrorOr<T> SampleProfileReaderBinary::readNumber() {
553 unsigned NumBytesRead = 0;
554 uint64_t Val = decodeULEB128(p: Data, n: &NumBytesRead);
555
556 if (Val > std::numeric_limits<T>::max()) {
557 std::error_code EC = sampleprof_error::malformed;
558 reportError(LineNumber: 0, Msg: EC.message());
559 return EC;
560 } else if (Data + NumBytesRead > End) {
561 std::error_code EC = sampleprof_error::truncated;
562 reportError(LineNumber: 0, Msg: EC.message());
563 return EC;
564 }
565
566 Data += NumBytesRead;
567 return static_cast<T>(Val);
568}
569
570ErrorOr<StringRef> SampleProfileReaderBinary::readString() {
571 StringRef Str(reinterpret_cast<const char *>(Data));
572 if (Data + Str.size() + 1 > End) {
573 std::error_code EC = sampleprof_error::truncated;
574 reportError(LineNumber: 0, Msg: EC.message());
575 return EC;
576 }
577
578 Data += Str.size() + 1;
579 return Str;
580}
581
582template <typename T>
583ErrorOr<T> SampleProfileReaderBinary::readUnencodedNumber() {
584 if (Data + sizeof(T) > End) {
585 std::error_code EC = sampleprof_error::truncated;
586 reportError(LineNumber: 0, Msg: EC.message());
587 return EC;
588 }
589
590 using namespace support;
591 T Val = endian::readNext<T, llvm::endianness::little>(Data);
592 return Val;
593}
594
595template <typename T>
596inline ErrorOr<size_t> SampleProfileReaderBinary::readStringIndex(T &Table) {
597 auto Idx = readNumber<size_t>();
598 if (std::error_code EC = Idx.getError())
599 return EC;
600 if (*Idx >= Table.size())
601 return sampleprof_error::truncated_name_table;
602 return *Idx;
603}
604
605ErrorOr<FunctionId>
606SampleProfileReaderBinary::readStringFromTable(size_t *RetIdx) {
607 auto Idx = readStringIndex(Table&: NameTable);
608 if (std::error_code EC = Idx.getError())
609 return EC;
610 if (RetIdx)
611 *RetIdx = *Idx;
612 return NameTable[*Idx];
613}
614
615ErrorOr<SampleContextFrames>
616SampleProfileReaderBinary::readContextFromTable(size_t *RetIdx) {
617 auto ContextIdx = readNumber<size_t>();
618 if (std::error_code EC = ContextIdx.getError())
619 return EC;
620 if (*ContextIdx >= CSNameTable.size())
621 return sampleprof_error::truncated_name_table;
622 if (RetIdx)
623 *RetIdx = *ContextIdx;
624 return CSNameTable[*ContextIdx];
625}
626
627ErrorOr<std::pair<SampleContext, uint64_t>>
628SampleProfileReaderBinary::readSampleContextFromTable() {
629 SampleContext Context;
630 size_t Idx;
631 if (ProfileIsCS) {
632 auto FContext(readContextFromTable(RetIdx: &Idx));
633 if (std::error_code EC = FContext.getError())
634 return EC;
635 Context = SampleContext(*FContext);
636 } else {
637 auto FName(readStringFromTable(RetIdx: &Idx));
638 if (std::error_code EC = FName.getError())
639 return EC;
640 Context = SampleContext(*FName);
641 }
642 // Since MD5SampleContextStart may point to the profile's file data, need to
643 // make sure it is reading the same value on big endian CPU.
644 uint64_t Hash = support::endian::read64le(P: MD5SampleContextStart + Idx);
645 // Lazy computing of hash value, write back to the table to cache it. Only
646 // compute the context's hash value if it is being referenced for the first
647 // time.
648 if (Hash == 0) {
649 assert(MD5SampleContextStart == MD5SampleContextTable.data());
650 Hash = Context.getHashCode();
651 support::endian::write64le(P: &MD5SampleContextTable[Idx], V: Hash);
652 }
653 return std::make_pair(x&: Context, y&: Hash);
654}
655
656std::error_code
657SampleProfileReaderBinary::readVTableTypeCountMap(TypeCountMap &M) {
658 auto NumVTableTypes = readNumber<uint32_t>();
659 if (std::error_code EC = NumVTableTypes.getError())
660 return EC;
661
662 for (uint32_t I = 0; I < *NumVTableTypes; ++I) {
663 auto VTableType(readStringFromTable());
664 if (std::error_code EC = VTableType.getError())
665 return EC;
666
667 auto VTableSamples = readNumber<uint64_t>();
668 if (std::error_code EC = VTableSamples.getError())
669 return EC;
670 // The source profile should not have duplicate vtable records at the same
671 // location. In case duplicate vtables are found, reader can emit a warning
672 // but continue processing the profile.
673 if (!M.insert(x: std::make_pair(x&: *VTableType, y&: *VTableSamples)).second) {
674 Ctx.diagnose(DI: DiagnosticInfoSampleProfile(
675 Buffer->getBufferIdentifier(), 0,
676 "Duplicate vtable type " + VTableType->str() +
677 " at the same location. Additional counters will be ignored.",
678 DS_Warning));
679 continue;
680 }
681 }
682 return sampleprof_error::success;
683}
684
685std::error_code
686SampleProfileReaderBinary::readCallsiteVTableProf(FunctionSamples &FProfile) {
687 assert(ReadVTableProf &&
688 "Cannot read vtable profiles if ReadVTableProf is false");
689
690 // Read the vtable type profile for the callsite.
691 auto NumCallsites = readNumber<uint32_t>();
692 if (std::error_code EC = NumCallsites.getError())
693 return EC;
694
695 for (uint32_t I = 0; I < *NumCallsites; ++I) {
696 auto LineOffset = readNumber<uint64_t>();
697 if (std::error_code EC = LineOffset.getError())
698 return EC;
699
700 if (!isOffsetLegal(L: *LineOffset))
701 return sampleprof_error::illegal_line_offset;
702
703 auto Discriminator = readNumber<uint64_t>();
704 if (std::error_code EC = Discriminator.getError())
705 return EC;
706
707 // Here we handle FS discriminators:
708 const uint32_t DiscriminatorVal = (*Discriminator) & getDiscriminatorMask();
709
710 if (std::error_code EC = readVTableTypeCountMap(M&: FProfile.getTypeSamplesAt(
711 Loc: LineLocation(*LineOffset, DiscriminatorVal))))
712 return EC;
713 }
714 return sampleprof_error::success;
715}
716
717std::error_code
718SampleProfileReaderBinary::readProfile(FunctionSamples &FProfile) {
719 auto NumSamples = readNumber<uint64_t>();
720 if (std::error_code EC = NumSamples.getError())
721 return EC;
722 FProfile.addTotalSamples(Num: *NumSamples);
723
724 // Read the samples in the body.
725 auto NumRecords = readNumber<uint32_t>();
726 if (std::error_code EC = NumRecords.getError())
727 return EC;
728
729 for (uint32_t I = 0; I < *NumRecords; ++I) {
730 auto LineOffset = readNumber<uint64_t>();
731 if (std::error_code EC = LineOffset.getError())
732 return EC;
733
734 if (!isOffsetLegal(L: *LineOffset)) {
735 return sampleprof_error::illegal_line_offset;
736 }
737
738 auto Discriminator = readNumber<uint64_t>();
739 if (std::error_code EC = Discriminator.getError())
740 return EC;
741
742 auto NumSamples = readNumber<uint64_t>();
743 if (std::error_code EC = NumSamples.getError())
744 return EC;
745
746 auto NumCalls = readNumber<uint32_t>();
747 if (std::error_code EC = NumCalls.getError())
748 return EC;
749
750 // Here we handle FS discriminators:
751 uint32_t DiscriminatorVal = (*Discriminator) & getDiscriminatorMask();
752
753 for (uint32_t J = 0; J < *NumCalls; ++J) {
754 auto CalledFunction(readStringFromTable());
755 if (std::error_code EC = CalledFunction.getError())
756 return EC;
757
758 auto CalledFunctionSamples = readNumber<uint64_t>();
759 if (std::error_code EC = CalledFunctionSamples.getError())
760 return EC;
761
762 FProfile.addCalledTargetSamples(LineOffset: *LineOffset, Discriminator: DiscriminatorVal,
763 Func: *CalledFunction, Num: *CalledFunctionSamples);
764 }
765
766 FProfile.addBodySamples(LineOffset: *LineOffset, Discriminator: DiscriminatorVal, Num: *NumSamples);
767 }
768
769 // Read all the samples for inlined function calls.
770 auto NumCallsites = readNumber<uint32_t>();
771 if (std::error_code EC = NumCallsites.getError())
772 return EC;
773
774 for (uint32_t J = 0; J < *NumCallsites; ++J) {
775 auto LineOffset = readNumber<uint64_t>();
776 if (std::error_code EC = LineOffset.getError())
777 return EC;
778
779 auto Discriminator = readNumber<uint64_t>();
780 if (std::error_code EC = Discriminator.getError())
781 return EC;
782
783 auto FName(readStringFromTable());
784 if (std::error_code EC = FName.getError())
785 return EC;
786
787 // Here we handle FS discriminators:
788 uint32_t DiscriminatorVal = (*Discriminator) & getDiscriminatorMask();
789
790 FunctionSamples &CalleeProfile = FProfile.functionSamplesAt(
791 Loc: LineLocation(*LineOffset, DiscriminatorVal))[*FName];
792 CalleeProfile.setFunction(*FName);
793 if (std::error_code EC = readProfile(FProfile&: CalleeProfile))
794 return EC;
795 }
796
797 if (ReadVTableProf)
798 return readCallsiteVTableProf(FProfile);
799
800 return sampleprof_error::success;
801}
802
803std::error_code
804SampleProfileReaderBinary::readFuncProfile(const uint8_t *Start,
805 SampleProfileMap &Profiles) {
806 Data = Start;
807 auto NumHeadSamples = readNumber<uint64_t>();
808 if (std::error_code EC = NumHeadSamples.getError())
809 return EC;
810
811 auto FContextHash(readSampleContextFromTable());
812 if (std::error_code EC = FContextHash.getError())
813 return EC;
814
815 auto &[FContext, Hash] = *FContextHash;
816 // Use the cached hash value for insertion instead of recalculating it.
817 auto Res = Profiles.try_emplace(Hash, Key: FContext, Args: FunctionSamples());
818 FunctionSamples &FProfile = Res.first->second;
819 FProfile.setContext(FContext);
820 FProfile.addHeadSamples(Num: *NumHeadSamples);
821
822 if (FContext.hasContext())
823 CSProfileCount++;
824
825 if (std::error_code EC = readProfile(FProfile))
826 return EC;
827 return sampleprof_error::success;
828}
829
830std::error_code
831SampleProfileReaderBinary::readFuncProfile(const uint8_t *Start) {
832 return readFuncProfile(Start, Profiles);
833}
834
835std::error_code SampleProfileReaderBinary::readImpl() {
836 ProfileIsFS = ProfileIsFSDisciminator;
837 FunctionSamples::ProfileIsFS = ProfileIsFS;
838 while (Data < End) {
839 if (std::error_code EC = readFuncProfile(Start: Data))
840 return EC;
841 }
842
843 return sampleprof_error::success;
844}
845
846std::error_code SampleProfileReaderExtBinaryBase::readOneSection(
847 const uint8_t *Start, uint64_t Size, const SecHdrTableEntry &Entry) {
848 Data = Start;
849 End = Start + Size;
850 switch (Entry.Type) {
851 case SecProfSummary:
852 if (std::error_code EC = readSummary())
853 return EC;
854 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagPartial))
855 Summary->setPartialProfile(true);
856 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagFullContext))
857 FunctionSamples::ProfileIsCS = ProfileIsCS = true;
858 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagIsPreInlined))
859 FunctionSamples::ProfileIsPreInlined = ProfileIsPreInlined = true;
860 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagFSDiscriminator))
861 FunctionSamples::ProfileIsFS = ProfileIsFS = true;
862 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagHasVTableTypeProf))
863 ReadVTableProf = true;
864 break;
865 case SecNameTable: {
866 bool FixedLengthMD5 =
867 hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagFixedLengthMD5);
868 bool UseMD5 = hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagMD5Name);
869 // UseMD5 means if THIS section uses MD5, ProfileIsMD5 means if the entire
870 // profile uses MD5 for function name matching in IPO passes.
871 ProfileIsMD5 = ProfileIsMD5 || UseMD5;
872 FunctionSamples::HasUniqSuffix =
873 hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagUniqSuffix);
874 if (std::error_code EC = readNameTableSec(IsMD5: UseMD5, FixedLengthMD5))
875 return EC;
876 break;
877 }
878 case SecCSNameTable: {
879 if (std::error_code EC = readCSNameTableSec())
880 return EC;
881 break;
882 }
883 case SecLBRProfile:
884 ProfileSecRange = std::make_pair(x&: Data, y&: End);
885 if (std::error_code EC = readFuncProfiles())
886 return EC;
887 break;
888 case SecFuncOffsetTable:
889 // If module is absent, we are using LLVM tools, and need to read all
890 // profiles, so skip reading the function offset table.
891 if (!M) {
892 Data = End;
893 } else {
894 assert((!ProfileIsCS ||
895 hasSecFlag(Entry, SecFuncOffsetFlags::SecFlagOrdered)) &&
896 "func offset table should always be sorted in CS profile");
897 if (std::error_code EC = readFuncOffsetTable())
898 return EC;
899 }
900 break;
901 case SecFuncMetadata: {
902 ProfileIsProbeBased =
903 hasSecFlag(Entry, Flag: SecFuncMetadataFlags::SecFlagIsProbeBased);
904 FunctionSamples::ProfileIsProbeBased = ProfileIsProbeBased;
905 ProfileHasAttribute =
906 hasSecFlag(Entry, Flag: SecFuncMetadataFlags::SecFlagHasAttribute);
907 if (std::error_code EC = readFuncMetadata(ProfileHasAttribute))
908 return EC;
909 break;
910 }
911 case SecProfileSymbolList:
912 if (std::error_code EC = readProfileSymbolList())
913 return EC;
914 break;
915 default:
916 if (std::error_code EC = readCustomSection(Entry))
917 return EC;
918 break;
919 }
920 return sampleprof_error::success;
921}
922
923bool SampleProfileReaderExtBinaryBase::useFuncOffsetList() const {
924 // If profile is CS, the function offset section is expected to consist of
925 // sequences of contexts in pre-order layout
926 // (e.g. [A, A:1 @ B, A:1 @ B:2.3 @ C] [D, D:1 @ E]), so that when a matched
927 // context in the module is found, the profiles of all its callees are
928 // recursively loaded. A list is needed since the order of profiles matters.
929 if (ProfileIsCS)
930 return true;
931
932 // If the profile is MD5, use the map container to lookup functions in
933 // the module. A remapper has no use on MD5 names.
934 if (useMD5())
935 return false;
936
937 // Profile is not MD5 and if a remapper is present, the remapped name of
938 // every function needed to be matched against the module, so use the list
939 // container since each entry is accessed.
940 if (Remapper)
941 return true;
942
943 // Otherwise use the map container for faster lookup.
944 // TODO: If the cardinality of the function offset section is much smaller
945 // than the number of functions in the module, using the list container can
946 // be always faster, but we need to figure out the constant factor to
947 // determine the cutoff.
948 return false;
949}
950
951std::error_code
952SampleProfileReaderExtBinaryBase::read(const DenseSet<StringRef> &FuncsToUse,
953 SampleProfileMap &Profiles) {
954 if (FuncsToUse.empty())
955 return sampleprof_error::success;
956
957 Data = ProfileSecRange.first;
958 End = ProfileSecRange.second;
959 if (std::error_code EC = readFuncProfiles(FuncsToUse, Profiles))
960 return EC;
961 End = Data;
962 DenseSet<FunctionSamples *> ProfilesToReadMetadata;
963 for (auto FName : FuncsToUse) {
964 auto I = Profiles.find(Ctx: FName);
965 if (I != Profiles.end())
966 ProfilesToReadMetadata.insert(V: &I->second);
967 }
968
969 if (std::error_code EC =
970 readFuncMetadata(ProfileHasAttribute, Profiles&: ProfilesToReadMetadata))
971 return EC;
972 return sampleprof_error::success;
973}
974
975bool SampleProfileReaderExtBinaryBase::collectFuncsFromModule() {
976 if (!M)
977 return false;
978 FuncsToUse.clear();
979 for (auto &F : *M)
980 FuncsToUse.insert(V: FunctionSamples::getCanonicalFnName(F));
981 return true;
982}
983
984std::error_code SampleProfileReaderExtBinaryBase::readFuncOffsetTable() {
985 // If there are more than one function offset section, the profile associated
986 // with the previous section has to be done reading before next one is read.
987 FuncOffsetTable.reset();
988 FuncOffsetList.clear();
989
990 auto Size = readNumber<uint64_t>();
991 if (std::error_code EC = Size.getError())
992 return EC;
993
994 bool UseFuncOffsetList = useFuncOffsetList();
995 if (UseFuncOffsetList)
996 FuncOffsetList.reserve(n: *Size);
997 else
998 FuncOffsetTable.emplace(args: InMemoryMode, args&: *Size);
999
1000 for (uint64_t I = 0; I < *Size; ++I) {
1001 auto FContextHash(readSampleContextFromTable());
1002 if (std::error_code EC = FContextHash.getError())
1003 return EC;
1004
1005 auto &[FContext, Hash] = *FContextHash;
1006 auto Offset = readNumber<uint64_t>();
1007 if (std::error_code EC = Offset.getError())
1008 return EC;
1009
1010 if (UseFuncOffsetList)
1011 FuncOffsetList.emplace_back(args&: FContext, args&: *Offset);
1012 else
1013 // Because Porfiles replace existing value with new value if collision
1014 // happens, we also use the latest offset so that they are consistent.
1015 FuncOffsetTable->insert(GUID: Hash, Offset: *Offset);
1016 }
1017
1018 return sampleprof_error::success;
1019}
1020
1021std::error_code SampleProfileReaderExtBinaryBase::readFuncProfiles(
1022 const DenseSet<StringRef> &FuncsToUse, SampleProfileMap &Profiles) {
1023 const uint8_t *Start = Data;
1024
1025 if (Remapper) {
1026 for (auto Name : FuncsToUse) {
1027 Remapper->insert(FunctionName: Name);
1028 }
1029 }
1030
1031 if (ProfileIsCS) {
1032 assert(useFuncOffsetList());
1033 DenseSet<uint64_t> FuncGuidsToUse;
1034 if (useMD5()) {
1035 for (auto Name : FuncsToUse)
1036 FuncGuidsToUse.insert(V: Function::getGUIDAssumingExternalLinkage(GlobalName: Name));
1037 }
1038
1039 // For each function in current module, load all context profiles for
1040 // the function as well as their callee contexts which can help profile
1041 // guided importing for ThinLTO. This can be achieved by walking
1042 // through an ordered context container, where contexts are laid out
1043 // as if they were walked in preorder of a context trie. While
1044 // traversing the trie, a link to the highest common ancestor node is
1045 // kept so that all of its decendants will be loaded.
1046 const SampleContext *CommonContext = nullptr;
1047 for (const auto &NameOffset : FuncOffsetList) {
1048 const auto &FContext = NameOffset.first;
1049 FunctionId FName = FContext.getFunction();
1050 StringRef FNameString;
1051 if (!useMD5())
1052 FNameString = FName.stringRef();
1053
1054 // For function in the current module, keep its farthest ancestor
1055 // context. This can be used to load itself and its child and
1056 // sibling contexts.
1057 if ((useMD5() && FuncGuidsToUse.count(V: FName.getHashCode())) ||
1058 (!useMD5() && (FuncsToUse.count(V: FNameString) ||
1059 (Remapper && Remapper->exist(FunctionName: FNameString))))) {
1060 if (!CommonContext || !CommonContext->isPrefixOf(That: FContext))
1061 CommonContext = &FContext;
1062 }
1063
1064 if (CommonContext == &FContext ||
1065 (CommonContext && CommonContext->isPrefixOf(That: FContext))) {
1066 // Load profile for the current context which originated from
1067 // the common ancestor.
1068 const uint8_t *FuncProfileAddr = Start + NameOffset.second;
1069 if (std::error_code EC = readFuncProfile(Start: FuncProfileAddr))
1070 return EC;
1071 }
1072 }
1073 } else if (useMD5()) {
1074 assert(!useFuncOffsetList());
1075 for (auto Name : FuncsToUse) {
1076 auto GUID = MD5Hash(Str: Name);
1077 if (auto Offset = FuncOffsetTable->lookup(GUID)) {
1078 const uint8_t *FuncProfileAddr = Start + *Offset;
1079 if (std::error_code EC = readFuncProfile(Start: FuncProfileAddr, Profiles))
1080 return EC;
1081 }
1082 }
1083 } else if (Remapper) {
1084 assert(useFuncOffsetList());
1085 for (auto NameOffset : FuncOffsetList) {
1086 SampleContext FContext(NameOffset.first);
1087 auto FuncName = FContext.getFunction();
1088 StringRef FuncNameStr = FuncName.stringRef();
1089 if (!FuncsToUse.count(V: FuncNameStr) && !Remapper->exist(FunctionName: FuncNameStr))
1090 continue;
1091 const uint8_t *FuncProfileAddr = Start + NameOffset.second;
1092 if (std::error_code EC = readFuncProfile(Start: FuncProfileAddr, Profiles))
1093 return EC;
1094 }
1095 } else {
1096 assert(!useFuncOffsetList());
1097 for (auto Name : FuncsToUse) {
1098 if (auto Offset = FuncOffsetTable->lookup(GUID: MD5Hash(Str: Name))) {
1099 const uint8_t *FuncProfileAddr = Start + *Offset;
1100 if (std::error_code EC = readFuncProfile(Start: FuncProfileAddr, Profiles))
1101 return EC;
1102 }
1103 }
1104 }
1105
1106 return sampleprof_error::success;
1107}
1108
1109std::error_code SampleProfileReaderExtBinaryBase::readFuncProfiles() {
1110 // Collect functions used by current module if the Reader has been
1111 // given a module.
1112 // collectFuncsFromModule uses FunctionSamples::getCanonicalFnName
1113 // which will query FunctionSamples::HasUniqSuffix, so it has to be
1114 // called after FunctionSamples::HasUniqSuffix is set, i.e. after
1115 // NameTable section is read.
1116 bool LoadFuncsToBeUsed = collectFuncsFromModule();
1117
1118 // When LoadFuncsToBeUsed is false, we are using LLVM tool, need to read all
1119 // profiles.
1120 if (!LoadFuncsToBeUsed) {
1121 while (Data < End) {
1122 if (std::error_code EC = readFuncProfile(Start: Data))
1123 return EC;
1124 }
1125 assert(Data == End && "More data is read than expected");
1126 } else {
1127 // Load function profiles on demand.
1128 if (std::error_code EC = readFuncProfiles(FuncsToUse, Profiles))
1129 return EC;
1130 Data = End;
1131 }
1132 assert((CSProfileCount == 0 || CSProfileCount == Profiles.size()) &&
1133 "Cannot have both context-sensitive and regular profile");
1134 assert((!CSProfileCount || ProfileIsCS) &&
1135 "Section flag should be consistent with actual profile");
1136 return sampleprof_error::success;
1137}
1138
1139std::error_code SampleProfileReaderExtBinaryBase::readProfileSymbolList() {
1140 if (!ProfSymList)
1141 ProfSymList = std::make_unique<ProfileSymbolList>();
1142
1143 if (std::error_code EC = ProfSymList->read(Data, ListSize: End - Data))
1144 return EC;
1145
1146 Data = End;
1147 return sampleprof_error::success;
1148}
1149
1150std::error_code SampleProfileReaderExtBinaryBase::decompressSection(
1151 const uint8_t *SecStart, const uint64_t SecSize,
1152 const uint8_t *&DecompressBuf, uint64_t &DecompressBufSize) {
1153 Data = SecStart;
1154 End = SecStart + SecSize;
1155 auto DecompressSize = readNumber<uint64_t>();
1156 if (std::error_code EC = DecompressSize.getError())
1157 return EC;
1158 DecompressBufSize = *DecompressSize;
1159
1160 auto CompressSize = readNumber<uint64_t>();
1161 if (std::error_code EC = CompressSize.getError())
1162 return EC;
1163
1164 if (!llvm::compression::zlib::isAvailable())
1165 return sampleprof_error::zlib_unavailable;
1166
1167 uint8_t *Buffer = Allocator.Allocate<uint8_t>(Num: DecompressBufSize);
1168 size_t UCSize = DecompressBufSize;
1169 llvm::Error E = compression::zlib::decompress(Input: ArrayRef(Data, *CompressSize),
1170 Output: Buffer, UncompressedSize&: UCSize);
1171 if (E)
1172 return sampleprof_error::uncompress_failed;
1173 DecompressBuf = reinterpret_cast<const uint8_t *>(Buffer);
1174 return sampleprof_error::success;
1175}
1176
1177std::error_code SampleProfileReaderExtBinaryBase::readImpl() {
1178 const uint8_t *BufStart =
1179 reinterpret_cast<const uint8_t *>(Buffer->getBufferStart());
1180
1181 for (auto &Entry : SecHdrTable) {
1182 // Skip empty section.
1183 if (!Entry.Size)
1184 continue;
1185
1186 // Skip sections without inlined functions when SkipFlatProf is true.
1187 if (SkipFlatProf && hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagFlat))
1188 continue;
1189
1190 const uint8_t *SecStart = BufStart + Entry.Offset;
1191 uint64_t SecSize = Entry.Size;
1192
1193 // If the section is compressed, decompress it into a buffer
1194 // DecompressBuf before reading the actual data. The pointee of
1195 // 'Data' will be changed to buffer hold by DecompressBuf
1196 // temporarily when reading the actual data.
1197 bool isCompressed = hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagCompress);
1198 if (isCompressed) {
1199 const uint8_t *DecompressBuf;
1200 uint64_t DecompressBufSize;
1201 if (std::error_code EC = decompressSection(
1202 SecStart, SecSize, DecompressBuf, DecompressBufSize))
1203 return EC;
1204 SecStart = DecompressBuf;
1205 SecSize = DecompressBufSize;
1206 }
1207
1208 if (std::error_code EC = readOneSection(Start: SecStart, Size: SecSize, Entry))
1209 return EC;
1210 if (Data != SecStart + SecSize)
1211 return sampleprof_error::malformed;
1212
1213 // Change the pointee of 'Data' from DecompressBuf to original Buffer.
1214 if (isCompressed) {
1215 Data = BufStart + Entry.Offset;
1216 End = BufStart + Buffer->getBufferSize();
1217 }
1218 }
1219
1220 return sampleprof_error::success;
1221}
1222
1223std::error_code SampleProfileReaderRawBinary::verifySPMagic(uint64_t Magic) {
1224 if (Magic == SPMagic())
1225 return sampleprof_error::success;
1226 return sampleprof_error::bad_magic;
1227}
1228
1229std::error_code SampleProfileReaderExtBinary::verifySPMagic(uint64_t Magic) {
1230 if (Magic == SPMagic(Format: SPF_Ext_Binary))
1231 return sampleprof_error::success;
1232 return sampleprof_error::bad_magic;
1233}
1234
1235std::error_code SampleProfileReaderBinary::readNameTable() {
1236 auto Size = readNumber<size_t>();
1237 if (std::error_code EC = Size.getError())
1238 return EC;
1239
1240 // Normally if useMD5 is true, the name table should have MD5 values, not
1241 // strings, however in the case that ExtBinary profile has multiple name
1242 // tables mixing string and MD5, all of them have to be normalized to use MD5,
1243 // because optimization passes can only handle either type.
1244 bool UseMD5 = useMD5();
1245
1246 auto &TableVec = NameTable.setToEager();
1247 TableVec.reserve(n: *Size);
1248 if (!ProfileIsCS) {
1249 MD5SampleContextTable.clear();
1250 if (UseMD5)
1251 MD5SampleContextTable.reserve(n: *Size);
1252 else
1253 // If we are using strings, delay MD5 computation since only a portion of
1254 // names are used by top level functions. Use 0 to indicate MD5 value is
1255 // to be calculated as no known string has a MD5 value of 0.
1256 MD5SampleContextTable.resize(new_size: *Size);
1257 }
1258 for (size_t I = 0; I < *Size; ++I) {
1259 auto Name(readString());
1260 if (std::error_code EC = Name.getError())
1261 return EC;
1262 if (UseMD5) {
1263 FunctionId FID(*Name);
1264 if (!ProfileIsCS)
1265 MD5SampleContextTable.emplace_back(args: FID.getHashCode());
1266 TableVec.emplace_back(args&: FID);
1267 } else
1268 TableVec.push_back(x: FunctionId(*Name));
1269 }
1270 if (!ProfileIsCS)
1271 MD5SampleContextStart = MD5SampleContextTable.data();
1272 return sampleprof_error::success;
1273}
1274
1275std::error_code
1276SampleProfileReaderExtBinaryBase::readNameTableSec(bool IsMD5,
1277 bool FixedLengthMD5) {
1278 if (FixedLengthMD5) {
1279 if (!IsMD5)
1280 errs() << "If FixedLengthMD5 is true, UseMD5 has to be true";
1281 auto Size = readNumber<size_t>();
1282 if (std::error_code EC = Size.getError())
1283 return EC;
1284
1285 assert(Data + (*Size) * sizeof(uint64_t) == End &&
1286 "Fixed length MD5 name table does not contain specified number of "
1287 "entries");
1288 if (Data + (*Size) * sizeof(uint64_t) > End)
1289 return sampleprof_error::truncated;
1290
1291 if (LazyLoadNameTable) {
1292 NameTable.setLazy(S: Data, Sz: *Size);
1293 } else {
1294 auto &TableVec = NameTable.setToEager();
1295 TableVec.reserve(n: *Size);
1296 for (size_t I = 0; I < *Size; ++I) {
1297 using namespace support;
1298 uint64_t FID = endian::read<uint64_t, unaligned>(
1299 memory: Data + I * sizeof(uint64_t), endian: endianness::little);
1300 TableVec.emplace_back(args: FunctionId(FID));
1301 }
1302 }
1303 if (!ProfileIsCS)
1304 MD5SampleContextStart = reinterpret_cast<const uint64_t *>(Data);
1305 Data = Data + (*Size) * sizeof(uint64_t);
1306 return sampleprof_error::success;
1307 }
1308
1309 if (IsMD5) {
1310 assert(!FixedLengthMD5 && "FixedLengthMD5 should be unreachable here");
1311 auto Size = readNumber<size_t>();
1312 if (std::error_code EC = Size.getError())
1313 return EC;
1314
1315 auto &TableVec = NameTable.setToEager();
1316 TableVec.reserve(n: *Size);
1317 if (!ProfileIsCS)
1318 MD5SampleContextTable.resize(new_size: *Size);
1319 for (size_t I = 0; I < *Size; ++I) {
1320 auto FID = readNumber<uint64_t>();
1321 if (std::error_code EC = FID.getError())
1322 return EC;
1323 if (!ProfileIsCS)
1324 support::endian::write64le(P: &MD5SampleContextTable[I], V: *FID);
1325 TableVec.emplace_back(args: FunctionId(*FID));
1326 }
1327 if (!ProfileIsCS)
1328 MD5SampleContextStart = MD5SampleContextTable.data();
1329 return sampleprof_error::success;
1330 }
1331
1332 return SampleProfileReaderBinary::readNameTable();
1333}
1334
1335// Read in the CS name table section, which basically contains a list of context
1336// vectors. Each element of a context vector, aka a frame, refers to the
1337// underlying raw function names that are stored in the name table, as well as
1338// a callsite identifier that only makes sense for non-leaf frames.
1339std::error_code SampleProfileReaderExtBinaryBase::readCSNameTableSec() {
1340 auto Size = readNumber<size_t>();
1341 if (std::error_code EC = Size.getError())
1342 return EC;
1343
1344 CSNameTable.clear();
1345 CSNameTable.reserve(n: *Size);
1346 if (ProfileIsCS) {
1347 // Delay MD5 computation of CS context until they are needed. Use 0 to
1348 // indicate MD5 value is to be calculated as no known string has a MD5
1349 // value of 0.
1350 MD5SampleContextTable.clear();
1351 MD5SampleContextTable.resize(new_size: *Size);
1352 MD5SampleContextStart = MD5SampleContextTable.data();
1353 }
1354 for (size_t I = 0; I < *Size; ++I) {
1355 CSNameTable.emplace_back(args: SampleContextFrameVector());
1356 auto ContextSize = readNumber<uint32_t>();
1357 if (std::error_code EC = ContextSize.getError())
1358 return EC;
1359 for (uint32_t J = 0; J < *ContextSize; ++J) {
1360 auto FName(readStringFromTable());
1361 if (std::error_code EC = FName.getError())
1362 return EC;
1363 auto LineOffset = readNumber<uint64_t>();
1364 if (std::error_code EC = LineOffset.getError())
1365 return EC;
1366
1367 if (!isOffsetLegal(L: *LineOffset))
1368 return sampleprof_error::illegal_line_offset;
1369
1370 auto Discriminator = readNumber<uint64_t>();
1371 if (std::error_code EC = Discriminator.getError())
1372 return EC;
1373
1374 CSNameTable.back().emplace_back(
1375 Args&: FName.get(), Args: LineLocation(LineOffset.get(), Discriminator.get()));
1376 }
1377 }
1378
1379 return sampleprof_error::success;
1380}
1381
1382std::error_code
1383SampleProfileReaderExtBinaryBase::readFuncMetadata(bool ProfileHasAttribute,
1384 FunctionSamples *FProfile) {
1385 if (Data < End) {
1386 if (ProfileIsProbeBased) {
1387 auto Checksum = readNumber<uint64_t>();
1388 if (std::error_code EC = Checksum.getError())
1389 return EC;
1390 if (FProfile)
1391 FProfile->setFunctionHash(*Checksum);
1392 }
1393
1394 if (ProfileHasAttribute) {
1395 auto Attributes = readNumber<uint32_t>();
1396 if (std::error_code EC = Attributes.getError())
1397 return EC;
1398 if (FProfile)
1399 FProfile->getContext().setAllAttributes(*Attributes);
1400 }
1401
1402 if (!ProfileIsCS) {
1403 // Read all the attributes for inlined function calls.
1404 auto NumCallsites = readNumber<uint32_t>();
1405 if (std::error_code EC = NumCallsites.getError())
1406 return EC;
1407
1408 for (uint32_t J = 0; J < *NumCallsites; ++J) {
1409 auto LineOffset = readNumber<uint64_t>();
1410 if (std::error_code EC = LineOffset.getError())
1411 return EC;
1412
1413 auto Discriminator = readNumber<uint64_t>();
1414 if (std::error_code EC = Discriminator.getError())
1415 return EC;
1416
1417 auto FContextHash(readSampleContextFromTable());
1418 if (std::error_code EC = FContextHash.getError())
1419 return EC;
1420
1421 auto &[FContext, Hash] = *FContextHash;
1422 FunctionSamples *CalleeProfile = nullptr;
1423 if (FProfile) {
1424 CalleeProfile = const_cast<FunctionSamples *>(
1425 &FProfile->functionSamplesAt(Loc: LineLocation(
1426 *LineOffset,
1427 *Discriminator))[FContext.getFunction()]);
1428 }
1429 if (std::error_code EC =
1430 readFuncMetadata(ProfileHasAttribute, FProfile: CalleeProfile))
1431 return EC;
1432 }
1433 }
1434 }
1435
1436 return sampleprof_error::success;
1437}
1438
1439std::error_code SampleProfileReaderExtBinaryBase::readFuncMetadata(
1440 bool ProfileHasAttribute, DenseSet<FunctionSamples *> &Profiles) {
1441 if (FuncMetadataIndex.empty())
1442 return sampleprof_error::success;
1443
1444 for (auto *FProfile : Profiles) {
1445 auto R = FuncMetadataIndex.find(Val: FProfile->getContext().getHashCode());
1446 if (R == FuncMetadataIndex.end())
1447 continue;
1448
1449 Data = R->second.first;
1450 End = R->second.second;
1451 if (std::error_code EC = readFuncMetadata(ProfileHasAttribute, FProfile))
1452 return EC;
1453 assert(Data == End && "More data is read than expected");
1454 }
1455 return sampleprof_error::success;
1456}
1457
1458std::error_code
1459SampleProfileReaderExtBinaryBase::readFuncMetadata(bool ProfileHasAttribute) {
1460 while (Data < End) {
1461 auto FContextHash(readSampleContextFromTable());
1462 if (std::error_code EC = FContextHash.getError())
1463 return EC;
1464 auto &[FContext, Hash] = *FContextHash;
1465 FunctionSamples *FProfile = nullptr;
1466 auto It = Profiles.find(Ctx: FContext);
1467 if (It != Profiles.end())
1468 FProfile = &It->second;
1469
1470 const uint8_t *Start = Data;
1471 if (std::error_code EC = readFuncMetadata(ProfileHasAttribute, FProfile))
1472 return EC;
1473
1474 FuncMetadataIndex[FContext.getHashCode()] = {Start, Data};
1475 }
1476
1477 assert(Data == End && "More data is read than expected");
1478 return sampleprof_error::success;
1479}
1480
1481std::error_code
1482SampleProfileReaderExtBinaryBase::readSecHdrTableEntry(uint64_t Idx) {
1483 SecHdrTableEntry Entry;
1484 auto Type = readUnencodedNumber<uint64_t>();
1485 if (std::error_code EC = Type.getError())
1486 return EC;
1487 Entry.Type = static_cast<SecType>(*Type);
1488
1489 auto Flags = readUnencodedNumber<uint64_t>();
1490 if (std::error_code EC = Flags.getError())
1491 return EC;
1492 Entry.Flags = *Flags;
1493
1494 auto Offset = readUnencodedNumber<uint64_t>();
1495 if (std::error_code EC = Offset.getError())
1496 return EC;
1497 Entry.Offset = *Offset;
1498
1499 auto Size = readUnencodedNumber<uint64_t>();
1500 if (std::error_code EC = Size.getError())
1501 return EC;
1502 Entry.Size = *Size;
1503
1504 Entry.LayoutIndex = Idx;
1505 SecHdrTable.push_back(x: std::move(Entry));
1506 return sampleprof_error::success;
1507}
1508
1509std::error_code SampleProfileReaderExtBinaryBase::readSecHdrTable() {
1510 auto EntryNum = readUnencodedNumber<uint64_t>();
1511 if (std::error_code EC = EntryNum.getError())
1512 return EC;
1513
1514 for (uint64_t i = 0; i < (*EntryNum); i++)
1515 if (std::error_code EC = readSecHdrTableEntry(Idx: i))
1516 return EC;
1517
1518 return sampleprof_error::success;
1519}
1520
1521std::error_code SampleProfileReaderExtBinaryBase::readHeader() {
1522 const uint8_t *BufStart =
1523 reinterpret_cast<const uint8_t *>(Buffer->getBufferStart());
1524 Data = BufStart;
1525 End = BufStart + Buffer->getBufferSize();
1526
1527 if (std::error_code EC = readMagicIdent())
1528 return EC;
1529
1530 if (std::error_code EC = readSecHdrTable())
1531 return EC;
1532
1533 return sampleprof_error::success;
1534}
1535
1536uint64_t SampleProfileReaderExtBinaryBase::getSectionSize(SecType Type) {
1537 uint64_t Size = 0;
1538 for (auto &Entry : SecHdrTable) {
1539 if (Entry.Type == Type)
1540 Size += Entry.Size;
1541 }
1542 return Size;
1543}
1544
1545uint64_t SampleProfileReaderExtBinaryBase::getFileSize() {
1546 // Sections in SecHdrTable is not necessarily in the same order as
1547 // sections in the profile because section like FuncOffsetTable needs
1548 // to be written after section LBRProfile but needs to be read before
1549 // section LBRProfile, so we cannot simply use the last entry in
1550 // SecHdrTable to calculate the file size.
1551 uint64_t FileSize = 0;
1552 for (auto &Entry : SecHdrTable) {
1553 FileSize = std::max(a: Entry.Offset + Entry.Size, b: FileSize);
1554 }
1555 return FileSize;
1556}
1557
1558static std::string getSecFlagsStr(const SecHdrTableEntry &Entry) {
1559 std::string Flags;
1560 if (hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagCompress))
1561 Flags.append(s: "{compressed,");
1562 else
1563 Flags.append(s: "{");
1564
1565 if (hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagFlat))
1566 Flags.append(s: "flat,");
1567
1568 switch (Entry.Type) {
1569 case SecNameTable:
1570 if (hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagFixedLengthMD5))
1571 Flags.append(s: "fixlenmd5,");
1572 else if (hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagMD5Name))
1573 Flags.append(s: "md5,");
1574 if (hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagUniqSuffix))
1575 Flags.append(s: "uniq,");
1576 break;
1577 case SecProfSummary:
1578 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagPartial))
1579 Flags.append(s: "partial,");
1580 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagFullContext))
1581 Flags.append(s: "context,");
1582 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagIsPreInlined))
1583 Flags.append(s: "preInlined,");
1584 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagFSDiscriminator))
1585 Flags.append(s: "fs-discriminator,");
1586 break;
1587 case SecFuncOffsetTable:
1588 if (hasSecFlag(Entry, Flag: SecFuncOffsetFlags::SecFlagOrdered))
1589 Flags.append(s: "ordered,");
1590 break;
1591 case SecFuncMetadata:
1592 if (hasSecFlag(Entry, Flag: SecFuncMetadataFlags::SecFlagIsProbeBased))
1593 Flags.append(s: "probe,");
1594 if (hasSecFlag(Entry, Flag: SecFuncMetadataFlags::SecFlagHasAttribute))
1595 Flags.append(s: "attr,");
1596 break;
1597 default:
1598 break;
1599 }
1600 char &last = Flags.back();
1601 if (last == ',')
1602 last = '}';
1603 else
1604 Flags.append(s: "}");
1605 return Flags;
1606}
1607
1608bool SampleProfileReaderExtBinaryBase::dumpSectionInfo(raw_ostream &OS) {
1609 uint64_t TotalSecsSize = 0;
1610 for (auto &Entry : SecHdrTable) {
1611 OS << getSecName(Type: Entry.Type) << " - Offset: " << Entry.Offset
1612 << ", Size: " << Entry.Size << ", Flags: " << getSecFlagsStr(Entry)
1613 << "\n";
1614 ;
1615 TotalSecsSize += Entry.Size;
1616 }
1617 uint64_t HeaderSize = SecHdrTable.front().Offset;
1618 assert(HeaderSize + TotalSecsSize == getFileSize() &&
1619 "Size of 'header + sections' doesn't match the total size of profile");
1620
1621 OS << "Header Size: " << HeaderSize << "\n";
1622 OS << "Total Sections Size: " << TotalSecsSize << "\n";
1623 OS << "File Size: " << getFileSize() << "\n";
1624 return true;
1625}
1626
1627std::error_code SampleProfileReaderBinary::readMagicIdent() {
1628 // Read and check the magic identifier.
1629 auto Magic = readNumber<uint64_t>();
1630 if (std::error_code EC = Magic.getError())
1631 return EC;
1632 else if (std::error_code EC = verifySPMagic(Magic: *Magic))
1633 return EC;
1634
1635 // Read the version number.
1636 auto Version = readNumber<uint64_t>();
1637 if (std::error_code EC = Version.getError())
1638 return EC;
1639 else if (*Version != SPVersion())
1640 return sampleprof_error::unsupported_version;
1641
1642 return sampleprof_error::success;
1643}
1644
1645std::error_code SampleProfileReaderBinary::readHeader() {
1646 Data = reinterpret_cast<const uint8_t *>(Buffer->getBufferStart());
1647 End = Data + Buffer->getBufferSize();
1648
1649 if (std::error_code EC = readMagicIdent())
1650 return EC;
1651
1652 if (std::error_code EC = readSummary())
1653 return EC;
1654
1655 if (std::error_code EC = readNameTable())
1656 return EC;
1657 return sampleprof_error::success;
1658}
1659
1660std::error_code SampleProfileReaderBinary::readSummaryEntry(
1661 std::vector<ProfileSummaryEntry> &Entries) {
1662 auto Cutoff = readNumber<uint64_t>();
1663 if (std::error_code EC = Cutoff.getError())
1664 return EC;
1665
1666 auto MinBlockCount = readNumber<uint64_t>();
1667 if (std::error_code EC = MinBlockCount.getError())
1668 return EC;
1669
1670 auto NumBlocks = readNumber<uint64_t>();
1671 if (std::error_code EC = NumBlocks.getError())
1672 return EC;
1673
1674 Entries.emplace_back(args&: *Cutoff, args&: *MinBlockCount, args&: *NumBlocks);
1675 return sampleprof_error::success;
1676}
1677
1678std::error_code SampleProfileReaderBinary::readSummary() {
1679 auto TotalCount = readNumber<uint64_t>();
1680 if (std::error_code EC = TotalCount.getError())
1681 return EC;
1682
1683 auto MaxBlockCount = readNumber<uint64_t>();
1684 if (std::error_code EC = MaxBlockCount.getError())
1685 return EC;
1686
1687 auto MaxFunctionCount = readNumber<uint64_t>();
1688 if (std::error_code EC = MaxFunctionCount.getError())
1689 return EC;
1690
1691 auto NumBlocks = readNumber<uint64_t>();
1692 if (std::error_code EC = NumBlocks.getError())
1693 return EC;
1694
1695 auto NumFunctions = readNumber<uint64_t>();
1696 if (std::error_code EC = NumFunctions.getError())
1697 return EC;
1698
1699 auto NumSummaryEntries = readNumber<uint64_t>();
1700 if (std::error_code EC = NumSummaryEntries.getError())
1701 return EC;
1702
1703 std::vector<ProfileSummaryEntry> Entries;
1704 for (unsigned i = 0; i < *NumSummaryEntries; i++) {
1705 std::error_code EC = readSummaryEntry(Entries);
1706 if (EC != sampleprof_error::success)
1707 return EC;
1708 }
1709 Summary = std::make_unique<ProfileSummary>(
1710 args: ProfileSummary::PSK_Sample, args&: Entries, args&: *TotalCount, args&: *MaxBlockCount, args: 0,
1711 args&: *MaxFunctionCount, args&: *NumBlocks, args&: *NumFunctions);
1712
1713 return sampleprof_error::success;
1714}
1715
1716bool SampleProfileReaderRawBinary::hasFormat(const MemoryBuffer &Buffer) {
1717 const uint8_t *Data =
1718 reinterpret_cast<const uint8_t *>(Buffer.getBufferStart());
1719 uint64_t Magic = decodeULEB128(p: Data);
1720 return Magic == SPMagic();
1721}
1722
1723bool SampleProfileReaderExtBinary::hasFormat(const MemoryBuffer &Buffer) {
1724 const uint8_t *Data =
1725 reinterpret_cast<const uint8_t *>(Buffer.getBufferStart());
1726 uint64_t Magic = decodeULEB128(p: Data);
1727 return Magic == SPMagic(Format: SPF_Ext_Binary);
1728}
1729
1730std::error_code SampleProfileReaderGCC::skipNextWord() {
1731 uint32_t dummy;
1732 if (!GcovBuffer.readInt(Val&: dummy))
1733 return sampleprof_error::truncated;
1734 return sampleprof_error::success;
1735}
1736
1737template <typename T> ErrorOr<T> SampleProfileReaderGCC::readNumber() {
1738 if (sizeof(T) <= sizeof(uint32_t)) {
1739 uint32_t Val;
1740 if (GcovBuffer.readInt(Val) && Val <= std::numeric_limits<T>::max())
1741 return static_cast<T>(Val);
1742 } else if (sizeof(T) <= sizeof(uint64_t)) {
1743 uint64_t Val;
1744 if (GcovBuffer.readInt64(Val) && Val <= std::numeric_limits<T>::max())
1745 return static_cast<T>(Val);
1746 }
1747
1748 std::error_code EC = sampleprof_error::malformed;
1749 reportError(LineNumber: 0, Msg: EC.message());
1750 return EC;
1751}
1752
1753ErrorOr<StringRef> SampleProfileReaderGCC::readString() {
1754 StringRef Str;
1755 if (!GcovBuffer.readString(str&: Str))
1756 return sampleprof_error::truncated;
1757 return Str;
1758}
1759
1760std::error_code SampleProfileReaderGCC::readHeader() {
1761 // Read the magic identifier.
1762 if (!GcovBuffer.readGCDAFormat())
1763 return sampleprof_error::unrecognized_format;
1764
1765 // Read the version number. Note - the GCC reader does not validate this
1766 // version, but the profile creator generates v704.
1767 GCOV::GCOVVersion version;
1768 if (!GcovBuffer.readGCOVVersion(version))
1769 return sampleprof_error::unrecognized_format;
1770
1771 if (version != GCOV::V407)
1772 return sampleprof_error::unsupported_version;
1773
1774 // Skip the empty integer.
1775 if (std::error_code EC = skipNextWord())
1776 return EC;
1777
1778 return sampleprof_error::success;
1779}
1780
1781std::error_code SampleProfileReaderGCC::readSectionTag(uint32_t Expected) {
1782 uint32_t Tag;
1783 if (!GcovBuffer.readInt(Val&: Tag))
1784 return sampleprof_error::truncated;
1785
1786 if (Tag != Expected)
1787 return sampleprof_error::malformed;
1788
1789 if (std::error_code EC = skipNextWord())
1790 return EC;
1791
1792 return sampleprof_error::success;
1793}
1794
1795std::error_code SampleProfileReaderGCC::readNameTable() {
1796 if (std::error_code EC = readSectionTag(Expected: GCOVTagAFDOFileNames))
1797 return EC;
1798
1799 uint32_t Size;
1800 if (!GcovBuffer.readInt(Val&: Size))
1801 return sampleprof_error::truncated;
1802
1803 for (uint32_t I = 0; I < Size; ++I) {
1804 StringRef Str;
1805 if (!GcovBuffer.readString(str&: Str))
1806 return sampleprof_error::truncated;
1807 Names.push_back(x: std::string(Str));
1808 }
1809
1810 return sampleprof_error::success;
1811}
1812
1813std::error_code SampleProfileReaderGCC::readFunctionProfiles() {
1814 if (std::error_code EC = readSectionTag(Expected: GCOVTagAFDOFunction))
1815 return EC;
1816
1817 uint32_t NumFunctions;
1818 if (!GcovBuffer.readInt(Val&: NumFunctions))
1819 return sampleprof_error::truncated;
1820
1821 InlineCallStack Stack;
1822 for (uint32_t I = 0; I < NumFunctions; ++I)
1823 if (std::error_code EC = readOneFunctionProfile(InlineStack: Stack, Update: true, Offset: 0))
1824 return EC;
1825
1826 computeSummary();
1827 return sampleprof_error::success;
1828}
1829
1830std::error_code SampleProfileReaderGCC::readOneFunctionProfile(
1831 const InlineCallStack &InlineStack, bool Update, uint32_t Offset) {
1832 uint64_t HeadCount = 0;
1833 if (InlineStack.size() == 0)
1834 if (!GcovBuffer.readInt64(Val&: HeadCount))
1835 return sampleprof_error::truncated;
1836
1837 uint32_t NameIdx;
1838 if (!GcovBuffer.readInt(Val&: NameIdx))
1839 return sampleprof_error::truncated;
1840
1841 StringRef Name(Names[NameIdx]);
1842
1843 uint32_t NumPosCounts;
1844 if (!GcovBuffer.readInt(Val&: NumPosCounts))
1845 return sampleprof_error::truncated;
1846
1847 uint32_t NumCallsites;
1848 if (!GcovBuffer.readInt(Val&: NumCallsites))
1849 return sampleprof_error::truncated;
1850
1851 FunctionSamples *FProfile = nullptr;
1852 if (InlineStack.size() == 0) {
1853 // If this is a top function that we have already processed, do not
1854 // update its profile again. This happens in the presence of
1855 // function aliases. Since these aliases share the same function
1856 // body, there will be identical replicated profiles for the
1857 // original function. In this case, we simply not bother updating
1858 // the profile of the original function.
1859 FProfile = &Profiles[FunctionId(Name)];
1860 FProfile->addHeadSamples(Num: HeadCount);
1861 if (FProfile->getTotalSamples() > 0)
1862 Update = false;
1863 } else {
1864 // Otherwise, we are reading an inlined instance. The top of the
1865 // inline stack contains the profile of the caller. Insert this
1866 // callee in the caller's CallsiteMap.
1867 FunctionSamples *CallerProfile = InlineStack.front();
1868 uint32_t LineOffset = Offset >> 16;
1869 uint32_t Discriminator = Offset & 0xffff;
1870 FProfile = &CallerProfile->functionSamplesAt(
1871 Loc: LineLocation(LineOffset, Discriminator))[FunctionId(Name)];
1872 }
1873 FProfile->setFunction(FunctionId(Name));
1874
1875 for (uint32_t I = 0; I < NumPosCounts; ++I) {
1876 uint32_t Offset;
1877 if (!GcovBuffer.readInt(Val&: Offset))
1878 return sampleprof_error::truncated;
1879
1880 uint32_t NumTargets;
1881 if (!GcovBuffer.readInt(Val&: NumTargets))
1882 return sampleprof_error::truncated;
1883
1884 uint64_t Count;
1885 if (!GcovBuffer.readInt64(Val&: Count))
1886 return sampleprof_error::truncated;
1887
1888 // The line location is encoded in the offset as:
1889 // high 16 bits: line offset to the start of the function.
1890 // low 16 bits: discriminator.
1891 uint32_t LineOffset = Offset >> 16;
1892 uint32_t Discriminator = Offset & 0xffff;
1893
1894 InlineCallStack NewStack;
1895 NewStack.push_back(Elt: FProfile);
1896 llvm::append_range(C&: NewStack, R: InlineStack);
1897 if (Update) {
1898 // Walk up the inline stack, adding the samples on this line to
1899 // the total sample count of the callers in the chain.
1900 for (auto *CallerProfile : NewStack)
1901 CallerProfile->addTotalSamples(Num: Count);
1902
1903 // Update the body samples for the current profile.
1904 FProfile->addBodySamples(LineOffset, Discriminator, Num: Count);
1905 }
1906
1907 // Process the list of functions called at an indirect call site.
1908 // These are all the targets that a function pointer (or virtual
1909 // function) resolved at runtime.
1910 for (uint32_t J = 0; J < NumTargets; J++) {
1911 uint32_t HistVal;
1912 if (!GcovBuffer.readInt(Val&: HistVal))
1913 return sampleprof_error::truncated;
1914
1915 if (HistVal != HIST_TYPE_INDIR_CALL_TOPN)
1916 return sampleprof_error::malformed;
1917
1918 uint64_t TargetIdx;
1919 if (!GcovBuffer.readInt64(Val&: TargetIdx))
1920 return sampleprof_error::truncated;
1921 StringRef TargetName(Names[TargetIdx]);
1922
1923 uint64_t TargetCount;
1924 if (!GcovBuffer.readInt64(Val&: TargetCount))
1925 return sampleprof_error::truncated;
1926
1927 if (Update)
1928 FProfile->addCalledTargetSamples(LineOffset, Discriminator,
1929 Func: FunctionId(TargetName),
1930 Num: TargetCount);
1931 }
1932 }
1933
1934 // Process all the inlined callers into the current function. These
1935 // are all the callsites that were inlined into this function.
1936 for (uint32_t I = 0; I < NumCallsites; I++) {
1937 // The offset is encoded as:
1938 // high 16 bits: line offset to the start of the function.
1939 // low 16 bits: discriminator.
1940 uint32_t Offset;
1941 if (!GcovBuffer.readInt(Val&: Offset))
1942 return sampleprof_error::truncated;
1943 InlineCallStack NewStack;
1944 NewStack.push_back(Elt: FProfile);
1945 llvm::append_range(C&: NewStack, R: InlineStack);
1946 if (std::error_code EC = readOneFunctionProfile(InlineStack: NewStack, Update, Offset))
1947 return EC;
1948 }
1949
1950 return sampleprof_error::success;
1951}
1952
1953/// Read a GCC AutoFDO profile.
1954///
1955/// This format is generated by the Linux Perf conversion tool at
1956/// https://github.com/google/autofdo.
1957std::error_code SampleProfileReaderGCC::readImpl() {
1958 assert(!ProfileIsFSDisciminator && "Gcc profiles not support FSDisciminator");
1959 // Read the string table.
1960 if (std::error_code EC = readNameTable())
1961 return EC;
1962
1963 // Read the source profile.
1964 if (std::error_code EC = readFunctionProfiles())
1965 return EC;
1966
1967 return sampleprof_error::success;
1968}
1969
1970bool SampleProfileReaderGCC::hasFormat(const MemoryBuffer &Buffer) {
1971 StringRef Magic(Buffer.getBufferStart());
1972 return Magic == "adcg*704";
1973}
1974
1975void SampleProfileReaderItaniumRemapper::applyRemapping(LLVMContext &Ctx) {
1976 // If the reader uses MD5 to represent string, we can't remap it because
1977 // we don't know what the original function names were.
1978 if (Reader.useMD5()) {
1979 Ctx.diagnose(DI: DiagnosticInfoSampleProfile(
1980 Reader.getBuffer()->getBufferIdentifier(),
1981 "Profile data remapping cannot be applied to profile data "
1982 "using MD5 names (original mangled names are not available).",
1983 DS_Warning));
1984 return;
1985 }
1986
1987 // CSSPGO-TODO: Remapper is not yet supported.
1988 // We will need to remap the entire context string.
1989 assert(Remappings && "should be initialized while creating remapper");
1990 for (auto &Sample : Reader.getProfiles()) {
1991 DenseSet<FunctionId> NamesInSample;
1992 Sample.second.findAllNames(NameSet&: NamesInSample);
1993 for (auto &Name : NamesInSample) {
1994 StringRef NameStr = Name.stringRef();
1995 if (auto Key = Remappings->insert(FunctionName: NameStr))
1996 NameMap.insert(KV: {Key, NameStr});
1997 }
1998 }
1999
2000 RemappingApplied = true;
2001}
2002
2003std::optional<StringRef>
2004SampleProfileReaderItaniumRemapper::lookUpNameInProfile(StringRef Fname) {
2005 if (auto Key = Remappings->lookup(FunctionName: Fname)) {
2006 StringRef Result = NameMap.lookup(Val: Key);
2007 if (!Result.empty())
2008 return Result;
2009 }
2010 return std::nullopt;
2011}
2012
2013/// Prepare a memory buffer for the contents of \p Filename.
2014///
2015/// \returns an error code indicating the status of the buffer.
2016static ErrorOr<std::unique_ptr<MemoryBuffer>>
2017setupMemoryBuffer(const Twine &Filename, vfs::FileSystem &FS) {
2018 auto BufferOrErr = Filename.str() == "-" ? MemoryBuffer::getSTDIN()
2019 : FS.getBufferForFile(Name: Filename);
2020 if (std::error_code EC = BufferOrErr.getError())
2021 return EC;
2022 auto Buffer = std::move(BufferOrErr.get());
2023
2024 return std::move(Buffer);
2025}
2026
2027/// Create a sample profile reader based on the format of the input file.
2028///
2029/// \param Filename The file to open.
2030///
2031/// \param C The LLVM context to use to emit diagnostics.
2032///
2033/// \param P The FSDiscriminatorPass.
2034///
2035/// \param RemapFilename The file used for profile remapping.
2036///
2037/// \returns an error code indicating the status of the created reader.
2038ErrorOr<std::unique_ptr<SampleProfileReader>>
2039SampleProfileReader::create(StringRef Filename, LLVMContext &C,
2040 vfs::FileSystem &FS, FSDiscriminatorPass P,
2041 StringRef RemapFilename) {
2042 auto BufferOrError = setupMemoryBuffer(Filename, FS);
2043 if (std::error_code EC = BufferOrError.getError())
2044 return EC;
2045 return create(B&: BufferOrError.get(), C, FS, P, RemapFilename);
2046}
2047
2048/// Create a sample profile remapper from the given input, to remap the
2049/// function names in the given profile data.
2050///
2051/// \param Filename The file to open.
2052///
2053/// \param Reader The profile reader the remapper is going to be applied to.
2054///
2055/// \param C The LLVM context to use to emit diagnostics.
2056///
2057/// \returns an error code indicating the status of the created reader.
2058ErrorOr<std::unique_ptr<SampleProfileReaderItaniumRemapper>>
2059SampleProfileReaderItaniumRemapper::create(StringRef Filename,
2060 vfs::FileSystem &FS,
2061 SampleProfileReader &Reader,
2062 LLVMContext &C) {
2063 auto BufferOrError = setupMemoryBuffer(Filename, FS);
2064 if (std::error_code EC = BufferOrError.getError())
2065 return EC;
2066 return create(B&: BufferOrError.get(), Reader, C);
2067}
2068
2069/// Create a sample profile remapper from the given input, to remap the
2070/// function names in the given profile data.
2071///
2072/// \param B The memory buffer to create the reader from (assumes ownership).
2073///
2074/// \param C The LLVM context to use to emit diagnostics.
2075///
2076/// \param Reader The profile reader the remapper is going to be applied to.
2077///
2078/// \returns an error code indicating the status of the created reader.
2079ErrorOr<std::unique_ptr<SampleProfileReaderItaniumRemapper>>
2080SampleProfileReaderItaniumRemapper::create(std::unique_ptr<MemoryBuffer> &B,
2081 SampleProfileReader &Reader,
2082 LLVMContext &C) {
2083 auto Remappings = std::make_unique<SymbolRemappingReader>();
2084 if (Error E = Remappings->read(B&: *B)) {
2085 handleAllErrors(
2086 E: std::move(E), Handlers: [&](const SymbolRemappingParseError &ParseError) {
2087 C.diagnose(DI: DiagnosticInfoSampleProfile(B->getBufferIdentifier(),
2088 ParseError.getLineNum(),
2089 ParseError.getMessage()));
2090 });
2091 return sampleprof_error::malformed;
2092 }
2093
2094 return std::make_unique<SampleProfileReaderItaniumRemapper>(
2095 args: std::move(B), args: std::move(Remappings), args&: Reader);
2096}
2097
2098/// Create a sample profile reader based on the format of the input data.
2099///
2100/// \param B The memory buffer to create the reader from (assumes ownership).
2101///
2102/// \param C The LLVM context to use to emit diagnostics.
2103///
2104/// \param P The FSDiscriminatorPass.
2105///
2106/// \param RemapFilename The file used for profile remapping.
2107///
2108/// \returns an error code indicating the status of the created reader.
2109ErrorOr<std::unique_ptr<SampleProfileReader>>
2110SampleProfileReader::create(std::unique_ptr<MemoryBuffer> &B, LLVMContext &C,
2111 vfs::FileSystem &FS, FSDiscriminatorPass P,
2112 StringRef RemapFilename) {
2113 std::unique_ptr<SampleProfileReader> Reader;
2114 if (SampleProfileReaderRawBinary::hasFormat(Buffer: *B))
2115 Reader.reset(p: new SampleProfileReaderRawBinary(std::move(B), C));
2116 else if (SampleProfileReaderExtBinary::hasFormat(Buffer: *B))
2117 Reader.reset(p: new SampleProfileReaderExtBinary(std::move(B), C));
2118 else if (SampleProfileReaderGCC::hasFormat(Buffer: *B))
2119 Reader.reset(p: new SampleProfileReaderGCC(std::move(B), C));
2120 else if (SampleProfileReaderText::hasFormat(Buffer: *B))
2121 Reader.reset(p: new SampleProfileReaderText(std::move(B), C));
2122 else
2123 return sampleprof_error::unrecognized_format;
2124
2125 if (!RemapFilename.empty()) {
2126 auto ReaderOrErr = SampleProfileReaderItaniumRemapper::create(
2127 Filename: RemapFilename, FS, Reader&: *Reader, C);
2128 if (std::error_code EC = ReaderOrErr.getError()) {
2129 std::string Msg = "Could not create remapper: " + EC.message();
2130 C.diagnose(DI: DiagnosticInfoSampleProfile(RemapFilename, Msg));
2131 return EC;
2132 }
2133 Reader->Remapper = std::move(ReaderOrErr.get());
2134 }
2135
2136 if (std::error_code EC = Reader->readHeader()) {
2137 return EC;
2138 }
2139
2140 Reader->setDiscriminatorMaskedBitFrom(P);
2141
2142 return std::move(Reader);
2143}
2144
2145// For text and GCC file formats, we compute the summary after reading the
2146// profile. Binary format has the profile summary in its header.
2147void SampleProfileReader::computeSummary() {
2148 SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs);
2149 Summary = Builder.computeSummaryForProfiles(Profiles);
2150}
2151