1//===- SampleProfReader.cpp - Read LLVM sample profile data ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the class that reads LLVM sample profiles. It
10// supports three file formats: text, binary and gcov.
11//
12// The textual representation is useful for debugging and testing purposes. The
13// binary representation is more compact, resulting in smaller file sizes.
14//
15// The gcov encoding is the one generated by GCC's AutoFDO profile creation
16// tool (https://github.com/google/autofdo)
17//
18// All three encodings can be used interchangeably as an input sample profile.
19//
20//===----------------------------------------------------------------------===//
21
22#include "llvm/ProfileData/SampleProfReader.h"
23#include "llvm/ADT/DenseMap.h"
24#include "llvm/ADT/STLExtras.h"
25#include "llvm/ADT/StringRef.h"
26#include "llvm/IR/Module.h"
27#include "llvm/IR/ProfileSummary.h"
28#include "llvm/ProfileData/ProfileCommon.h"
29#include "llvm/ProfileData/SampleProf.h"
30#include "llvm/Support/CommandLine.h"
31#include "llvm/Support/Compression.h"
32#include "llvm/Support/ErrorOr.h"
33#include "llvm/Support/JSON.h"
34#include "llvm/Support/LEB128.h"
35#include "llvm/Support/LineIterator.h"
36#include "llvm/Support/MD5.h"
37#include "llvm/Support/MemoryBuffer.h"
38#include "llvm/Support/VirtualFileSystem.h"
39#include "llvm/Support/raw_ostream.h"
40#include <algorithm>
41#include <cstddef>
42#include <cstdint>
43#include <limits>
44#include <memory>
45#include <system_error>
46#include <vector>
47
48using namespace llvm;
49using namespace sampleprof;
50
51#define DEBUG_TYPE "samplepgo-reader"
52
53// This internal option specifies if the profile uses FS discriminators.
54// It only applies to text, and binary format profiles.
55// For ext-binary format profiles, the flag is set in the summary.
56static cl::opt<bool> ProfileIsFSDisciminator(
57 "profile-isfs", cl::Hidden, cl::init(Val: false),
58 cl::desc("Profile uses flow sensitive discriminators"));
59
60static cl::opt<bool>
61 LazyLoadNameTable("sample-profile-lazy-load-name-table", cl::init(Val: true),
62 cl::Hidden,
63 cl::desc("Lazy load the name table from the profile."));
64
65/// Dump the function profile for \p FName.
66///
67/// \param FContext Name + context of the function to print.
68/// \param OS Stream to emit the output to.
69void SampleProfileReader::dumpFunctionProfile(const FunctionSamples &FS,
70 raw_ostream &OS) {
71 OS << "Function: " << FS.getContext().toString() << ": " << FS;
72}
73
74/// Dump all the function profiles found on stream \p OS.
75void SampleProfileReader::dump(raw_ostream &OS) {
76 std::vector<NameFunctionSamples> V;
77 sortFuncProfiles(ProfileMap: Profiles, SortedProfiles&: V);
78 for (const auto &I : V)
79 dumpFunctionProfile(FS: *I.second, OS);
80}
81
82static void dumpFunctionProfileJson(const FunctionSamples &S,
83 json::OStream &JOS, bool TopLevel = false) {
84 auto DumpBody = [&](const BodySampleMap &BodySamples) {
85 for (const auto &I : BodySamples) {
86 const LineLocation &Loc = I.first;
87 const SampleRecord &Sample = I.second;
88 JOS.object(Contents: [&] {
89 JOS.attribute(Key: "line", Contents: Loc.LineOffset);
90 if (Loc.Discriminator)
91 JOS.attribute(Key: "discriminator", Contents: Loc.Discriminator);
92 JOS.attribute(Key: "samples", Contents: Sample.getSamples());
93
94 auto CallTargets = Sample.getSortedCallTargets();
95 if (!CallTargets.empty()) {
96 JOS.attributeArray(Key: "calls", Contents: [&] {
97 for (const auto &J : CallTargets) {
98 JOS.object(Contents: [&] {
99 JOS.attribute(Key: "function", Contents: J.first.str());
100 JOS.attribute(Key: "samples", Contents: J.second);
101 });
102 }
103 });
104 }
105 });
106 }
107 };
108
109 auto DumpCallsiteSamples = [&](const CallsiteSampleMap &CallsiteSamples) {
110 for (const auto &I : CallsiteSamples)
111 for (const auto &FS : I.second) {
112 const LineLocation &Loc = I.first;
113 const FunctionSamples &CalleeSamples = FS.second;
114 JOS.object(Contents: [&] {
115 JOS.attribute(Key: "line", Contents: Loc.LineOffset);
116 if (Loc.Discriminator)
117 JOS.attribute(Key: "discriminator", Contents: Loc.Discriminator);
118 JOS.attributeArray(
119 Key: "samples", Contents: [&] { dumpFunctionProfileJson(S: CalleeSamples, JOS); });
120 });
121 }
122 };
123
124 JOS.object(Contents: [&] {
125 JOS.attribute(Key: "name", Contents: S.getFunction().str());
126 JOS.attribute(Key: "total", Contents: S.getTotalSamples());
127 if (TopLevel)
128 JOS.attribute(Key: "head", Contents: S.getHeadSamples());
129
130 const auto &BodySamples = S.getBodySamples();
131 if (!BodySamples.empty())
132 JOS.attributeArray(Key: "body", Contents: [&] { DumpBody(BodySamples); });
133
134 const auto &CallsiteSamples = S.getCallsiteSamples();
135 if (!CallsiteSamples.empty())
136 JOS.attributeArray(Key: "callsites",
137 Contents: [&] { DumpCallsiteSamples(CallsiteSamples); });
138 });
139}
140
141/// Dump all the function profiles found on stream \p OS in the JSON format.
142void SampleProfileReader::dumpJson(raw_ostream &OS) {
143 std::vector<NameFunctionSamples> V;
144 sortFuncProfiles(ProfileMap: Profiles, SortedProfiles&: V);
145 json::OStream JOS(OS, 2);
146 JOS.arrayBegin();
147 for (const auto &F : V)
148 dumpFunctionProfileJson(S: *F.second, JOS, TopLevel: true);
149 JOS.arrayEnd();
150
151 // Emit a newline character at the end as json::OStream doesn't emit one.
152 OS << "\n";
153}
154
155/// Parse \p Input as function head.
156///
157/// Parse one line of \p Input, and update function name in \p FName,
158/// function's total sample count in \p NumSamples, function's entry
159/// count in \p NumHeadSamples.
160///
161/// \returns true if parsing is successful.
162static bool ParseHead(const StringRef &Input, StringRef &FName,
163 uint64_t &NumSamples, uint64_t &NumHeadSamples) {
164 if (Input[0] == ' ')
165 return false;
166 size_t n2 = Input.rfind(C: ':');
167 size_t n1 = Input.rfind(C: ':', From: n2 - 1);
168 FName = Input.substr(Start: 0, N: n1);
169 if (Input.substr(Start: n1 + 1, N: n2 - n1 - 1).getAsInteger(Radix: 10, Result&: NumSamples))
170 return false;
171 if (Input.substr(Start: n2 + 1).getAsInteger(Radix: 10, Result&: NumHeadSamples))
172 return false;
173 return true;
174}
175
176/// Returns true if line offset \p L is legal (only has 16 bits).
177static bool isOffsetLegal(unsigned L) { return (L & 0xffff) == L; }
178
179/// Parse \p Input that contains metadata.
180/// Possible metadata:
181/// - CFG Checksum information:
182/// !CFGChecksum: 12345
183/// - CFG Checksum information:
184/// !Attributes: 1
185/// Stores the FunctionHash (a.k.a. CFG Checksum) into \p FunctionHash.
186static bool parseMetadata(const StringRef &Input, uint64_t &FunctionHash,
187 uint32_t &Attributes) {
188 if (Input.starts_with(Prefix: "!CFGChecksum:")) {
189 StringRef CFGInfo = Input.substr(Start: strlen(s: "!CFGChecksum:")).trim();
190 return !CFGInfo.getAsInteger(Radix: 10, Result&: FunctionHash);
191 }
192
193 if (Input.starts_with(Prefix: "!Attributes:")) {
194 StringRef Attrib = Input.substr(Start: strlen(s: "!Attributes:")).trim();
195 return !Attrib.getAsInteger(Radix: 10, Result&: Attributes);
196 }
197
198 return false;
199}
200
201enum class LineType {
202 CallSiteProfile,
203 BodyProfile,
204 Metadata,
205 VirtualCallTypeProfile,
206};
207
208// Parse `Input` as a white-space separated list of `vtable:count` pairs. An
209// example input line is `_ZTVbar:1471 _ZTVfoo:630`.
210static bool parseTypeCountMap(StringRef Input,
211 DenseMap<StringRef, uint64_t> &TypeCountMap) {
212 for (size_t Index = Input.find_first_not_of(C: ' '); Index != StringRef::npos;) {
213 size_t ColonIndex = Input.find(C: ':', From: Index);
214 if (ColonIndex == StringRef::npos)
215 return false; // No colon found, invalid format.
216 StringRef TypeName = Input.substr(Start: Index, N: ColonIndex - Index);
217 // CountIndex is the start index of count.
218 size_t CountStartIndex = ColonIndex + 1;
219 // NextIndex is the start index after the 'target:count' pair.
220 size_t NextIndex = Input.find_first_of(C: ' ', From: CountStartIndex);
221 uint64_t Count;
222 if (Input.substr(Start: CountStartIndex, N: NextIndex - CountStartIndex)
223 .getAsInteger(Radix: 10, Result&: Count))
224 return false; // Invalid count.
225 // Error on duplicated type names in one line of input.
226 auto [Iter, Inserted] = TypeCountMap.insert(KV: {TypeName, Count});
227 if (!Inserted)
228 return false;
229 Index = (NextIndex == StringRef::npos)
230 ? StringRef::npos
231 : Input.find_first_not_of(C: ' ', From: NextIndex);
232 }
233 return true;
234}
235
236/// Parse \p Input as line sample.
237///
238/// \param Input input line.
239/// \param LineTy Type of this line.
240/// \param Depth the depth of the inline stack.
241/// \param NumSamples total samples of the line/inlined callsite.
242/// \param LineOffset line offset to the start of the function.
243/// \param Discriminator discriminator of the line.
244/// \param TargetCountMap map from indirect call target to count.
245/// \param FunctionHash the function's CFG hash, used by pseudo probe.
246///
247/// returns true if parsing is successful.
248static bool ParseLine(const StringRef &Input, LineType &LineTy, uint32_t &Depth,
249 uint64_t &NumSamples, uint32_t &LineOffset,
250 uint32_t &Discriminator, StringRef &CalleeName,
251 DenseMap<StringRef, uint64_t> &TargetCountMap,
252 DenseMap<StringRef, uint64_t> &TypeCountMap,
253 uint64_t &FunctionHash, uint32_t &Attributes,
254 bool &IsFlat) {
255 for (Depth = 0; Input[Depth] == ' '; Depth++)
256 ;
257 if (Depth == 0)
258 return false;
259
260 if (Input[Depth] == '!') {
261 LineTy = LineType::Metadata;
262 // This metadata is only for manual inspection only. We already created a
263 // FunctionSamples and put it in the profile map, so there is no point
264 // to skip profiles even they have no use for ThinLTO.
265 if (Input == StringRef(" !Flat")) {
266 IsFlat = true;
267 return true;
268 }
269 return parseMetadata(Input: Input.substr(Start: Depth), FunctionHash, Attributes);
270 }
271
272 size_t n1 = Input.find(C: ':');
273 StringRef Loc = Input.substr(Start: Depth, N: n1 - Depth);
274 size_t n2 = Loc.find(C: '.');
275 if (n2 == StringRef::npos) {
276 if (Loc.getAsInteger(Radix: 10, Result&: LineOffset) || !isOffsetLegal(L: LineOffset))
277 return false;
278 Discriminator = 0;
279 } else {
280 if (Loc.substr(Start: 0, N: n2).getAsInteger(Radix: 10, Result&: LineOffset))
281 return false;
282 if (Loc.substr(Start: n2 + 1).getAsInteger(Radix: 10, Result&: Discriminator))
283 return false;
284 }
285
286 StringRef Rest = Input.substr(Start: n1 + 2);
287 if (isDigit(C: Rest[0])) {
288 LineTy = LineType::BodyProfile;
289 size_t n3 = Rest.find(C: ' ');
290 if (n3 == StringRef::npos) {
291 if (Rest.getAsInteger(Radix: 10, Result&: NumSamples))
292 return false;
293 } else {
294 if (Rest.substr(Start: 0, N: n3).getAsInteger(Radix: 10, Result&: NumSamples))
295 return false;
296 }
297 // Find call targets and their sample counts.
298 // Note: In some cases, there are symbols in the profile which are not
299 // mangled. To accommodate such cases, use colon + integer pairs as the
300 // anchor points.
301 // An example:
302 // _M_construct<char *>:1000 string_view<std::allocator<char> >:437
303 // ":1000" and ":437" are used as anchor points so the string above will
304 // be interpreted as
305 // target: _M_construct<char *>
306 // count: 1000
307 // target: string_view<std::allocator<char> >
308 // count: 437
309 while (n3 != StringRef::npos) {
310 n3 += Rest.substr(Start: n3).find_first_not_of(C: ' ');
311 Rest = Rest.substr(Start: n3);
312 n3 = Rest.find_first_of(C: ':');
313 if (n3 == StringRef::npos || n3 == 0)
314 return false;
315
316 StringRef Target;
317 uint64_t count, n4;
318 while (true) {
319 // Get the segment after the current colon.
320 StringRef AfterColon = Rest.substr(Start: n3 + 1);
321 // Get the target symbol before the current colon.
322 Target = Rest.substr(Start: 0, N: n3);
323 // Check if the word after the current colon is an integer.
324 n4 = AfterColon.find_first_of(C: ' ');
325 n4 = (n4 != StringRef::npos) ? n3 + n4 + 1 : Rest.size();
326 StringRef WordAfterColon = Rest.substr(Start: n3 + 1, N: n4 - n3 - 1);
327 if (!WordAfterColon.getAsInteger(Radix: 10, Result&: count))
328 break;
329
330 // Try to find the next colon.
331 uint64_t n5 = AfterColon.find_first_of(C: ':');
332 if (n5 == StringRef::npos)
333 return false;
334 n3 += n5 + 1;
335 }
336
337 // An anchor point is found. Save the {target, count} pair
338 TargetCountMap[Target] = count;
339 if (n4 == Rest.size())
340 break;
341 // Change n3 to the next blank space after colon + integer pair.
342 n3 = n4;
343 }
344 } else if (Rest.starts_with(Prefix: kVTableProfPrefix)) {
345 LineTy = LineType::VirtualCallTypeProfile;
346 return parseTypeCountMap(Input: Rest.substr(Start: strlen(s: kVTableProfPrefix)),
347 TypeCountMap);
348 } else {
349 LineTy = LineType::CallSiteProfile;
350 size_t n3 = Rest.find_last_of(C: ':');
351 CalleeName = Rest.substr(Start: 0, N: n3);
352 if (Rest.substr(Start: n3 + 1).getAsInteger(Radix: 10, Result&: NumSamples))
353 return false;
354 }
355 return true;
356}
357
358/// Load samples from a text file.
359///
360/// See the documentation at the top of the file for an explanation of
361/// the expected format.
362///
363/// \returns true if the file was loaded successfully, false otherwise.
364std::error_code SampleProfileReaderText::readImpl() {
365 line_iterator LineIt(*Buffer, /*SkipBlanks=*/true, '#');
366 sampleprof_error Result = sampleprof_error::success;
367
368 InlineCallStack InlineStack;
369 uint32_t TopLevelProbeProfileCount = 0;
370
371 // DepthMetadata tracks whether we have processed metadata for the current
372 // top-level or nested function profile.
373 uint32_t DepthMetadata = 0;
374
375 std::vector<SampleContext *> FlatSamples;
376
377 ProfileIsFS = ProfileIsFSDisciminator;
378 FunctionSamples::ProfileIsFS = ProfileIsFS;
379 for (; !LineIt.is_at_eof(); ++LineIt) {
380 size_t pos = LineIt->find_first_not_of(C: ' ');
381 if (pos == LineIt->npos || (*LineIt)[pos] == '#')
382 continue;
383 // Read the header of each function.
384 //
385 // Note that for function identifiers we are actually expecting
386 // mangled names, but we may not always get them. This happens when
387 // the compiler decides not to emit the function (e.g., it was inlined
388 // and removed). In this case, the binary will not have the linkage
389 // name for the function, so the profiler will emit the function's
390 // unmangled name, which may contain characters like ':' and '>' in its
391 // name (member functions, templates, etc).
392 //
393 // The only requirement we place on the identifier, then, is that it
394 // should not begin with a number.
395 if ((*LineIt)[0] != ' ') {
396 uint64_t NumSamples, NumHeadSamples;
397 StringRef FName;
398 if (!ParseHead(Input: *LineIt, FName, NumSamples, NumHeadSamples)) {
399 reportError(LineNumber: LineIt.line_number(),
400 Msg: "Expected 'mangled_name:NUM:NUM', found " + *LineIt);
401 return sampleprof_error::malformed;
402 }
403 DepthMetadata = 0;
404 SampleContext FContext(FName, CSNameTable);
405 if (FContext.hasContext())
406 ++CSProfileCount;
407 FunctionSamples &FProfile = Profiles.create(Ctx: FContext);
408 mergeSampleProfErrors(Accumulator&: Result, Result: FProfile.addTotalSamples(Num: NumSamples));
409 mergeSampleProfErrors(Accumulator&: Result, Result: FProfile.addHeadSamples(Num: NumHeadSamples));
410 InlineStack.clear();
411 InlineStack.push_back(Elt: &FProfile);
412 } else {
413 uint64_t NumSamples;
414 StringRef FName;
415 DenseMap<StringRef, uint64_t> TargetCountMap;
416 DenseMap<StringRef, uint64_t> TypeCountMap;
417 uint32_t Depth, LineOffset, Discriminator;
418 LineType LineTy = LineType::BodyProfile;
419 uint64_t FunctionHash = 0;
420 uint32_t Attributes = 0;
421 bool IsFlat = false;
422 // TODO: Update ParseLine to return an error code instead of a bool and
423 // report it.
424 if (!ParseLine(Input: *LineIt, LineTy, Depth, NumSamples, LineOffset,
425 Discriminator, CalleeName&: FName, TargetCountMap, TypeCountMap,
426 FunctionHash, Attributes, IsFlat)) {
427 switch (LineTy) {
428 case LineType::Metadata:
429 reportError(LineNumber: LineIt.line_number(),
430 Msg: "Cannot parse metadata: " + *LineIt);
431 break;
432 case LineType::VirtualCallTypeProfile:
433 reportError(LineNumber: LineIt.line_number(),
434 Msg: "Expected 'vtables [mangled_vtable:NUM]+', found " +
435 *LineIt);
436 break;
437 default:
438 reportError(LineNumber: LineIt.line_number(),
439 Msg: "Expected 'NUM[.NUM]: NUM[ mangled_name:NUM]*', found " +
440 *LineIt);
441 }
442 return sampleprof_error::malformed;
443 }
444 if (LineTy != LineType::Metadata && Depth == DepthMetadata) {
445 // Metadata must be put at the end of a function profile.
446 reportError(LineNumber: LineIt.line_number(),
447 Msg: "Found non-metadata after metadata: " + *LineIt);
448 return sampleprof_error::malformed;
449 }
450
451 // Here we handle FS discriminators.
452 Discriminator &= getDiscriminatorMask();
453
454 while (InlineStack.size() > Depth) {
455 InlineStack.pop_back();
456 }
457 switch (LineTy) {
458 case LineType::CallSiteProfile: {
459 FunctionSamples &FSamples = InlineStack.back()->functionSamplesAt(
460 Loc: LineLocation(LineOffset, Discriminator))[FunctionId(FName)];
461 FSamples.setFunction(FunctionId(FName));
462 mergeSampleProfErrors(Accumulator&: Result, Result: FSamples.addTotalSamples(Num: NumSamples));
463 InlineStack.push_back(Elt: &FSamples);
464 DepthMetadata = 0;
465 break;
466 }
467
468 case LineType::VirtualCallTypeProfile: {
469 mergeSampleProfErrors(
470 Accumulator&: Result, Result: InlineStack.back()->addCallsiteVTableTypeProfAt(
471 Loc: LineLocation(LineOffset, Discriminator), Other: TypeCountMap));
472 break;
473 }
474
475 case LineType::BodyProfile: {
476 FunctionSamples &FProfile = *InlineStack.back();
477 for (const auto &name_count : TargetCountMap) {
478 mergeSampleProfErrors(Accumulator&: Result, Result: FProfile.addCalledTargetSamples(
479 LineOffset, Discriminator,
480 Func: FunctionId(name_count.first),
481 Num: name_count.second));
482 }
483 mergeSampleProfErrors(
484 Accumulator&: Result,
485 Result: FProfile.addBodySamples(LineOffset, Discriminator, Num: NumSamples));
486 break;
487 }
488 case LineType::Metadata: {
489 FunctionSamples &FProfile = *InlineStack.back();
490 if (FunctionHash) {
491 FProfile.setFunctionHash(FunctionHash);
492 if (Depth == 1)
493 ++TopLevelProbeProfileCount;
494 }
495 FProfile.getContext().setAllAttributes(Attributes);
496 if (Attributes & (uint32_t)ContextShouldBeInlined)
497 ProfileIsPreInlined = true;
498 DepthMetadata = Depth;
499 if (IsFlat) {
500 if (Depth == 1)
501 FlatSamples.push_back(x: &FProfile.getContext());
502 else
503 Ctx.diagnose(DI: DiagnosticInfoSampleProfile(
504 Buffer->getBufferIdentifier(), LineIt.line_number(),
505 "!Flat may only be used at top level function.", DS_Warning));
506 }
507 break;
508 }
509 }
510 }
511 }
512
513 // Honor the option to skip flat functions. Since they are already added to
514 // the profile map, remove them all here.
515 if (SkipFlatProf)
516 for (SampleContext *FlatSample : FlatSamples)
517 Profiles.erase(Ctx: *FlatSample);
518
519 assert((CSProfileCount == 0 || CSProfileCount == Profiles.size()) &&
520 "Cannot have both context-sensitive and regular profile");
521 ProfileIsCS = (CSProfileCount > 0);
522 assert((TopLevelProbeProfileCount == 0 ||
523 TopLevelProbeProfileCount == Profiles.size()) &&
524 "Cannot have both probe-based profiles and regular profiles");
525 ProfileIsProbeBased = (TopLevelProbeProfileCount > 0);
526 FunctionSamples::ProfileIsProbeBased = ProfileIsProbeBased;
527 FunctionSamples::ProfileIsCS = ProfileIsCS;
528 FunctionSamples::ProfileIsPreInlined = ProfileIsPreInlined;
529
530 if (Result == sampleprof_error::success)
531 computeSummary();
532
533 return Result;
534}
535
536bool SampleProfileReaderText::hasFormat(const MemoryBuffer &Buffer) {
537 bool result = false;
538
539 // Check that the first non-comment line is a valid function header.
540 line_iterator LineIt(Buffer, /*SkipBlanks=*/true, '#');
541 if (!LineIt.is_at_eof()) {
542 if ((*LineIt)[0] != ' ') {
543 uint64_t NumSamples, NumHeadSamples;
544 StringRef FName;
545 result = ParseHead(Input: *LineIt, FName, NumSamples, NumHeadSamples);
546 }
547 }
548
549 return result;
550}
551
552template <typename T> ErrorOr<T> SampleProfileReaderBinary::readNumber() {
553 unsigned NumBytesRead = 0;
554 uint64_t Val = decodeULEB128(p: Data, n: &NumBytesRead);
555
556 if (Val > std::numeric_limits<T>::max()) {
557 std::error_code EC = sampleprof_error::malformed;
558 reportError(LineNumber: 0, Msg: EC.message());
559 return EC;
560 } else if (Data + NumBytesRead > End) {
561 std::error_code EC = sampleprof_error::truncated;
562 reportError(LineNumber: 0, Msg: EC.message());
563 return EC;
564 }
565
566 Data += NumBytesRead;
567 return static_cast<T>(Val);
568}
569
570ErrorOr<StringRef> SampleProfileReaderBinary::readString() {
571 StringRef Str(reinterpret_cast<const char *>(Data));
572 if (Data + Str.size() + 1 > End) {
573 std::error_code EC = sampleprof_error::truncated;
574 reportError(LineNumber: 0, Msg: EC.message());
575 return EC;
576 }
577
578 Data += Str.size() + 1;
579 return Str;
580}
581
582template <typename T>
583ErrorOr<T> SampleProfileReaderBinary::readUnencodedNumber() {
584 if (Data + sizeof(T) > End) {
585 std::error_code EC = sampleprof_error::truncated;
586 reportError(LineNumber: 0, Msg: EC.message());
587 return EC;
588 }
589
590 using namespace support;
591 T Val = endian::readNext<T, llvm::endianness::little>(Data);
592 return Val;
593}
594
595template <typename T>
596inline ErrorOr<size_t> SampleProfileReaderBinary::readStringIndex(T &Table) {
597 auto Idx = readNumber<size_t>();
598 if (std::error_code EC = Idx.getError())
599 return EC;
600 if (*Idx >= Table.size())
601 return sampleprof_error::truncated_name_table;
602 return *Idx;
603}
604
605ErrorOr<FunctionId>
606SampleProfileReaderBinary::readStringFromTable(size_t *RetIdx) {
607 auto Idx = readStringIndex(Table&: NameTable);
608 if (std::error_code EC = Idx.getError())
609 return EC;
610 if (RetIdx)
611 *RetIdx = *Idx;
612 return NameTable[*Idx];
613}
614
615ErrorOr<SampleContextFrames>
616SampleProfileReaderBinary::readContextFromTable(size_t *RetIdx) {
617 auto ContextIdx = readNumber<size_t>();
618 if (std::error_code EC = ContextIdx.getError())
619 return EC;
620 if (*ContextIdx >= CSNameTable.size())
621 return sampleprof_error::truncated_name_table;
622 if (RetIdx)
623 *RetIdx = *ContextIdx;
624 return CSNameTable[*ContextIdx];
625}
626
627ErrorOr<std::pair<SampleContext, uint64_t>>
628SampleProfileReaderBinary::readSampleContextFromTable() {
629 SampleContext Context;
630 size_t Idx;
631 if (ProfileIsCS) {
632 auto FContext(readContextFromTable(RetIdx: &Idx));
633 if (std::error_code EC = FContext.getError())
634 return EC;
635 Context = SampleContext(*FContext);
636 } else {
637 auto FName(readStringFromTable(RetIdx: &Idx));
638 if (std::error_code EC = FName.getError())
639 return EC;
640 Context = SampleContext(*FName);
641 }
642 // Since MD5SampleContextStart may point to the profile's file data, need to
643 // make sure it is reading the same value on big endian CPU.
644 uint64_t Hash = support::endian::read64le(P: MD5SampleContextStart + Idx);
645 // Lazy computing of hash value, write back to the table to cache it. Only
646 // compute the context's hash value if it is being referenced for the first
647 // time.
648 if (Hash == 0) {
649 assert(MD5SampleContextStart == MD5SampleContextTable.data());
650 Hash = Context.getHashCode();
651 support::endian::write64le(P: &MD5SampleContextTable[Idx], V: Hash);
652 }
653 return std::make_pair(x&: Context, y&: Hash);
654}
655
656std::error_code
657SampleProfileReaderBinary::readVTableTypeCountMap(TypeCountMap &M) {
658 auto NumVTableTypes = readNumber<uint32_t>();
659 if (std::error_code EC = NumVTableTypes.getError())
660 return EC;
661
662 for (uint32_t I = 0; I < *NumVTableTypes; ++I) {
663 auto VTableType(readStringFromTable());
664 if (std::error_code EC = VTableType.getError())
665 return EC;
666
667 auto VTableSamples = readNumber<uint64_t>();
668 if (std::error_code EC = VTableSamples.getError())
669 return EC;
670 // The source profile should not have duplicate vtable records at the same
671 // location. In case duplicate vtables are found, reader can emit a warning
672 // but continue processing the profile.
673 if (!M.insert(x: std::make_pair(x&: *VTableType, y&: *VTableSamples)).second) {
674 Ctx.diagnose(DI: DiagnosticInfoSampleProfile(
675 Buffer->getBufferIdentifier(), 0,
676 "Duplicate vtable type " + VTableType->str() +
677 " at the same location. Additional counters will be ignored.",
678 DS_Warning));
679 continue;
680 }
681 }
682 return sampleprof_error::success;
683}
684
685std::error_code
686SampleProfileReaderBinary::readCallsiteVTableProf(FunctionSamples &FProfile) {
687 assert(ReadVTableProf &&
688 "Cannot read vtable profiles if ReadVTableProf is false");
689
690 // Read the vtable type profile for the callsite.
691 auto NumCallsites = readNumber<uint32_t>();
692 if (std::error_code EC = NumCallsites.getError())
693 return EC;
694
695 for (uint32_t I = 0; I < *NumCallsites; ++I) {
696 auto LineOffset = readNumber<uint64_t>();
697 if (std::error_code EC = LineOffset.getError())
698 return EC;
699
700 if (!isOffsetLegal(L: *LineOffset))
701 return sampleprof_error::illegal_line_offset;
702
703 auto Discriminator = readNumber<uint64_t>();
704 if (std::error_code EC = Discriminator.getError())
705 return EC;
706
707 // Here we handle FS discriminators:
708 const uint32_t DiscriminatorVal = (*Discriminator) & getDiscriminatorMask();
709
710 if (std::error_code EC = readVTableTypeCountMap(M&: FProfile.getTypeSamplesAt(
711 Loc: LineLocation(*LineOffset, DiscriminatorVal))))
712 return EC;
713 }
714 return sampleprof_error::success;
715}
716
717std::error_code
718SampleProfileReaderBinary::readProfile(FunctionSamples &FProfile) {
719 auto NumSamples = readNumber<uint64_t>();
720 if (std::error_code EC = NumSamples.getError())
721 return EC;
722 FProfile.addTotalSamples(Num: *NumSamples);
723
724 // Read the samples in the body.
725 auto NumRecords = readNumber<uint32_t>();
726 if (std::error_code EC = NumRecords.getError())
727 return EC;
728
729 for (uint32_t I = 0; I < *NumRecords; ++I) {
730 auto LineOffset = readNumber<uint64_t>();
731 if (std::error_code EC = LineOffset.getError())
732 return EC;
733
734 if (!isOffsetLegal(L: *LineOffset)) {
735 return sampleprof_error::illegal_line_offset;
736 }
737
738 auto Discriminator = readNumber<uint64_t>();
739 if (std::error_code EC = Discriminator.getError())
740 return EC;
741
742 auto NumSamples = readNumber<uint64_t>();
743 if (std::error_code EC = NumSamples.getError())
744 return EC;
745
746 auto NumCalls = readNumber<uint32_t>();
747 if (std::error_code EC = NumCalls.getError())
748 return EC;
749
750 // Here we handle FS discriminators:
751 uint32_t DiscriminatorVal = (*Discriminator) & getDiscriminatorMask();
752
753 for (uint32_t J = 0; J < *NumCalls; ++J) {
754 auto CalledFunction(readStringFromTable());
755 if (std::error_code EC = CalledFunction.getError())
756 return EC;
757
758 auto CalledFunctionSamples = readNumber<uint64_t>();
759 if (std::error_code EC = CalledFunctionSamples.getError())
760 return EC;
761
762 FProfile.addCalledTargetSamples(LineOffset: *LineOffset, Discriminator: DiscriminatorVal,
763 Func: *CalledFunction, Num: *CalledFunctionSamples);
764 }
765
766 FProfile.addBodySamples(LineOffset: *LineOffset, Discriminator: DiscriminatorVal, Num: *NumSamples);
767 }
768
769 // Read all the samples for inlined function calls.
770 auto NumCallsites = readNumber<uint32_t>();
771 if (std::error_code EC = NumCallsites.getError())
772 return EC;
773
774 for (uint32_t J = 0; J < *NumCallsites; ++J) {
775 auto LineOffset = readNumber<uint64_t>();
776 if (std::error_code EC = LineOffset.getError())
777 return EC;
778
779 auto Discriminator = readNumber<uint64_t>();
780 if (std::error_code EC = Discriminator.getError())
781 return EC;
782
783 auto FName(readStringFromTable());
784 if (std::error_code EC = FName.getError())
785 return EC;
786
787 // Here we handle FS discriminators:
788 uint32_t DiscriminatorVal = (*Discriminator) & getDiscriminatorMask();
789
790 FunctionSamples &CalleeProfile = FProfile.functionSamplesAt(
791 Loc: LineLocation(*LineOffset, DiscriminatorVal))[*FName];
792 CalleeProfile.setFunction(*FName);
793 if (std::error_code EC = readProfile(FProfile&: CalleeProfile))
794 return EC;
795 }
796
797 if (ReadVTableProf)
798 return readCallsiteVTableProf(FProfile);
799
800 return sampleprof_error::success;
801}
802
803std::error_code
804SampleProfileReaderBinary::readFuncProfile(const uint8_t *Start,
805 SampleProfileMap &Profiles) {
806 Data = Start;
807 auto NumHeadSamples = readNumber<uint64_t>();
808 if (std::error_code EC = NumHeadSamples.getError())
809 return EC;
810
811 auto FContextHash(readSampleContextFromTable());
812 if (std::error_code EC = FContextHash.getError())
813 return EC;
814
815 auto &[FContext, Hash] = *FContextHash;
816 // Use the cached hash value for insertion instead of recalculating it.
817 auto Res = Profiles.try_emplace(Hash, Key: FContext, Args: FunctionSamples());
818 FunctionSamples &FProfile = Res.first->second;
819 FProfile.setContext(FContext);
820 FProfile.addHeadSamples(Num: *NumHeadSamples);
821
822 if (FContext.hasContext())
823 CSProfileCount++;
824
825 if (std::error_code EC = readProfile(FProfile))
826 return EC;
827 return sampleprof_error::success;
828}
829
830std::error_code
831SampleProfileReaderBinary::readFuncProfile(const uint8_t *Start) {
832 return readFuncProfile(Start, Profiles);
833}
834
835std::error_code SampleProfileReaderBinary::readImpl() {
836 ProfileIsFS = ProfileIsFSDisciminator;
837 FunctionSamples::ProfileIsFS = ProfileIsFS;
838 while (Data < End) {
839 if (std::error_code EC = readFuncProfile(Start: Data))
840 return EC;
841 }
842
843 return sampleprof_error::success;
844}
845
846std::error_code SampleProfileReaderExtBinaryBase::readOneSection(
847 const uint8_t *Start, uint64_t Size, const SecHdrTableEntry &Entry) {
848 Data = Start;
849 End = Start + Size;
850 switch (Entry.Type) {
851 case SecProfSummary:
852 if (std::error_code EC = readSummary())
853 return EC;
854 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagPartial))
855 Summary->setPartialProfile(true);
856 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagFullContext))
857 FunctionSamples::ProfileIsCS = ProfileIsCS = true;
858 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagIsPreInlined))
859 FunctionSamples::ProfileIsPreInlined = ProfileIsPreInlined = true;
860 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagFSDiscriminator))
861 FunctionSamples::ProfileIsFS = ProfileIsFS = true;
862 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagHasVTableTypeProf))
863 ReadVTableProf = true;
864 break;
865 case SecNameTable: {
866 bool FixedLengthMD5 =
867 hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagFixedLengthMD5);
868 bool UseMD5 = hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagMD5Name);
869 // UseMD5 means if THIS section uses MD5, ProfileIsMD5 means if the entire
870 // profile uses MD5 for function name matching in IPO passes.
871 ProfileIsMD5 = ProfileIsMD5 || UseMD5;
872 FunctionSamples::HasUniqSuffix =
873 hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagUniqSuffix);
874 if (std::error_code EC = readNameTableSec(IsMD5: UseMD5, FixedLengthMD5))
875 return EC;
876 break;
877 }
878 case SecCSNameTable: {
879 if (std::error_code EC = readCSNameTableSec())
880 return EC;
881 break;
882 }
883 case SecLBRProfile:
884 ProfileSecRange = std::make_pair(x&: Data, y&: End);
885 if (std::error_code EC = readFuncProfiles())
886 return EC;
887 break;
888 case SecFuncOffsetTable:
889 // If module is absent, we are using LLVM tools, and need to read all
890 // profiles, so skip reading the function offset table.
891 if (!M) {
892 Data = End;
893 } else {
894 assert((!ProfileIsCS ||
895 hasSecFlag(Entry, SecFuncOffsetFlags::SecFlagOrdered)) &&
896 "func offset table should always be sorted in CS profile");
897 if (std::error_code EC = readFuncOffsetTable())
898 return EC;
899 }
900 break;
901 case SecFuncMetadata: {
902 ProfileIsProbeBased =
903 hasSecFlag(Entry, Flag: SecFuncMetadataFlags::SecFlagIsProbeBased);
904 FunctionSamples::ProfileIsProbeBased = ProfileIsProbeBased;
905 ProfileHasAttribute =
906 hasSecFlag(Entry, Flag: SecFuncMetadataFlags::SecFlagHasAttribute);
907 if (std::error_code EC = readFuncMetadata())
908 return EC;
909 break;
910 }
911 case SecProfileSymbolList:
912 if (std::error_code EC = readProfileSymbolList(
913 IsMD5: hasSecFlag(Entry, Flag: SecProfileSymbolListFlags::SecFlagMD5)))
914 return EC;
915 break;
916 default:
917 if (std::error_code EC = readCustomSection(Entry))
918 return EC;
919 break;
920 }
921 return sampleprof_error::success;
922}
923
924bool SampleProfileReaderExtBinaryBase::useFuncOffsetList() const {
925 // If profile is CS, the function offset section is expected to consist of
926 // sequences of contexts in pre-order layout
927 // (e.g. [A, A:1 @ B, A:1 @ B:2.3 @ C] [D, D:1 @ E]), so that when a matched
928 // context in the module is found, the profiles of all its callees are
929 // recursively loaded. A list is needed since the order of profiles matters.
930 if (ProfileIsCS)
931 return true;
932
933 // If the profile is MD5, use the map container to lookup functions in
934 // the module. A remapper has no use on MD5 names.
935 if (useMD5())
936 return false;
937
938 // Profile is not MD5 and if a remapper is present, the remapped name of
939 // every function needed to be matched against the module, so use the list
940 // container since each entry is accessed.
941 if (Remapper)
942 return true;
943
944 // Otherwise use the map container for faster lookup.
945 // TODO: If the cardinality of the function offset section is much smaller
946 // than the number of functions in the module, using the list container can
947 // be always faster, but we need to figure out the constant factor to
948 // determine the cutoff.
949 return false;
950}
951
952std::error_code
953SampleProfileReaderExtBinaryBase::read(const DenseSet<StringRef> &FuncsToUse,
954 SampleProfileMap &Profiles) {
955 if (FuncsToUse.empty())
956 return sampleprof_error::success;
957
958 Data = ProfileSecRange.first;
959 End = ProfileSecRange.second;
960 if (std::error_code EC = readFuncProfiles(FuncsToUse, Profiles))
961 return EC;
962 End = Data;
963 DenseSet<FunctionSamples *> ProfilesToReadMetadata;
964 for (auto FName : FuncsToUse) {
965 auto I = Profiles.find(Ctx: FName);
966 if (I != Profiles.end())
967 ProfilesToReadMetadata.insert(V: &I->second);
968 }
969
970 if (std::error_code EC = readFuncMetadata(Profiles&: ProfilesToReadMetadata))
971 return EC;
972 return sampleprof_error::success;
973}
974
975bool SampleProfileReaderExtBinaryBase::collectFuncsFromModule() {
976 if (!M)
977 return false;
978 FuncsToUse.clear();
979 for (auto &F : *M)
980 FuncsToUse.insert(V: FunctionSamples::getCanonicalFnName(F));
981 return true;
982}
983
984std::error_code SampleProfileReaderExtBinaryBase::readFuncOffsetTable() {
985 // If there are more than one function offset section, the profile associated
986 // with the previous section has to be done reading before next one is read.
987 FuncOffsetTable.reset();
988 FuncOffsetList.clear();
989
990 auto Size = readNumber<uint64_t>();
991 if (std::error_code EC = Size.getError())
992 return EC;
993
994 bool UseFuncOffsetList = useFuncOffsetList();
995 if (UseFuncOffsetList)
996 FuncOffsetList.reserve(n: *Size);
997 else
998 FuncOffsetTable.emplace(args: InMemoryMode, args&: *Size);
999
1000 for (uint64_t I = 0; I < *Size; ++I) {
1001 auto FContextHash(readSampleContextFromTable());
1002 if (std::error_code EC = FContextHash.getError())
1003 return EC;
1004
1005 auto &[FContext, Hash] = *FContextHash;
1006 auto Offset = readNumber<uint64_t>();
1007 if (std::error_code EC = Offset.getError())
1008 return EC;
1009
1010 if (UseFuncOffsetList)
1011 FuncOffsetList.emplace_back(args&: FContext, args&: *Offset);
1012 else
1013 // Because Porfiles replace existing value with new value if collision
1014 // happens, we also use the latest offset so that they are consistent.
1015 FuncOffsetTable->insert(GUID: Hash, Offset: *Offset);
1016 }
1017
1018 return sampleprof_error::success;
1019}
1020
1021std::error_code SampleProfileReaderExtBinaryBase::readFuncProfiles(
1022 const DenseSet<StringRef> &FuncsToUse, SampleProfileMap &Profiles) {
1023 const uint8_t *Start = Data;
1024
1025 if (Remapper) {
1026 for (auto Name : FuncsToUse) {
1027 Remapper->insert(FunctionName: Name);
1028 }
1029 }
1030
1031 if (ProfileIsCS) {
1032 assert(useFuncOffsetList());
1033 DenseSet<uint64_t> FuncGuidsToUse;
1034 if (useMD5()) {
1035 for (auto Name : FuncsToUse)
1036 FuncGuidsToUse.insert(V: Function::getGUIDAssumingExternalLinkage(GlobalName: Name));
1037 }
1038
1039 // For each function in current module, load all context profiles for
1040 // the function as well as their callee contexts which can help profile
1041 // guided importing for ThinLTO. This can be achieved by walking
1042 // through an ordered context container, where contexts are laid out
1043 // as if they were walked in preorder of a context trie. While
1044 // traversing the trie, a link to the highest common ancestor node is
1045 // kept so that all of its decendants will be loaded.
1046 const SampleContext *CommonContext = nullptr;
1047 for (const auto &NameOffset : FuncOffsetList) {
1048 const auto &FContext = NameOffset.first;
1049 FunctionId FName = FContext.getFunction();
1050 StringRef FNameString;
1051 if (!useMD5())
1052 FNameString = FName.stringRef();
1053
1054 // For function in the current module, keep its farthest ancestor
1055 // context. This can be used to load itself and its child and
1056 // sibling contexts.
1057 if ((useMD5() && FuncGuidsToUse.count(V: FName.getHashCode())) ||
1058 (!useMD5() && (FuncsToUse.count(V: FNameString) ||
1059 (Remapper && Remapper->exist(FunctionName: FNameString))))) {
1060 if (!CommonContext || !CommonContext->isPrefixOf(That: FContext))
1061 CommonContext = &FContext;
1062 }
1063
1064 if (CommonContext == &FContext ||
1065 (CommonContext && CommonContext->isPrefixOf(That: FContext))) {
1066 // Load profile for the current context which originated from
1067 // the common ancestor.
1068 const uint8_t *FuncProfileAddr = Start + NameOffset.second;
1069 if (std::error_code EC = readFuncProfile(Start: FuncProfileAddr))
1070 return EC;
1071 }
1072 }
1073 } else if (useMD5()) {
1074 assert(!useFuncOffsetList());
1075 for (auto Name : FuncsToUse) {
1076 auto GUID = MD5Hash(Str: Name);
1077 if (auto Offset = FuncOffsetTable->lookup(GUID)) {
1078 const uint8_t *FuncProfileAddr = Start + *Offset;
1079 if (std::error_code EC = readFuncProfile(Start: FuncProfileAddr, Profiles))
1080 return EC;
1081 }
1082 }
1083 } else if (Remapper) {
1084 assert(useFuncOffsetList());
1085 for (auto NameOffset : FuncOffsetList) {
1086 SampleContext FContext(NameOffset.first);
1087 auto FuncName = FContext.getFunction();
1088 StringRef FuncNameStr = FuncName.stringRef();
1089 if (!FuncsToUse.count(V: FuncNameStr) && !Remapper->exist(FunctionName: FuncNameStr))
1090 continue;
1091 const uint8_t *FuncProfileAddr = Start + NameOffset.second;
1092 if (std::error_code EC = readFuncProfile(Start: FuncProfileAddr, Profiles))
1093 return EC;
1094 }
1095 } else {
1096 assert(!useFuncOffsetList());
1097 for (auto Name : FuncsToUse) {
1098 if (auto Offset = FuncOffsetTable->lookup(GUID: MD5Hash(Str: Name))) {
1099 const uint8_t *FuncProfileAddr = Start + *Offset;
1100 if (std::error_code EC = readFuncProfile(Start: FuncProfileAddr, Profiles))
1101 return EC;
1102 }
1103 }
1104 }
1105
1106 return sampleprof_error::success;
1107}
1108
1109std::error_code SampleProfileReaderExtBinaryBase::readFuncProfiles() {
1110 // Collect functions used by current module if the Reader has been
1111 // given a module.
1112 // collectFuncsFromModule uses FunctionSamples::getCanonicalFnName
1113 // which will query FunctionSamples::HasUniqSuffix, so it has to be
1114 // called after FunctionSamples::HasUniqSuffix is set, i.e. after
1115 // NameTable section is read.
1116 bool LoadFuncsToBeUsed = collectFuncsFromModule();
1117
1118 // When LoadFuncsToBeUsed is false, we are using LLVM tool, need to read all
1119 // profiles.
1120 if (!LoadFuncsToBeUsed) {
1121 while (Data < End) {
1122 if (std::error_code EC = readFuncProfile(Start: Data))
1123 return EC;
1124 }
1125 assert(Data == End && "More data is read than expected");
1126 } else {
1127 // Load function profiles on demand.
1128 if (std::error_code EC = readFuncProfiles(FuncsToUse, Profiles))
1129 return EC;
1130 Data = End;
1131 }
1132 assert((CSProfileCount == 0 || CSProfileCount == Profiles.size()) &&
1133 "Cannot have both context-sensitive and regular profile");
1134 assert((!CSProfileCount || ProfileIsCS) &&
1135 "Section flag should be consistent with actual profile");
1136 return sampleprof_error::success;
1137}
1138
1139std::error_code
1140SampleProfileReaderExtBinaryBase::readProfileSymbolList(bool IsMD5) {
1141 if (IsMD5)
1142 return readMD5ProfileSymbolList();
1143 return readStringBasedProfileSymbolList();
1144}
1145
1146std::error_code SampleProfileReaderExtBinaryBase::readMD5ProfileSymbolList() {
1147 size_t Size = End - Data;
1148 if (Size % sizeof(uint64_t) != 0)
1149 return sampleprof_error::truncated;
1150 const auto *Table = reinterpret_cast<const support::ulittle64_t *>(Data);
1151 size_t NumEntries = Size / sizeof(uint64_t);
1152 if (!ProfSymList)
1153 ProfSymList = std::make_unique<ProfileSymbolList>();
1154 ProfSymList->setColdGUIDTable(
1155 EytzingerTableSpan<support::ulittle64_t>(Table, NumEntries));
1156 Data = End;
1157 return sampleprof_error::success;
1158}
1159
1160std::error_code
1161SampleProfileReaderExtBinaryBase::readStringBasedProfileSymbolList() {
1162 if (!ProfSymList)
1163 ProfSymList = std::make_unique<ProfileSymbolList>();
1164
1165 if (std::error_code EC = ProfSymList->read(Data, ListSize: End - Data))
1166 return EC;
1167
1168 Data = End;
1169 return sampleprof_error::success;
1170}
1171
1172std::error_code SampleProfileReaderExtBinaryBase::decompressSection(
1173 const uint8_t *SecStart, const uint64_t SecSize,
1174 const uint8_t *&DecompressBuf, uint64_t &DecompressBufSize) {
1175 Data = SecStart;
1176 End = SecStart + SecSize;
1177 auto DecompressSize = readNumber<uint64_t>();
1178 if (std::error_code EC = DecompressSize.getError())
1179 return EC;
1180 DecompressBufSize = *DecompressSize;
1181
1182 auto CompressSize = readNumber<uint64_t>();
1183 if (std::error_code EC = CompressSize.getError())
1184 return EC;
1185
1186 if (!llvm::compression::zlib::isAvailable())
1187 return sampleprof_error::zlib_unavailable;
1188
1189 uint8_t *Buffer = Allocator.Allocate<uint8_t>(Num: DecompressBufSize);
1190 size_t UCSize = DecompressBufSize;
1191 llvm::Error E = compression::zlib::decompress(Input: ArrayRef(Data, *CompressSize),
1192 Output: Buffer, UncompressedSize&: UCSize);
1193 if (E)
1194 return sampleprof_error::uncompress_failed;
1195 DecompressBuf = reinterpret_cast<const uint8_t *>(Buffer);
1196 return sampleprof_error::success;
1197}
1198
1199std::error_code SampleProfileReaderExtBinaryBase::readImpl() {
1200 const uint8_t *BufStart =
1201 reinterpret_cast<const uint8_t *>(Buffer->getBufferStart());
1202
1203 for (auto &Entry : SecHdrTable) {
1204 // Skip empty section.
1205 if (!Entry.Size)
1206 continue;
1207
1208 // Skip sections without inlined functions when SkipFlatProf is true.
1209 if (SkipFlatProf && hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagFlat))
1210 continue;
1211
1212 const uint8_t *SecStart = BufStart + Entry.Offset;
1213 uint64_t SecSize = Entry.Size;
1214
1215 // If the section is compressed, decompress it into a buffer
1216 // DecompressBuf before reading the actual data. The pointee of
1217 // 'Data' will be changed to buffer hold by DecompressBuf
1218 // temporarily when reading the actual data.
1219 bool isCompressed = hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagCompress);
1220 if (isCompressed) {
1221 const uint8_t *DecompressBuf;
1222 uint64_t DecompressBufSize;
1223 if (std::error_code EC = decompressSection(
1224 SecStart, SecSize, DecompressBuf, DecompressBufSize))
1225 return EC;
1226 SecStart = DecompressBuf;
1227 SecSize = DecompressBufSize;
1228 }
1229
1230 if (std::error_code EC = readOneSection(Start: SecStart, Size: SecSize, Entry))
1231 return EC;
1232 if (Data != SecStart + SecSize)
1233 return sampleprof_error::malformed;
1234
1235 // Change the pointee of 'Data' from DecompressBuf to original Buffer.
1236 if (isCompressed) {
1237 Data = BufStart + Entry.Offset;
1238 End = BufStart + Buffer->getBufferSize();
1239 }
1240 }
1241
1242 return sampleprof_error::success;
1243}
1244
1245std::error_code SampleProfileReaderRawBinary::verifySPMagic(uint64_t Magic) {
1246 if (Magic == SPMagic())
1247 return sampleprof_error::success;
1248 return sampleprof_error::bad_magic;
1249}
1250
1251std::error_code SampleProfileReaderExtBinary::verifySPMagic(uint64_t Magic) {
1252 if (Magic == SPMagic(Format: SPF_Ext_Binary))
1253 return sampleprof_error::success;
1254 return sampleprof_error::bad_magic;
1255}
1256
1257std::error_code SampleProfileReaderBinary::readNameTable() {
1258 auto Size = readNumber<size_t>();
1259 if (std::error_code EC = Size.getError())
1260 return EC;
1261
1262 // Normally if useMD5 is true, the name table should have MD5 values, not
1263 // strings, however in the case that ExtBinary profile has multiple name
1264 // tables mixing string and MD5, all of them have to be normalized to use MD5,
1265 // because optimization passes can only handle either type.
1266 bool UseMD5 = useMD5();
1267
1268 auto &TableVec = NameTable.setToEager();
1269 TableVec.reserve(n: *Size);
1270 if (!ProfileIsCS) {
1271 MD5SampleContextTable.clear();
1272 if (UseMD5)
1273 MD5SampleContextTable.reserve(n: *Size);
1274 else
1275 // If we are using strings, delay MD5 computation since only a portion of
1276 // names are used by top level functions. Use 0 to indicate MD5 value is
1277 // to be calculated as no known string has a MD5 value of 0.
1278 MD5SampleContextTable.resize(new_size: *Size);
1279 }
1280 for (size_t I = 0; I < *Size; ++I) {
1281 auto Name(readString());
1282 if (std::error_code EC = Name.getError())
1283 return EC;
1284 if (UseMD5) {
1285 FunctionId FID(*Name);
1286 if (!ProfileIsCS)
1287 MD5SampleContextTable.emplace_back(args: FID.getHashCode());
1288 TableVec.emplace_back(args&: FID);
1289 } else
1290 TableVec.push_back(x: FunctionId(*Name));
1291 }
1292 if (!ProfileIsCS)
1293 MD5SampleContextStart = MD5SampleContextTable.data();
1294 return sampleprof_error::success;
1295}
1296
1297std::error_code
1298SampleProfileReaderExtBinaryBase::readNameTableSec(bool IsMD5,
1299 bool FixedLengthMD5) {
1300 if (FixedLengthMD5) {
1301 if (!IsMD5)
1302 errs() << "If FixedLengthMD5 is true, UseMD5 has to be true";
1303 auto Size = readNumber<size_t>();
1304 if (std::error_code EC = Size.getError())
1305 return EC;
1306
1307 assert(Data + (*Size) * sizeof(uint64_t) == End &&
1308 "Fixed length MD5 name table does not contain specified number of "
1309 "entries");
1310 if (Data + (*Size) * sizeof(uint64_t) > End)
1311 return sampleprof_error::truncated;
1312
1313 if (LazyLoadNameTable) {
1314 NameTable.setLazy(S: Data, Sz: *Size);
1315 } else {
1316 auto &TableVec = NameTable.setToEager();
1317 TableVec.reserve(n: *Size);
1318 for (size_t I = 0; I < *Size; ++I) {
1319 using namespace support;
1320 uint64_t FID = endian::read<uint64_t, unaligned>(
1321 memory: Data + I * sizeof(uint64_t), endian: endianness::little);
1322 TableVec.emplace_back(args: FunctionId(FID));
1323 }
1324 }
1325 if (!ProfileIsCS)
1326 MD5SampleContextStart = reinterpret_cast<const uint64_t *>(Data);
1327 Data = Data + (*Size) * sizeof(uint64_t);
1328 return sampleprof_error::success;
1329 }
1330
1331 if (IsMD5) {
1332 assert(!FixedLengthMD5 && "FixedLengthMD5 should be unreachable here");
1333 auto Size = readNumber<size_t>();
1334 if (std::error_code EC = Size.getError())
1335 return EC;
1336
1337 auto &TableVec = NameTable.setToEager();
1338 TableVec.reserve(n: *Size);
1339 if (!ProfileIsCS)
1340 MD5SampleContextTable.resize(new_size: *Size);
1341 for (size_t I = 0; I < *Size; ++I) {
1342 auto FID = readNumber<uint64_t>();
1343 if (std::error_code EC = FID.getError())
1344 return EC;
1345 if (!ProfileIsCS)
1346 support::endian::write64le(P: &MD5SampleContextTable[I], V: *FID);
1347 TableVec.emplace_back(args: FunctionId(*FID));
1348 }
1349 if (!ProfileIsCS)
1350 MD5SampleContextStart = MD5SampleContextTable.data();
1351 return sampleprof_error::success;
1352 }
1353
1354 return SampleProfileReaderBinary::readNameTable();
1355}
1356
1357// Read in the CS name table section, which basically contains a list of context
1358// vectors. Each element of a context vector, aka a frame, refers to the
1359// underlying raw function names that are stored in the name table, as well as
1360// a callsite identifier that only makes sense for non-leaf frames.
1361std::error_code SampleProfileReaderExtBinaryBase::readCSNameTableSec() {
1362 auto Size = readNumber<size_t>();
1363 if (std::error_code EC = Size.getError())
1364 return EC;
1365
1366 CSNameTable.clear();
1367 CSNameTable.reserve(n: *Size);
1368 if (ProfileIsCS) {
1369 // Delay MD5 computation of CS context until they are needed. Use 0 to
1370 // indicate MD5 value is to be calculated as no known string has a MD5
1371 // value of 0.
1372 MD5SampleContextTable.clear();
1373 MD5SampleContextTable.resize(new_size: *Size);
1374 MD5SampleContextStart = MD5SampleContextTable.data();
1375 }
1376 for (size_t I = 0; I < *Size; ++I) {
1377 CSNameTable.emplace_back(args: SampleContextFrameVector());
1378 auto ContextSize = readNumber<uint32_t>();
1379 if (std::error_code EC = ContextSize.getError())
1380 return EC;
1381 for (uint32_t J = 0; J < *ContextSize; ++J) {
1382 auto FName(readStringFromTable());
1383 if (std::error_code EC = FName.getError())
1384 return EC;
1385 auto LineOffset = readNumber<uint64_t>();
1386 if (std::error_code EC = LineOffset.getError())
1387 return EC;
1388
1389 if (!isOffsetLegal(L: *LineOffset))
1390 return sampleprof_error::illegal_line_offset;
1391
1392 auto Discriminator = readNumber<uint64_t>();
1393 if (std::error_code EC = Discriminator.getError())
1394 return EC;
1395
1396 CSNameTable.back().emplace_back(
1397 Args&: FName.get(), Args: LineLocation(LineOffset.get(), Discriminator.get()));
1398 }
1399 }
1400
1401 return sampleprof_error::success;
1402}
1403
1404std::error_code
1405SampleProfileReaderExtBinaryBase::readFuncMetadata(FunctionSamples *FProfile) {
1406 if (Data < End) {
1407 if (ProfileIsProbeBased) {
1408 auto Checksum = readNumber<uint64_t>();
1409 if (std::error_code EC = Checksum.getError())
1410 return EC;
1411 if (FProfile)
1412 FProfile->setFunctionHash(*Checksum);
1413 }
1414
1415 if (ProfileHasAttribute) {
1416 auto Attributes = readNumber<uint32_t>();
1417 if (std::error_code EC = Attributes.getError())
1418 return EC;
1419 if (FProfile)
1420 FProfile->getContext().setAllAttributes(*Attributes);
1421 }
1422
1423 if (!ProfileIsCS) {
1424 // Read all the attributes for inlined function calls.
1425 auto NumCallsites = readNumber<uint32_t>();
1426 if (std::error_code EC = NumCallsites.getError())
1427 return EC;
1428
1429 for (uint32_t J = 0; J < *NumCallsites; ++J) {
1430 auto LineOffset = readNumber<uint64_t>();
1431 if (std::error_code EC = LineOffset.getError())
1432 return EC;
1433
1434 auto Discriminator = readNumber<uint64_t>();
1435 if (std::error_code EC = Discriminator.getError())
1436 return EC;
1437
1438 auto FContextHash(readSampleContextFromTable());
1439 if (std::error_code EC = FContextHash.getError())
1440 return EC;
1441
1442 auto &[FContext, Hash] = *FContextHash;
1443 FunctionSamples *CalleeProfile = nullptr;
1444 if (FProfile) {
1445 CalleeProfile = const_cast<FunctionSamples *>(
1446 &FProfile->functionSamplesAt(Loc: LineLocation(
1447 *LineOffset, *Discriminator))[FContext.getFunction()]);
1448 }
1449 if (std::error_code EC = readFuncMetadata(FProfile: CalleeProfile))
1450 return EC;
1451 }
1452 }
1453 }
1454
1455 return sampleprof_error::success;
1456}
1457
1458std::error_code SampleProfileReaderExtBinaryBase::readFuncMetadata(
1459 DenseSet<FunctionSamples *> &Profiles) {
1460 if (FuncMetadataIndex.empty())
1461 return sampleprof_error::success;
1462
1463 for (auto *FProfile : Profiles) {
1464 auto R = FuncMetadataIndex.find(Val: FProfile->getContext().getHashCode());
1465 if (R == FuncMetadataIndex.end())
1466 continue;
1467
1468 Data = R->second.first;
1469 End = R->second.second;
1470 if (std::error_code EC = readFuncMetadata(FProfile))
1471 return EC;
1472 assert(Data == End && "More data is read than expected");
1473 }
1474 return sampleprof_error::success;
1475}
1476
1477std::error_code SampleProfileReaderExtBinaryBase::readFuncMetadata() {
1478 while (Data < End) {
1479 auto FContextHash(readSampleContextFromTable());
1480 if (std::error_code EC = FContextHash.getError())
1481 return EC;
1482 auto &[FContext, Hash] = *FContextHash;
1483 FunctionSamples *FProfile = nullptr;
1484 auto It = Profiles.find(Ctx: FContext);
1485 if (It != Profiles.end())
1486 FProfile = &It->second;
1487
1488 const uint8_t *Start = Data;
1489 if (std::error_code EC = readFuncMetadata(FProfile))
1490 return EC;
1491
1492 FuncMetadataIndex[FContext.getHashCode()] = {Start, Data};
1493 }
1494
1495 assert(Data == End && "More data is read than expected");
1496 return sampleprof_error::success;
1497}
1498
1499std::error_code
1500SampleProfileReaderExtBinaryBase::readSecHdrTableEntry(uint64_t Idx) {
1501 SecHdrTableEntry Entry;
1502 auto Type = readUnencodedNumber<uint64_t>();
1503 if (std::error_code EC = Type.getError())
1504 return EC;
1505 Entry.Type = static_cast<SecType>(*Type);
1506
1507 auto Flags = readUnencodedNumber<uint64_t>();
1508 if (std::error_code EC = Flags.getError())
1509 return EC;
1510 Entry.Flags = *Flags;
1511
1512 auto Offset = readUnencodedNumber<uint64_t>();
1513 if (std::error_code EC = Offset.getError())
1514 return EC;
1515 Entry.Offset = *Offset;
1516
1517 auto Size = readUnencodedNumber<uint64_t>();
1518 if (std::error_code EC = Size.getError())
1519 return EC;
1520 Entry.Size = *Size;
1521
1522 Entry.LayoutIndex = Idx;
1523 SecHdrTable.push_back(x: std::move(Entry));
1524 return sampleprof_error::success;
1525}
1526
1527std::error_code SampleProfileReaderExtBinaryBase::readSecHdrTable() {
1528 auto EntryNum = readUnencodedNumber<uint64_t>();
1529 if (std::error_code EC = EntryNum.getError())
1530 return EC;
1531
1532 for (uint64_t i = 0; i < (*EntryNum); i++)
1533 if (std::error_code EC = readSecHdrTableEntry(Idx: i))
1534 return EC;
1535
1536 return sampleprof_error::success;
1537}
1538
1539std::error_code SampleProfileReaderExtBinaryBase::readHeader() {
1540 const uint8_t *BufStart =
1541 reinterpret_cast<const uint8_t *>(Buffer->getBufferStart());
1542 Data = BufStart;
1543 End = BufStart + Buffer->getBufferSize();
1544
1545 if (std::error_code EC = readMagicIdent())
1546 return EC;
1547
1548 if (std::error_code EC = readSecHdrTable())
1549 return EC;
1550
1551 return sampleprof_error::success;
1552}
1553
1554uint64_t SampleProfileReaderExtBinaryBase::getSectionSize(SecType Type) {
1555 uint64_t Size = 0;
1556 for (auto &Entry : SecHdrTable) {
1557 if (Entry.Type == Type)
1558 Size += Entry.Size;
1559 }
1560 return Size;
1561}
1562
1563uint64_t SampleProfileReaderExtBinaryBase::getFileSize() {
1564 // Sections in SecHdrTable is not necessarily in the same order as
1565 // sections in the profile because section like FuncOffsetTable needs
1566 // to be written after section LBRProfile but needs to be read before
1567 // section LBRProfile, so we cannot simply use the last entry in
1568 // SecHdrTable to calculate the file size.
1569 uint64_t FileSize = 0;
1570 for (auto &Entry : SecHdrTable) {
1571 FileSize = std::max(a: Entry.Offset + Entry.Size, b: FileSize);
1572 }
1573 return FileSize;
1574}
1575
1576static std::string getSecFlagsStr(const SecHdrTableEntry &Entry) {
1577 std::string Flags;
1578 if (hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagCompress))
1579 Flags.append(s: "{compressed,");
1580 else
1581 Flags.append(s: "{");
1582
1583 if (hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagFlat))
1584 Flags.append(s: "flat,");
1585
1586 switch (Entry.Type) {
1587 case SecNameTable:
1588 if (hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagFixedLengthMD5))
1589 Flags.append(s: "fixlenmd5,");
1590 else if (hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagMD5Name))
1591 Flags.append(s: "md5,");
1592 if (hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagUniqSuffix))
1593 Flags.append(s: "uniq,");
1594 break;
1595 case SecProfSummary:
1596 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagPartial))
1597 Flags.append(s: "partial,");
1598 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagFullContext))
1599 Flags.append(s: "context,");
1600 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagIsPreInlined))
1601 Flags.append(s: "preInlined,");
1602 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagFSDiscriminator))
1603 Flags.append(s: "fs-discriminator,");
1604 break;
1605 case SecFuncOffsetTable:
1606 if (hasSecFlag(Entry, Flag: SecFuncOffsetFlags::SecFlagOrdered))
1607 Flags.append(s: "ordered,");
1608 break;
1609 case SecFuncMetadata:
1610 if (hasSecFlag(Entry, Flag: SecFuncMetadataFlags::SecFlagIsProbeBased))
1611 Flags.append(s: "probe,");
1612 if (hasSecFlag(Entry, Flag: SecFuncMetadataFlags::SecFlagHasAttribute))
1613 Flags.append(s: "attr,");
1614 break;
1615 case SecProfileSymbolList:
1616 if (hasSecFlag(Entry, Flag: SecProfileSymbolListFlags::SecFlagMD5))
1617 Flags.append(s: "md5,");
1618 break;
1619 default:
1620 break;
1621 }
1622 char &last = Flags.back();
1623 if (last == ',')
1624 last = '}';
1625 else
1626 Flags.append(s: "}");
1627 return Flags;
1628}
1629
1630bool SampleProfileReaderExtBinaryBase::dumpSectionInfo(raw_ostream &OS) {
1631 uint64_t TotalSecsSize = 0;
1632 for (auto &Entry : SecHdrTable) {
1633 OS << getSecName(Type: Entry.Type) << " - Offset: " << Entry.Offset
1634 << ", Size: " << Entry.Size << ", Flags: " << getSecFlagsStr(Entry)
1635 << "\n";
1636 ;
1637 TotalSecsSize += Entry.Size;
1638 }
1639 uint64_t HeaderSize = SecHdrTable.front().Offset;
1640 assert(HeaderSize + TotalSecsSize == getFileSize() &&
1641 "Size of 'header + sections' doesn't match the total size of profile");
1642
1643 OS << "Header Size: " << HeaderSize << "\n";
1644 OS << "Total Sections Size: " << TotalSecsSize << "\n";
1645 OS << "File Size: " << getFileSize() << "\n";
1646 return true;
1647}
1648
1649std::error_code SampleProfileReaderBinary::readMagicIdent() {
1650 // Read and check the magic identifier.
1651 auto Magic = readNumber<uint64_t>();
1652 if (std::error_code EC = Magic.getError())
1653 return EC;
1654 else if (std::error_code EC = verifySPMagic(Magic: *Magic))
1655 return EC;
1656
1657 // Read the version number.
1658 auto Version = readNumber<uint64_t>();
1659 if (std::error_code EC = Version.getError())
1660 return EC;
1661 else if (!formatVersionIsSupported(Version: *Version))
1662 return sampleprof_error::unsupported_version;
1663 FormatVersion = *Version;
1664
1665 return sampleprof_error::success;
1666}
1667
1668std::error_code SampleProfileReaderBinary::readHeader() {
1669 Data = reinterpret_cast<const uint8_t *>(Buffer->getBufferStart());
1670 End = Data + Buffer->getBufferSize();
1671
1672 if (std::error_code EC = readMagicIdent())
1673 return EC;
1674
1675 if (std::error_code EC = readSummary())
1676 return EC;
1677
1678 if (std::error_code EC = readNameTable())
1679 return EC;
1680 return sampleprof_error::success;
1681}
1682
1683std::error_code SampleProfileReaderBinary::readSummaryEntry(
1684 std::vector<ProfileSummaryEntry> &Entries) {
1685 auto Cutoff = readNumber<uint64_t>();
1686 if (std::error_code EC = Cutoff.getError())
1687 return EC;
1688
1689 auto MinBlockCount = readNumber<uint64_t>();
1690 if (std::error_code EC = MinBlockCount.getError())
1691 return EC;
1692
1693 auto NumBlocks = readNumber<uint64_t>();
1694 if (std::error_code EC = NumBlocks.getError())
1695 return EC;
1696
1697 Entries.emplace_back(args&: *Cutoff, args&: *MinBlockCount, args&: *NumBlocks);
1698 return sampleprof_error::success;
1699}
1700
1701std::error_code SampleProfileReaderBinary::readSummary() {
1702 auto TotalCount = readNumber<uint64_t>();
1703 if (std::error_code EC = TotalCount.getError())
1704 return EC;
1705
1706 auto MaxBlockCount = readNumber<uint64_t>();
1707 if (std::error_code EC = MaxBlockCount.getError())
1708 return EC;
1709
1710 auto MaxFunctionCount = readNumber<uint64_t>();
1711 if (std::error_code EC = MaxFunctionCount.getError())
1712 return EC;
1713
1714 auto NumBlocks = readNumber<uint64_t>();
1715 if (std::error_code EC = NumBlocks.getError())
1716 return EC;
1717
1718 auto NumFunctions = readNumber<uint64_t>();
1719 if (std::error_code EC = NumFunctions.getError())
1720 return EC;
1721
1722 auto NumSummaryEntries = readNumber<uint64_t>();
1723 if (std::error_code EC = NumSummaryEntries.getError())
1724 return EC;
1725
1726 std::vector<ProfileSummaryEntry> Entries;
1727 for (unsigned i = 0; i < *NumSummaryEntries; i++) {
1728 std::error_code EC = readSummaryEntry(Entries);
1729 if (EC != sampleprof_error::success)
1730 return EC;
1731 }
1732 Summary = std::make_unique<ProfileSummary>(
1733 args: ProfileSummary::PSK_Sample, args&: Entries, args&: *TotalCount, args&: *MaxBlockCount, args: 0,
1734 args&: *MaxFunctionCount, args&: *NumBlocks, args&: *NumFunctions);
1735
1736 return sampleprof_error::success;
1737}
1738
1739bool SampleProfileReaderRawBinary::hasFormat(const MemoryBuffer &Buffer) {
1740 const uint8_t *Data =
1741 reinterpret_cast<const uint8_t *>(Buffer.getBufferStart());
1742 uint64_t Magic = decodeULEB128(p: Data);
1743 return Magic == SPMagic();
1744}
1745
1746bool SampleProfileReaderExtBinary::hasFormat(const MemoryBuffer &Buffer) {
1747 const uint8_t *Data =
1748 reinterpret_cast<const uint8_t *>(Buffer.getBufferStart());
1749 uint64_t Magic = decodeULEB128(p: Data);
1750 return Magic == SPMagic(Format: SPF_Ext_Binary);
1751}
1752
1753std::error_code SampleProfileReaderGCC::skipNextWord() {
1754 uint32_t dummy;
1755 if (!GcovBuffer.readInt(Val&: dummy))
1756 return sampleprof_error::truncated;
1757 return sampleprof_error::success;
1758}
1759
1760template <typename T> ErrorOr<T> SampleProfileReaderGCC::readNumber() {
1761 if (sizeof(T) <= sizeof(uint32_t)) {
1762 uint32_t Val;
1763 if (GcovBuffer.readInt(Val) && Val <= std::numeric_limits<T>::max())
1764 return static_cast<T>(Val);
1765 } else if (sizeof(T) <= sizeof(uint64_t)) {
1766 uint64_t Val;
1767 if (GcovBuffer.readInt64(Val) && Val <= std::numeric_limits<T>::max())
1768 return static_cast<T>(Val);
1769 }
1770
1771 std::error_code EC = sampleprof_error::malformed;
1772 reportError(LineNumber: 0, Msg: EC.message());
1773 return EC;
1774}
1775
1776ErrorOr<StringRef> SampleProfileReaderGCC::readString() {
1777 StringRef Str;
1778 if (!GcovBuffer.readString(str&: Str))
1779 return sampleprof_error::truncated;
1780 return Str;
1781}
1782
1783std::error_code SampleProfileReaderGCC::readHeader() {
1784 // Read the magic identifier.
1785 if (!GcovBuffer.readGCDAFormat())
1786 return sampleprof_error::unrecognized_format;
1787
1788 // Read the version number. Note - the GCC reader does not validate this
1789 // version, but the profile creator generates v704.
1790 GCOV::GCOVVersion version;
1791 if (!GcovBuffer.readGCOVVersion(version))
1792 return sampleprof_error::unrecognized_format;
1793
1794 if (version != GCOV::V407)
1795 return sampleprof_error::unsupported_version;
1796
1797 // Skip the empty integer.
1798 if (std::error_code EC = skipNextWord())
1799 return EC;
1800
1801 return sampleprof_error::success;
1802}
1803
1804std::error_code SampleProfileReaderGCC::readSectionTag(uint32_t Expected) {
1805 uint32_t Tag;
1806 if (!GcovBuffer.readInt(Val&: Tag))
1807 return sampleprof_error::truncated;
1808
1809 if (Tag != Expected)
1810 return sampleprof_error::malformed;
1811
1812 if (std::error_code EC = skipNextWord())
1813 return EC;
1814
1815 return sampleprof_error::success;
1816}
1817
1818std::error_code SampleProfileReaderGCC::readNameTable() {
1819 if (std::error_code EC = readSectionTag(Expected: GCOVTagAFDOFileNames))
1820 return EC;
1821
1822 uint32_t Size;
1823 if (!GcovBuffer.readInt(Val&: Size))
1824 return sampleprof_error::truncated;
1825
1826 for (uint32_t I = 0; I < Size; ++I) {
1827 StringRef Str;
1828 if (!GcovBuffer.readString(str&: Str))
1829 return sampleprof_error::truncated;
1830 Names.push_back(x: std::string(Str));
1831 }
1832
1833 return sampleprof_error::success;
1834}
1835
1836std::error_code SampleProfileReaderGCC::readFunctionProfiles() {
1837 if (std::error_code EC = readSectionTag(Expected: GCOVTagAFDOFunction))
1838 return EC;
1839
1840 uint32_t NumFunctions;
1841 if (!GcovBuffer.readInt(Val&: NumFunctions))
1842 return sampleprof_error::truncated;
1843
1844 InlineCallStack Stack;
1845 for (uint32_t I = 0; I < NumFunctions; ++I)
1846 if (std::error_code EC = readOneFunctionProfile(InlineStack: Stack, Update: true, Offset: 0))
1847 return EC;
1848
1849 computeSummary();
1850 return sampleprof_error::success;
1851}
1852
1853std::error_code SampleProfileReaderGCC::readOneFunctionProfile(
1854 const InlineCallStack &InlineStack, bool Update, uint32_t Offset) {
1855 uint64_t HeadCount = 0;
1856 if (InlineStack.size() == 0)
1857 if (!GcovBuffer.readInt64(Val&: HeadCount))
1858 return sampleprof_error::truncated;
1859
1860 uint32_t NameIdx;
1861 if (!GcovBuffer.readInt(Val&: NameIdx))
1862 return sampleprof_error::truncated;
1863
1864 StringRef Name(Names[NameIdx]);
1865
1866 uint32_t NumPosCounts;
1867 if (!GcovBuffer.readInt(Val&: NumPosCounts))
1868 return sampleprof_error::truncated;
1869
1870 uint32_t NumCallsites;
1871 if (!GcovBuffer.readInt(Val&: NumCallsites))
1872 return sampleprof_error::truncated;
1873
1874 FunctionSamples *FProfile = nullptr;
1875 if (InlineStack.size() == 0) {
1876 // If this is a top function that we have already processed, do not
1877 // update its profile again. This happens in the presence of
1878 // function aliases. Since these aliases share the same function
1879 // body, there will be identical replicated profiles for the
1880 // original function. In this case, we simply not bother updating
1881 // the profile of the original function.
1882 FProfile = &Profiles[FunctionId(Name)];
1883 FProfile->addHeadSamples(Num: HeadCount);
1884 if (FProfile->getTotalSamples() > 0)
1885 Update = false;
1886 } else {
1887 // Otherwise, we are reading an inlined instance. The top of the
1888 // inline stack contains the profile of the caller. Insert this
1889 // callee in the caller's CallsiteMap.
1890 FunctionSamples *CallerProfile = InlineStack.front();
1891 uint32_t LineOffset = Offset >> 16;
1892 uint32_t Discriminator = Offset & 0xffff;
1893 FProfile = &CallerProfile->functionSamplesAt(
1894 Loc: LineLocation(LineOffset, Discriminator))[FunctionId(Name)];
1895 }
1896 FProfile->setFunction(FunctionId(Name));
1897
1898 for (uint32_t I = 0; I < NumPosCounts; ++I) {
1899 uint32_t Offset;
1900 if (!GcovBuffer.readInt(Val&: Offset))
1901 return sampleprof_error::truncated;
1902
1903 uint32_t NumTargets;
1904 if (!GcovBuffer.readInt(Val&: NumTargets))
1905 return sampleprof_error::truncated;
1906
1907 uint64_t Count;
1908 if (!GcovBuffer.readInt64(Val&: Count))
1909 return sampleprof_error::truncated;
1910
1911 // The line location is encoded in the offset as:
1912 // high 16 bits: line offset to the start of the function.
1913 // low 16 bits: discriminator.
1914 uint32_t LineOffset = Offset >> 16;
1915 uint32_t Discriminator = Offset & 0xffff;
1916
1917 InlineCallStack NewStack;
1918 NewStack.push_back(Elt: FProfile);
1919 llvm::append_range(C&: NewStack, R: InlineStack);
1920 if (Update) {
1921 // Walk up the inline stack, adding the samples on this line to
1922 // the total sample count of the callers in the chain.
1923 for (auto *CallerProfile : NewStack)
1924 CallerProfile->addTotalSamples(Num: Count);
1925
1926 // Update the body samples for the current profile.
1927 FProfile->addBodySamples(LineOffset, Discriminator, Num: Count);
1928 }
1929
1930 // Process the list of functions called at an indirect call site.
1931 // These are all the targets that a function pointer (or virtual
1932 // function) resolved at runtime.
1933 for (uint32_t J = 0; J < NumTargets; J++) {
1934 uint32_t HistVal;
1935 if (!GcovBuffer.readInt(Val&: HistVal))
1936 return sampleprof_error::truncated;
1937
1938 if (HistVal != HIST_TYPE_INDIR_CALL_TOPN)
1939 return sampleprof_error::malformed;
1940
1941 uint64_t TargetIdx;
1942 if (!GcovBuffer.readInt64(Val&: TargetIdx))
1943 return sampleprof_error::truncated;
1944 StringRef TargetName(Names[TargetIdx]);
1945
1946 uint64_t TargetCount;
1947 if (!GcovBuffer.readInt64(Val&: TargetCount))
1948 return sampleprof_error::truncated;
1949
1950 if (Update)
1951 FProfile->addCalledTargetSamples(LineOffset, Discriminator,
1952 Func: FunctionId(TargetName), Num: TargetCount);
1953 }
1954 }
1955
1956 // Process all the inlined callers into the current function. These
1957 // are all the callsites that were inlined into this function.
1958 for (uint32_t I = 0; I < NumCallsites; I++) {
1959 // The offset is encoded as:
1960 // high 16 bits: line offset to the start of the function.
1961 // low 16 bits: discriminator.
1962 uint32_t Offset;
1963 if (!GcovBuffer.readInt(Val&: Offset))
1964 return sampleprof_error::truncated;
1965 InlineCallStack NewStack;
1966 NewStack.push_back(Elt: FProfile);
1967 llvm::append_range(C&: NewStack, R: InlineStack);
1968 if (std::error_code EC = readOneFunctionProfile(InlineStack: NewStack, Update, Offset))
1969 return EC;
1970 }
1971
1972 return sampleprof_error::success;
1973}
1974
1975/// Read a GCC AutoFDO profile.
1976///
1977/// This format is generated by the Linux Perf conversion tool at
1978/// https://github.com/google/autofdo.
1979std::error_code SampleProfileReaderGCC::readImpl() {
1980 assert(!ProfileIsFSDisciminator && "Gcc profiles not support FSDisciminator");
1981 // Read the string table.
1982 if (std::error_code EC = readNameTable())
1983 return EC;
1984
1985 // Read the source profile.
1986 if (std::error_code EC = readFunctionProfiles())
1987 return EC;
1988
1989 return sampleprof_error::success;
1990}
1991
1992bool SampleProfileReaderGCC::hasFormat(const MemoryBuffer &Buffer) {
1993 StringRef Magic(Buffer.getBufferStart());
1994 return Magic == "adcg*704";
1995}
1996
1997void SampleProfileReaderItaniumRemapper::applyRemapping(LLVMContext &Ctx) {
1998 // If the reader uses MD5 to represent string, we can't remap it because
1999 // we don't know what the original function names were.
2000 if (Reader.useMD5()) {
2001 Ctx.diagnose(DI: DiagnosticInfoSampleProfile(
2002 Reader.getBuffer()->getBufferIdentifier(),
2003 "Profile data remapping cannot be applied to profile data "
2004 "using MD5 names (original mangled names are not available).",
2005 DS_Warning));
2006 return;
2007 }
2008
2009 // CSSPGO-TODO: Remapper is not yet supported.
2010 // We will need to remap the entire context string.
2011 assert(Remappings && "should be initialized while creating remapper");
2012 for (auto &Sample : Reader.getProfiles()) {
2013 DenseSet<FunctionId> NamesInSample;
2014 Sample.second.findAllNames(NameSet&: NamesInSample);
2015 for (auto &Name : NamesInSample) {
2016 StringRef NameStr = Name.stringRef();
2017 if (auto Key = Remappings->insert(FunctionName: NameStr))
2018 NameMap.insert(KV: {Key, NameStr});
2019 }
2020 }
2021
2022 RemappingApplied = true;
2023}
2024
2025std::optional<StringRef>
2026SampleProfileReaderItaniumRemapper::lookUpNameInProfile(StringRef Fname) {
2027 if (auto Key = Remappings->lookup(FunctionName: Fname)) {
2028 StringRef Result = NameMap.lookup(Val: Key);
2029 if (!Result.empty())
2030 return Result;
2031 }
2032 return std::nullopt;
2033}
2034
2035/// Prepare a memory buffer for the contents of \p Filename.
2036///
2037/// \returns an error code indicating the status of the buffer.
2038static ErrorOr<std::unique_ptr<MemoryBuffer>>
2039setupMemoryBuffer(const Twine &Filename, vfs::FileSystem &FS) {
2040 auto BufferOrErr = Filename.str() == "-" ? MemoryBuffer::getSTDIN()
2041 : FS.getBufferForFile(Name: Filename);
2042 if (std::error_code EC = BufferOrErr.getError())
2043 return EC;
2044 auto Buffer = std::move(BufferOrErr.get());
2045
2046 return std::move(Buffer);
2047}
2048
2049/// Create a sample profile reader based on the format of the input file.
2050///
2051/// \param Filename The file to open.
2052///
2053/// \param C The LLVM context to use to emit diagnostics.
2054///
2055/// \param P The FSDiscriminatorPass.
2056///
2057/// \param RemapFilename The file used for profile remapping.
2058///
2059/// \returns an error code indicating the status of the created reader.
2060ErrorOr<std::unique_ptr<SampleProfileReader>>
2061SampleProfileReader::create(StringRef Filename, LLVMContext &C,
2062 vfs::FileSystem &FS, FSDiscriminatorPass P,
2063 StringRef RemapFilename) {
2064 auto BufferOrError = setupMemoryBuffer(Filename, FS);
2065 if (std::error_code EC = BufferOrError.getError())
2066 return EC;
2067 return create(B&: BufferOrError.get(), C, FS, P, RemapFilename);
2068}
2069
2070/// Create a sample profile remapper from the given input, to remap the
2071/// function names in the given profile data.
2072///
2073/// \param Filename The file to open.
2074///
2075/// \param Reader The profile reader the remapper is going to be applied to.
2076///
2077/// \param C The LLVM context to use to emit diagnostics.
2078///
2079/// \returns an error code indicating the status of the created reader.
2080ErrorOr<std::unique_ptr<SampleProfileReaderItaniumRemapper>>
2081SampleProfileReaderItaniumRemapper::create(StringRef Filename,
2082 vfs::FileSystem &FS,
2083 SampleProfileReader &Reader,
2084 LLVMContext &C) {
2085 auto BufferOrError = setupMemoryBuffer(Filename, FS);
2086 if (std::error_code EC = BufferOrError.getError())
2087 return EC;
2088 return create(B&: BufferOrError.get(), Reader, C);
2089}
2090
2091/// Create a sample profile remapper from the given input, to remap the
2092/// function names in the given profile data.
2093///
2094/// \param B The memory buffer to create the reader from (assumes ownership).
2095///
2096/// \param C The LLVM context to use to emit diagnostics.
2097///
2098/// \param Reader The profile reader the remapper is going to be applied to.
2099///
2100/// \returns an error code indicating the status of the created reader.
2101ErrorOr<std::unique_ptr<SampleProfileReaderItaniumRemapper>>
2102SampleProfileReaderItaniumRemapper::create(std::unique_ptr<MemoryBuffer> &B,
2103 SampleProfileReader &Reader,
2104 LLVMContext &C) {
2105 auto Remappings = std::make_unique<SymbolRemappingReader>();
2106 if (Error E = Remappings->read(B&: *B)) {
2107 handleAllErrors(
2108 E: std::move(E), Handlers: [&](const SymbolRemappingParseError &ParseError) {
2109 C.diagnose(DI: DiagnosticInfoSampleProfile(B->getBufferIdentifier(),
2110 ParseError.getLineNum(),
2111 ParseError.getMessage()));
2112 });
2113 return sampleprof_error::malformed;
2114 }
2115
2116 return std::make_unique<SampleProfileReaderItaniumRemapper>(
2117 args: std::move(B), args: std::move(Remappings), args&: Reader);
2118}
2119
2120/// Create a sample profile reader based on the format of the input data.
2121///
2122/// \param B The memory buffer to create the reader from (assumes ownership).
2123///
2124/// \param C The LLVM context to use to emit diagnostics.
2125///
2126/// \param P The FSDiscriminatorPass.
2127///
2128/// \param RemapFilename The file used for profile remapping.
2129///
2130/// \returns an error code indicating the status of the created reader.
2131ErrorOr<std::unique_ptr<SampleProfileReader>>
2132SampleProfileReader::create(std::unique_ptr<MemoryBuffer> &B, LLVMContext &C,
2133 vfs::FileSystem &FS, FSDiscriminatorPass P,
2134 StringRef RemapFilename) {
2135 std::unique_ptr<SampleProfileReader> Reader;
2136 if (SampleProfileReaderRawBinary::hasFormat(Buffer: *B))
2137 Reader.reset(p: new SampleProfileReaderRawBinary(std::move(B), C));
2138 else if (SampleProfileReaderExtBinary::hasFormat(Buffer: *B))
2139 Reader.reset(p: new SampleProfileReaderExtBinary(std::move(B), C));
2140 else if (SampleProfileReaderGCC::hasFormat(Buffer: *B))
2141 Reader.reset(p: new SampleProfileReaderGCC(std::move(B), C));
2142 else if (SampleProfileReaderText::hasFormat(Buffer: *B))
2143 Reader.reset(p: new SampleProfileReaderText(std::move(B), C));
2144 else
2145 return sampleprof_error::unrecognized_format;
2146
2147 if (!RemapFilename.empty()) {
2148 auto ReaderOrErr = SampleProfileReaderItaniumRemapper::create(
2149 Filename: RemapFilename, FS, Reader&: *Reader, C);
2150 if (std::error_code EC = ReaderOrErr.getError()) {
2151 std::string Msg = "Could not create remapper: " + EC.message();
2152 C.diagnose(DI: DiagnosticInfoSampleProfile(RemapFilename, Msg));
2153 return EC;
2154 }
2155 Reader->Remapper = std::move(ReaderOrErr.get());
2156 }
2157
2158 if (std::error_code EC = Reader->readHeader()) {
2159 return EC;
2160 }
2161
2162 Reader->setDiscriminatorMaskedBitFrom(P);
2163
2164 return std::move(Reader);
2165}
2166
2167// For text and GCC file formats, we compute the summary after reading the
2168// profile. Binary format has the profile summary in its header.
2169void SampleProfileReader::computeSummary() {
2170 SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs);
2171 Summary = Builder.computeSummaryForProfiles(Profiles);
2172}
2173