1//===- SampleProfReader.cpp - Read LLVM sample profile data ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the class that reads LLVM sample profiles. It
10// supports three file formats: text, binary and gcov.
11//
12// The textual representation is useful for debugging and testing purposes. The
13// binary representation is more compact, resulting in smaller file sizes.
14//
15// The gcov encoding is the one generated by GCC's AutoFDO profile creation
16// tool (https://github.com/google/autofdo)
17//
18// All three encodings can be used interchangeably as an input sample profile.
19//
20//===----------------------------------------------------------------------===//
21
22#include "llvm/ProfileData/SampleProfReader.h"
23#include "llvm/ADT/DenseMap.h"
24#include "llvm/ADT/STLExtras.h"
25#include "llvm/ADT/StringRef.h"
26#include "llvm/IR/Module.h"
27#include "llvm/IR/ProfileSummary.h"
28#include "llvm/ProfileData/ProfileCommon.h"
29#include "llvm/ProfileData/SampleProf.h"
30#include "llvm/Support/CommandLine.h"
31#include "llvm/Support/Compression.h"
32#include "llvm/Support/ErrorOr.h"
33#include "llvm/Support/JSON.h"
34#include "llvm/Support/LEB128.h"
35#include "llvm/Support/LineIterator.h"
36#include "llvm/Support/MD5.h"
37#include "llvm/Support/MemoryBuffer.h"
38#include "llvm/Support/VirtualFileSystem.h"
39#include "llvm/Support/raw_ostream.h"
40#include <algorithm>
41#include <cstddef>
42#include <cstdint>
43#include <limits>
44#include <memory>
45#include <system_error>
46#include <vector>
47
48using namespace llvm;
49using namespace sampleprof;
50
51#define DEBUG_TYPE "samplepgo-reader"
52
53// This internal option specifies if the profile uses FS discriminators.
54// It only applies to text, and binary format profiles.
55// For ext-binary format profiles, the flag is set in the summary.
56static cl::opt<bool> ProfileIsFSDisciminator(
57 "profile-isfs", cl::Hidden, cl::init(Val: false),
58 cl::desc("Profile uses flow sensitive discriminators"));
59
60static cl::opt<bool>
61 LazyLoadNameTable("sample-profile-lazy-load-name-table", cl::init(Val: true),
62 cl::Hidden,
63 cl::desc("Lazy load the name table from the profile."));
64
65/// Dump the function profile for \p FName.
66///
67/// \param FContext Name + context of the function to print.
68/// \param OS Stream to emit the output to.
69void SampleProfileReader::dumpFunctionProfile(const FunctionSamples &FS,
70 raw_ostream &OS) {
71 OS << "Function: " << FS.getContext().toString() << ": " << FS;
72}
73
74/// Dump all the function profiles found on stream \p OS.
75void SampleProfileReader::dump(raw_ostream &OS) {
76 std::vector<NameFunctionSamples> V;
77 sortFuncProfiles(ProfileMap: Profiles, SortedProfiles&: V);
78 for (const auto &I : V)
79 dumpFunctionProfile(FS: *I.second, OS);
80}
81
82static void dumpFunctionProfileJson(const FunctionSamples &S,
83 json::OStream &JOS, bool TopLevel = false) {
84 auto DumpBody = [&](const BodySampleMap &BodySamples) {
85 for (const auto &I : BodySamples) {
86 const LineLocation &Loc = I.first;
87 const SampleRecord &Sample = I.second;
88 JOS.object(Contents: [&] {
89 JOS.attribute(Key: "line", Contents: Loc.LineOffset);
90 if (Loc.Discriminator)
91 JOS.attribute(Key: "discriminator", Contents: Loc.Discriminator);
92 JOS.attribute(Key: "samples", Contents: Sample.getSamples());
93
94 auto CallTargets = Sample.getSortedCallTargets();
95 if (!CallTargets.empty()) {
96 JOS.attributeArray(Key: "calls", Contents: [&] {
97 for (const auto &J : CallTargets) {
98 JOS.object(Contents: [&] {
99 JOS.attribute(Key: "function", Contents: J.first.str());
100 JOS.attribute(Key: "samples", Contents: J.second);
101 });
102 }
103 });
104 }
105 });
106 }
107 };
108
109 auto DumpCallsiteSamples = [&](const CallsiteSampleMap &CallsiteSamples) {
110 for (const auto &I : CallsiteSamples)
111 for (const auto &FS : I.second) {
112 const LineLocation &Loc = I.first;
113 const FunctionSamples &CalleeSamples = FS.second;
114 JOS.object(Contents: [&] {
115 JOS.attribute(Key: "line", Contents: Loc.LineOffset);
116 if (Loc.Discriminator)
117 JOS.attribute(Key: "discriminator", Contents: Loc.Discriminator);
118 JOS.attributeArray(
119 Key: "samples", Contents: [&] { dumpFunctionProfileJson(S: CalleeSamples, JOS); });
120 });
121 }
122 };
123
124 JOS.object(Contents: [&] {
125 JOS.attribute(Key: "name", Contents: S.getFunction().str());
126 JOS.attribute(Key: "total", Contents: S.getTotalSamples());
127 if (TopLevel)
128 JOS.attribute(Key: "head", Contents: S.getHeadSamples());
129
130 const auto &BodySamples = S.getBodySamples();
131 if (!BodySamples.empty())
132 JOS.attributeArray(Key: "body", Contents: [&] { DumpBody(BodySamples); });
133
134 const auto &CallsiteSamples = S.getCallsiteSamples();
135 if (!CallsiteSamples.empty())
136 JOS.attributeArray(Key: "callsites",
137 Contents: [&] { DumpCallsiteSamples(CallsiteSamples); });
138 });
139}
140
141/// Dump all the function profiles found on stream \p OS in the JSON format.
142void SampleProfileReader::dumpJson(raw_ostream &OS) {
143 std::vector<NameFunctionSamples> V;
144 sortFuncProfiles(ProfileMap: Profiles, SortedProfiles&: V);
145 json::OStream JOS(OS, 2);
146 JOS.arrayBegin();
147 for (const auto &F : V)
148 dumpFunctionProfileJson(S: *F.second, JOS, TopLevel: true);
149 JOS.arrayEnd();
150
151 // Emit a newline character at the end as json::OStream doesn't emit one.
152 OS << "\n";
153}
154
155/// Parse \p Input as function head.
156///
157/// Parse one line of \p Input, and update function name in \p FName,
158/// function's total sample count in \p NumSamples, function's entry
159/// count in \p NumHeadSamples.
160///
161/// \returns true if parsing is successful.
162static bool ParseHead(const StringRef &Input, StringRef &FName,
163 uint64_t &NumSamples, uint64_t &NumHeadSamples) {
164 if (Input[0] == ' ')
165 return false;
166 size_t n2 = Input.rfind(C: ':');
167 size_t n1 = Input.rfind(C: ':', From: n2 - 1);
168 FName = Input.substr(Start: 0, N: n1);
169 if (Input.substr(Start: n1 + 1, N: n2 - n1 - 1).getAsInteger(Radix: 10, Result&: NumSamples))
170 return false;
171 if (Input.substr(Start: n2 + 1).getAsInteger(Radix: 10, Result&: NumHeadSamples))
172 return false;
173 return true;
174}
175
176/// Returns true if line offset \p L is legal (only has 16 bits).
177static bool isOffsetLegal(unsigned L) { return (L & 0xffff) == L; }
178
179/// Parse \p Input that contains metadata.
180/// Possible metadata:
181/// - CFG Checksum information:
182/// !CFGChecksum: 12345
183/// - CFG Checksum information:
184/// !Attributes: 1
185/// Stores the FunctionHash (a.k.a. CFG Checksum) into \p FunctionHash.
186static bool parseMetadata(const StringRef &Input, uint64_t &FunctionHash,
187 uint32_t &Attributes) {
188 if (Input.starts_with(Prefix: "!CFGChecksum:")) {
189 StringRef CFGInfo = Input.substr(Start: strlen(s: "!CFGChecksum:")).trim();
190 return !CFGInfo.getAsInteger(Radix: 10, Result&: FunctionHash);
191 }
192
193 if (Input.starts_with(Prefix: "!Attributes:")) {
194 StringRef Attrib = Input.substr(Start: strlen(s: "!Attributes:")).trim();
195 return !Attrib.getAsInteger(Radix: 10, Result&: Attributes);
196 }
197
198 return false;
199}
200
201enum class LineType {
202 CallSiteProfile,
203 BodyProfile,
204 Metadata,
205 VirtualCallTypeProfile,
206};
207
208// Parse `Input` as a white-space separated list of `vtable:count` pairs. An
209// example input line is `_ZTVbar:1471 _ZTVfoo:630`.
210static bool parseTypeCountMap(StringRef Input,
211 DenseMap<StringRef, uint64_t> &TypeCountMap) {
212 for (size_t Index = Input.find_first_not_of(C: ' '); Index != StringRef::npos;) {
213 size_t ColonIndex = Input.find(C: ':', From: Index);
214 if (ColonIndex == StringRef::npos)
215 return false; // No colon found, invalid format.
216 StringRef TypeName = Input.substr(Start: Index, N: ColonIndex - Index);
217 // CountIndex is the start index of count.
218 size_t CountStartIndex = ColonIndex + 1;
219 // NextIndex is the start index after the 'target:count' pair.
220 size_t NextIndex = Input.find_first_of(C: ' ', From: CountStartIndex);
221 uint64_t Count;
222 if (Input.substr(Start: CountStartIndex, N: NextIndex - CountStartIndex)
223 .getAsInteger(Radix: 10, Result&: Count))
224 return false; // Invalid count.
225 // Error on duplicated type names in one line of input.
226 auto [Iter, Inserted] = TypeCountMap.insert(KV: {TypeName, Count});
227 if (!Inserted)
228 return false;
229 Index = (NextIndex == StringRef::npos)
230 ? StringRef::npos
231 : Input.find_first_not_of(C: ' ', From: NextIndex);
232 }
233 return true;
234}
235
236/// Parse \p Input as line sample.
237///
238/// \param Input input line.
239/// \param LineTy Type of this line.
240/// \param Depth the depth of the inline stack.
241/// \param NumSamples total samples of the line/inlined callsite.
242/// \param LineOffset line offset to the start of the function.
243/// \param Discriminator discriminator of the line.
244/// \param TargetCountMap map from indirect call target to count.
245/// \param FunctionHash the function's CFG hash, used by pseudo probe.
246///
247/// returns true if parsing is successful.
248static bool ParseLine(const StringRef &Input, LineType &LineTy, uint32_t &Depth,
249 uint64_t &NumSamples, uint32_t &LineOffset,
250 uint32_t &Discriminator, StringRef &CalleeName,
251 DenseMap<StringRef, uint64_t> &TargetCountMap,
252 DenseMap<StringRef, uint64_t> &TypeCountMap,
253 uint64_t &FunctionHash, uint32_t &Attributes,
254 bool &IsFlat) {
255 for (Depth = 0; Input[Depth] == ' '; Depth++)
256 ;
257 if (Depth == 0)
258 return false;
259
260 if (Input[Depth] == '!') {
261 LineTy = LineType::Metadata;
262 // This metadata is only for manual inspection only. We already created a
263 // FunctionSamples and put it in the profile map, so there is no point
264 // to skip profiles even they have no use for ThinLTO.
265 if (Input == StringRef(" !Flat")) {
266 IsFlat = true;
267 return true;
268 }
269 return parseMetadata(Input: Input.substr(Start: Depth), FunctionHash, Attributes);
270 }
271
272 size_t n1 = Input.find(C: ':');
273 StringRef Loc = Input.substr(Start: Depth, N: n1 - Depth);
274 size_t n2 = Loc.find(C: '.');
275 if (n2 == StringRef::npos) {
276 if (Loc.getAsInteger(Radix: 10, Result&: LineOffset) || !isOffsetLegal(L: LineOffset))
277 return false;
278 Discriminator = 0;
279 } else {
280 if (Loc.substr(Start: 0, N: n2).getAsInteger(Radix: 10, Result&: LineOffset))
281 return false;
282 if (Loc.substr(Start: n2 + 1).getAsInteger(Radix: 10, Result&: Discriminator))
283 return false;
284 }
285
286 StringRef Rest = Input.substr(Start: n1 + 2);
287 if (isDigit(C: Rest[0])) {
288 LineTy = LineType::BodyProfile;
289 size_t n3 = Rest.find(C: ' ');
290 if (n3 == StringRef::npos) {
291 if (Rest.getAsInteger(Radix: 10, Result&: NumSamples))
292 return false;
293 } else {
294 if (Rest.substr(Start: 0, N: n3).getAsInteger(Radix: 10, Result&: NumSamples))
295 return false;
296 }
297 // Find call targets and their sample counts.
298 // Note: In some cases, there are symbols in the profile which are not
299 // mangled. To accommodate such cases, use colon + integer pairs as the
300 // anchor points.
301 // An example:
302 // _M_construct<char *>:1000 string_view<std::allocator<char> >:437
303 // ":1000" and ":437" are used as anchor points so the string above will
304 // be interpreted as
305 // target: _M_construct<char *>
306 // count: 1000
307 // target: string_view<std::allocator<char> >
308 // count: 437
309 while (n3 != StringRef::npos) {
310 n3 += Rest.substr(Start: n3).find_first_not_of(C: ' ');
311 Rest = Rest.substr(Start: n3);
312 n3 = Rest.find_first_of(C: ':');
313 if (n3 == StringRef::npos || n3 == 0)
314 return false;
315
316 StringRef Target;
317 uint64_t count, n4;
318 while (true) {
319 // Get the segment after the current colon.
320 StringRef AfterColon = Rest.substr(Start: n3 + 1);
321 // Get the target symbol before the current colon.
322 Target = Rest.substr(Start: 0, N: n3);
323 // Check if the word after the current colon is an integer.
324 n4 = AfterColon.find_first_of(C: ' ');
325 n4 = (n4 != StringRef::npos) ? n3 + n4 + 1 : Rest.size();
326 StringRef WordAfterColon = Rest.substr(Start: n3 + 1, N: n4 - n3 - 1);
327 if (!WordAfterColon.getAsInteger(Radix: 10, Result&: count))
328 break;
329
330 // Try to find the next colon.
331 uint64_t n5 = AfterColon.find_first_of(C: ':');
332 if (n5 == StringRef::npos)
333 return false;
334 n3 += n5 + 1;
335 }
336
337 // An anchor point is found. Save the {target, count} pair
338 TargetCountMap[Target] = count;
339 if (n4 == Rest.size())
340 break;
341 // Change n3 to the next blank space after colon + integer pair.
342 n3 = n4;
343 }
344 } else if (Rest.starts_with(Prefix: kVTableProfPrefix)) {
345 LineTy = LineType::VirtualCallTypeProfile;
346 return parseTypeCountMap(Input: Rest.substr(Start: strlen(s: kVTableProfPrefix)),
347 TypeCountMap);
348 } else {
349 LineTy = LineType::CallSiteProfile;
350 size_t n3 = Rest.find_last_of(C: ':');
351 CalleeName = Rest.substr(Start: 0, N: n3);
352 if (Rest.substr(Start: n3 + 1).getAsInteger(Radix: 10, Result&: NumSamples))
353 return false;
354 }
355 return true;
356}
357
358/// Load samples from a text file.
359///
360/// See the documentation at the top of the file for an explanation of
361/// the expected format.
362///
363/// \returns true if the file was loaded successfully, false otherwise.
364std::error_code SampleProfileReaderText::readImpl() {
365 line_iterator LineIt(*Buffer, /*SkipBlanks=*/true, '#');
366 sampleprof_error Result = sampleprof_error::success;
367
368 InlineCallStack InlineStack;
369 uint32_t TopLevelProbeProfileCount = 0;
370
371 // DepthMetadata tracks whether we have processed metadata for the current
372 // top-level or nested function profile.
373 uint32_t DepthMetadata = 0;
374
375 std::vector<SampleContext *> FlatSamples;
376
377 ProfileIsFS = ProfileIsFSDisciminator;
378 FunctionSamples::ProfileIsFS = ProfileIsFS;
379 for (; !LineIt.is_at_eof(); ++LineIt) {
380 size_t pos = LineIt->find_first_not_of(C: ' ');
381 if (pos == LineIt->npos || (*LineIt)[pos] == '#')
382 continue;
383 // Read the header of each function.
384 //
385 // Note that for function identifiers we are actually expecting
386 // mangled names, but we may not always get them. This happens when
387 // the compiler decides not to emit the function (e.g., it was inlined
388 // and removed). In this case, the binary will not have the linkage
389 // name for the function, so the profiler will emit the function's
390 // unmangled name, which may contain characters like ':' and '>' in its
391 // name (member functions, templates, etc).
392 //
393 // The only requirement we place on the identifier, then, is that it
394 // should not begin with a number.
395 if ((*LineIt)[0] != ' ') {
396 uint64_t NumSamples, NumHeadSamples;
397 StringRef FName;
398 if (!ParseHead(Input: *LineIt, FName, NumSamples, NumHeadSamples)) {
399 reportError(LineNumber: LineIt.line_number(),
400 Msg: "Expected 'mangled_name:NUM:NUM', found " + *LineIt);
401 return sampleprof_error::malformed;
402 }
403 DepthMetadata = 0;
404 SampleContext FContext(FName, CSNameTable);
405 if (FContext.hasContext())
406 ++CSProfileCount;
407 FunctionSamples &FProfile = Profiles.create(Ctx: FContext);
408 mergeSampleProfErrors(Accumulator&: Result, Result: FProfile.addTotalSamples(Num: NumSamples));
409 mergeSampleProfErrors(Accumulator&: Result, Result: FProfile.addHeadSamples(Num: NumHeadSamples));
410 InlineStack.clear();
411 InlineStack.push_back(Elt: &FProfile);
412 } else {
413 uint64_t NumSamples;
414 StringRef FName;
415 DenseMap<StringRef, uint64_t> TargetCountMap;
416 DenseMap<StringRef, uint64_t> TypeCountMap;
417 uint32_t Depth, LineOffset, Discriminator;
418 LineType LineTy = LineType::BodyProfile;
419 uint64_t FunctionHash = 0;
420 uint32_t Attributes = 0;
421 bool IsFlat = false;
422 // TODO: Update ParseLine to return an error code instead of a bool and
423 // report it.
424 if (!ParseLine(Input: *LineIt, LineTy, Depth, NumSamples, LineOffset,
425 Discriminator, CalleeName&: FName, TargetCountMap, TypeCountMap,
426 FunctionHash, Attributes, IsFlat)) {
427 switch (LineTy) {
428 case LineType::Metadata:
429 reportError(LineNumber: LineIt.line_number(),
430 Msg: "Cannot parse metadata: " + *LineIt);
431 break;
432 case LineType::VirtualCallTypeProfile:
433 reportError(LineNumber: LineIt.line_number(),
434 Msg: "Expected 'vtables [mangled_vtable:NUM]+', found " +
435 *LineIt);
436 break;
437 default:
438 reportError(LineNumber: LineIt.line_number(),
439 Msg: "Expected 'NUM[.NUM]: NUM[ mangled_name:NUM]*', found " +
440 *LineIt);
441 }
442 return sampleprof_error::malformed;
443 }
444 if (LineTy != LineType::Metadata && Depth == DepthMetadata) {
445 // Metadata must be put at the end of a function profile.
446 reportError(LineNumber: LineIt.line_number(),
447 Msg: "Found non-metadata after metadata: " + *LineIt);
448 return sampleprof_error::malformed;
449 }
450
451 // Here we handle FS discriminators.
452 Discriminator &= getDiscriminatorMask();
453
454 while (InlineStack.size() > Depth) {
455 InlineStack.pop_back();
456 }
457 switch (LineTy) {
458 case LineType::CallSiteProfile: {
459 FunctionSamples &FSamples = InlineStack.back()->functionSamplesAt(
460 Loc: LineLocation(LineOffset, Discriminator))[FunctionId(FName)];
461 FSamples.setFunction(FunctionId(FName));
462 mergeSampleProfErrors(Accumulator&: Result, Result: FSamples.addTotalSamples(Num: NumSamples));
463 InlineStack.push_back(Elt: &FSamples);
464 DepthMetadata = 0;
465 break;
466 }
467
468 case LineType::VirtualCallTypeProfile: {
469 mergeSampleProfErrors(
470 Accumulator&: Result, Result: InlineStack.back()->addCallsiteVTableTypeProfAt(
471 Loc: LineLocation(LineOffset, Discriminator), Other: TypeCountMap));
472 break;
473 }
474
475 case LineType::BodyProfile: {
476 FunctionSamples &FProfile = *InlineStack.back();
477 for (const auto &name_count : TargetCountMap) {
478 mergeSampleProfErrors(Accumulator&: Result, Result: FProfile.addCalledTargetSamples(
479 LineOffset, Discriminator,
480 Func: FunctionId(name_count.first),
481 Num: name_count.second));
482 }
483 mergeSampleProfErrors(
484 Accumulator&: Result,
485 Result: FProfile.addBodySamples(LineOffset, Discriminator, Num: NumSamples));
486 break;
487 }
488 case LineType::Metadata: {
489 FunctionSamples &FProfile = *InlineStack.back();
490 if (FunctionHash) {
491 FProfile.setFunctionHash(FunctionHash);
492 if (Depth == 1)
493 ++TopLevelProbeProfileCount;
494 }
495 FProfile.getContext().setAllAttributes(Attributes);
496 if (Attributes & (uint32_t)ContextShouldBeInlined)
497 ProfileIsPreInlined = true;
498 DepthMetadata = Depth;
499 if (IsFlat) {
500 if (Depth == 1)
501 FlatSamples.push_back(x: &FProfile.getContext());
502 else
503 Ctx.diagnose(DI: DiagnosticInfoSampleProfile(
504 Buffer->getBufferIdentifier(), LineIt.line_number(),
505 "!Flat may only be used at top level function.", DS_Warning));
506 }
507 break;
508 }
509 }
510 }
511 }
512
513 // Honor the option to skip flat functions. Since they are already added to
514 // the profile map, remove them all here.
515 if (SkipFlatProf)
516 for (SampleContext *FlatSample : FlatSamples)
517 Profiles.erase(Ctx: *FlatSample);
518
519 assert((CSProfileCount == 0 || CSProfileCount == Profiles.size()) &&
520 "Cannot have both context-sensitive and regular profile");
521 ProfileIsCS = (CSProfileCount > 0);
522 assert((TopLevelProbeProfileCount == 0 ||
523 TopLevelProbeProfileCount == Profiles.size()) &&
524 "Cannot have both probe-based profiles and regular profiles");
525 ProfileIsProbeBased = (TopLevelProbeProfileCount > 0);
526 FunctionSamples::ProfileIsProbeBased = ProfileIsProbeBased;
527 FunctionSamples::ProfileIsCS = ProfileIsCS;
528 FunctionSamples::ProfileIsPreInlined = ProfileIsPreInlined;
529
530 if (Result == sampleprof_error::success)
531 computeSummary();
532
533 return Result;
534}
535
536bool SampleProfileReaderText::hasFormat(const MemoryBuffer &Buffer) {
537 bool result = false;
538
539 // Check that the first non-comment line is a valid function header.
540 line_iterator LineIt(Buffer, /*SkipBlanks=*/true, '#');
541 if (!LineIt.is_at_eof()) {
542 if ((*LineIt)[0] != ' ') {
543 uint64_t NumSamples, NumHeadSamples;
544 StringRef FName;
545 result = ParseHead(Input: *LineIt, FName, NumSamples, NumHeadSamples);
546 }
547 }
548
549 return result;
550}
551
552template <typename T> ErrorOr<T> SampleProfileReaderBinary::readNumber() {
553 unsigned NumBytesRead = 0;
554 uint64_t Val = decodeULEB128(p: Data, n: &NumBytesRead);
555
556 if (Val > std::numeric_limits<T>::max()) {
557 std::error_code EC = sampleprof_error::malformed;
558 reportError(LineNumber: 0, Msg: EC.message());
559 return EC;
560 } else if (Data + NumBytesRead > End) {
561 std::error_code EC = sampleprof_error::truncated;
562 reportError(LineNumber: 0, Msg: EC.message());
563 return EC;
564 }
565
566 Data += NumBytesRead;
567 return static_cast<T>(Val);
568}
569
570ErrorOr<StringRef> SampleProfileReaderBinary::readString() {
571 StringRef Str(reinterpret_cast<const char *>(Data));
572 if (Data + Str.size() + 1 > End) {
573 std::error_code EC = sampleprof_error::truncated;
574 reportError(LineNumber: 0, Msg: EC.message());
575 return EC;
576 }
577
578 Data += Str.size() + 1;
579 return Str;
580}
581
582template <typename T>
583ErrorOr<T> SampleProfileReaderBinary::readUnencodedNumber() {
584 if (Data + sizeof(T) > End) {
585 std::error_code EC = sampleprof_error::truncated;
586 reportError(LineNumber: 0, Msg: EC.message());
587 return EC;
588 }
589
590 using namespace support;
591 T Val = endian::readNext<T, llvm::endianness::little>(Data);
592 return Val;
593}
594
595template <typename T>
596inline ErrorOr<size_t> SampleProfileReaderBinary::readStringIndex(T &Table) {
597 auto Idx = readNumber<size_t>();
598 if (std::error_code EC = Idx.getError())
599 return EC;
600 if (*Idx >= Table.size())
601 return sampleprof_error::truncated_name_table;
602 return *Idx;
603}
604
605ErrorOr<FunctionId>
606SampleProfileReaderBinary::readStringFromTable(size_t *RetIdx) {
607 if (!NameTable)
608 return sampleprof_error::truncated_name_table;
609 auto Idx = readStringIndex(Table&: *NameTable);
610 if (std::error_code EC = Idx.getError())
611 return EC;
612 if (RetIdx)
613 *RetIdx = *Idx;
614 return (*NameTable)[*Idx];
615}
616
617ErrorOr<SampleContextFrames>
618SampleProfileReaderBinary::readContextFromTable(size_t *RetIdx) {
619 auto ContextIdx = readNumber<size_t>();
620 if (std::error_code EC = ContextIdx.getError())
621 return EC;
622 if (*ContextIdx >= CSNameTable.size())
623 return sampleprof_error::truncated_name_table;
624 if (RetIdx)
625 *RetIdx = *ContextIdx;
626 return CSNameTable[*ContextIdx];
627}
628
629ErrorOr<std::pair<SampleContext, uint64_t>>
630SampleProfileReaderBinary::readSampleContextFromTable() {
631 SampleContext Context;
632 size_t Idx;
633 if (ProfileIsCS) {
634 auto FContext(readContextFromTable(RetIdx: &Idx));
635 if (std::error_code EC = FContext.getError())
636 return EC;
637 Context = SampleContext(*FContext);
638 } else {
639 auto FName(readStringFromTable(RetIdx: &Idx));
640 if (std::error_code EC = FName.getError())
641 return EC;
642 Context = SampleContext(*FName);
643 }
644 // Since MD5SampleContextStart may point to the profile's file data, need to
645 // make sure it is reading the same value on big endian CPU.
646 uint64_t Hash = support::endian::read64le(P: MD5SampleContextStart + Idx);
647 // Lazy computing of hash value, write back to the table to cache it. Only
648 // compute the context's hash value if it is being referenced for the first
649 // time.
650 if (Hash == 0) {
651 assert(MD5SampleContextStart == MD5SampleContextTable.data());
652 Hash = Context.getHashCode();
653 support::endian::write64le(P: &MD5SampleContextTable[Idx], V: Hash);
654 }
655 return std::make_pair(x&: Context, y&: Hash);
656}
657
658std::error_code
659SampleProfileReaderBinary::readVTableTypeCountMap(TypeCountMap &M) {
660 auto NumVTableTypes = readNumber<uint32_t>();
661 if (std::error_code EC = NumVTableTypes.getError())
662 return EC;
663 M.reserve(Cap: *NumVTableTypes);
664
665 for (uint32_t I = 0; I < *NumVTableTypes; ++I) {
666 auto VTableType(readStringFromTable());
667 if (std::error_code EC = VTableType.getError())
668 return EC;
669
670 auto VTableSamples = readNumber<uint64_t>();
671 if (std::error_code EC = VTableSamples.getError())
672 return EC;
673 // The source profile should not have duplicate vtable records at the same
674 // location. In case duplicate vtables are found, reader can emit a warning
675 // but continue processing the profile.
676 if (!M.insert(KV: std::make_pair(x&: *VTableType, y&: *VTableSamples)).second) {
677 Ctx.diagnose(DI: DiagnosticInfoSampleProfile(
678 Buffer->getBufferIdentifier(), 0,
679 "Duplicate vtable type " + VTableType->str() +
680 " at the same location. Additional counters will be ignored.",
681 DS_Warning));
682 continue;
683 }
684 }
685 return sampleprof_error::success;
686}
687
688std::error_code
689SampleProfileReaderBinary::readCallsiteVTableProf(FunctionSamples &FProfile) {
690 assert(ReadVTableProf &&
691 "Cannot read vtable profiles if ReadVTableProf is false");
692
693 // Read the vtable type profile for the callsite.
694 auto NumCallsites = readNumber<uint32_t>();
695 if (std::error_code EC = NumCallsites.getError())
696 return EC;
697 FProfile.reserveCallsiteTypeCounts(NumEntries: *NumCallsites);
698
699 for (uint32_t I = 0; I < *NumCallsites; ++I) {
700 auto LineOffset = readNumber<uint64_t>();
701 if (std::error_code EC = LineOffset.getError())
702 return EC;
703
704 if (!isOffsetLegal(L: *LineOffset))
705 return sampleprof_error::illegal_line_offset;
706
707 auto Discriminator = readNumber<uint64_t>();
708 if (std::error_code EC = Discriminator.getError())
709 return EC;
710
711 // Here we handle FS discriminators:
712 const uint32_t DiscriminatorVal = (*Discriminator) & getDiscriminatorMask();
713
714 if (std::error_code EC = readVTableTypeCountMap(M&: FProfile.getTypeSamplesAt(
715 Loc: LineLocation(*LineOffset, DiscriminatorVal))))
716 return EC;
717 }
718 return sampleprof_error::success;
719}
720
721std::error_code
722SampleProfileReaderBinary::readProfile(FunctionSamples &FProfile) {
723 auto NumSamples = readNumber<uint64_t>();
724 if (std::error_code EC = NumSamples.getError())
725 return EC;
726 FProfile.addTotalSamples(Num: *NumSamples);
727
728 // Read the samples in the body.
729 auto NumRecords = readNumber<uint32_t>();
730 if (std::error_code EC = NumRecords.getError())
731 return EC;
732 FProfile.reserveBodySamples(NumEntries: *NumRecords);
733
734 for (uint32_t I = 0; I < *NumRecords; ++I) {
735 auto LineOffset = readNumber<uint64_t>();
736 if (std::error_code EC = LineOffset.getError())
737 return EC;
738
739 if (!isOffsetLegal(L: *LineOffset)) {
740 return sampleprof_error::illegal_line_offset;
741 }
742
743 auto Discriminator = readNumber<uint64_t>();
744 if (std::error_code EC = Discriminator.getError())
745 return EC;
746
747 auto NumSamples = readNumber<uint64_t>();
748 if (std::error_code EC = NumSamples.getError())
749 return EC;
750
751 auto NumCalls = readNumber<uint32_t>();
752 if (std::error_code EC = NumCalls.getError())
753 return EC;
754
755 // Here we handle FS discriminators:
756 uint32_t DiscriminatorVal = (*Discriminator) & getDiscriminatorMask();
757
758 for (uint32_t J = 0; J < *NumCalls; ++J) {
759 auto CalledFunction(readStringFromTable());
760 if (std::error_code EC = CalledFunction.getError())
761 return EC;
762
763 auto CalledFunctionSamples = readNumber<uint64_t>();
764 if (std::error_code EC = CalledFunctionSamples.getError())
765 return EC;
766
767 FProfile.addCalledTargetSamples(LineOffset: *LineOffset, Discriminator: DiscriminatorVal,
768 Func: *CalledFunction, Num: *CalledFunctionSamples);
769 }
770
771 FProfile.addBodySamples(LineOffset: *LineOffset, Discriminator: DiscriminatorVal, Num: *NumSamples);
772 }
773
774 // Read all the samples for inlined function calls.
775 auto NumCallsites = readNumber<uint32_t>();
776 if (std::error_code EC = NumCallsites.getError())
777 return EC;
778
779 for (uint32_t J = 0; J < *NumCallsites; ++J) {
780 auto LineOffset = readNumber<uint64_t>();
781 if (std::error_code EC = LineOffset.getError())
782 return EC;
783
784 auto Discriminator = readNumber<uint64_t>();
785 if (std::error_code EC = Discriminator.getError())
786 return EC;
787
788 auto FName(readStringFromTable());
789 if (std::error_code EC = FName.getError())
790 return EC;
791
792 // Here we handle FS discriminators:
793 uint32_t DiscriminatorVal = (*Discriminator) & getDiscriminatorMask();
794
795 FunctionSamples &CalleeProfile = FProfile.functionSamplesAt(
796 Loc: LineLocation(*LineOffset, DiscriminatorVal))[*FName];
797 CalleeProfile.setFunction(*FName);
798 if (std::error_code EC = readProfile(FProfile&: CalleeProfile))
799 return EC;
800 }
801
802 if (ReadVTableProf)
803 return readCallsiteVTableProf(FProfile);
804
805 return sampleprof_error::success;
806}
807
808std::error_code
809SampleProfileReaderBinary::readFuncProfile(const uint8_t *Start,
810 SampleProfileMap &Profiles) {
811 Data = Start;
812 auto NumHeadSamples = readNumber<uint64_t>();
813 if (std::error_code EC = NumHeadSamples.getError())
814 return EC;
815
816 auto FContextHash(readSampleContextFromTable());
817 if (std::error_code EC = FContextHash.getError())
818 return EC;
819
820 auto &[FContext, Hash] = *FContextHash;
821 // Use the cached hash value for insertion instead of recalculating it.
822 auto Res = Profiles.try_emplace(Hash, Key: FContext, Args: FunctionSamples());
823 FunctionSamples &FProfile = Res.first->second;
824 FProfile.setContext(FContext);
825 FProfile.addHeadSamples(Num: *NumHeadSamples);
826
827 if (FContext.hasContext())
828 CSProfileCount++;
829
830 if (std::error_code EC = readProfile(FProfile))
831 return EC;
832 return sampleprof_error::success;
833}
834
835std::error_code
836SampleProfileReaderBinary::readFuncProfile(const uint8_t *Start) {
837 return readFuncProfile(Start, Profiles);
838}
839
840std::error_code SampleProfileReaderBinary::readImpl() {
841 ProfileIsFS = ProfileIsFSDisciminator;
842 FunctionSamples::ProfileIsFS = ProfileIsFS;
843 while (Data < End) {
844 if (std::error_code EC = readFuncProfile(Start: Data))
845 return EC;
846 }
847
848 return sampleprof_error::success;
849}
850
851std::error_code SampleProfileReaderExtBinaryBase::readOneSection(
852 const uint8_t *Start, uint64_t Size, const SecHdrTableEntry &Entry) {
853 Data = Start;
854 End = Start + Size;
855 switch (Entry.Type) {
856 case SecProfSummary:
857 if (std::error_code EC = readSummary())
858 return EC;
859 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagPartial))
860 Summary->setPartialProfile(true);
861 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagFullContext))
862 FunctionSamples::ProfileIsCS = ProfileIsCS = true;
863 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagIsPreInlined))
864 FunctionSamples::ProfileIsPreInlined = ProfileIsPreInlined = true;
865 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagFSDiscriminator))
866 FunctionSamples::ProfileIsFS = ProfileIsFS = true;
867 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagHasVTableTypeProf))
868 ReadVTableProf = true;
869 break;
870 case SecNameTable: {
871 bool FixedLengthMD5 =
872 hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagFixedLengthMD5);
873 bool UseMD5 = hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagMD5Name);
874 // UseMD5 means if THIS section uses MD5, ProfileIsMD5 means if the entire
875 // profile uses MD5 for function name matching in IPO passes.
876 ProfileIsMD5 = ProfileIsMD5 || UseMD5;
877 FunctionSamples::HasUniqSuffix =
878 hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagUniqSuffix);
879 bool IsEytzinger = hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagEytzinger);
880 if (std::error_code EC =
881 readNameTableSec(IsMD5: UseMD5, FixedLengthMD5, IsEytzinger))
882 return EC;
883 break;
884 }
885 case SecCSNameTable: {
886 if (std::error_code EC = readCSNameTableSec())
887 return EC;
888 break;
889 }
890 case SecLBRProfile:
891 ProfileSecRange = std::make_pair(x&: Data, y&: End);
892 if (std::error_code EC = readFuncProfiles())
893 return EC;
894 break;
895 case SecFuncOffsetTable:
896 // If module is absent, we are using LLVM tools, and need to read all
897 // profiles, so skip reading the function offset table.
898 if (!M) {
899 Data = End;
900 } else {
901 bool IsEytzinger =
902 hasSecFlag(Entry, Flag: SecFuncOffsetFlags::SecFlagEytzinger);
903 bool IsFlat = hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagFlat);
904 // An unflagged function offset table inherently indexes the primary
905 // Nested symbol span.
906 bool IsNested = !IsFlat;
907 assert((!ProfileIsCS ||
908 hasSecFlag(Entry, SecFuncOffsetFlags::SecFlagOrdered) ||
909 IsEytzinger) &&
910 "func offset table should always be sorted or in Eytzinger BFS "
911 "order in CS profile");
912 if (std::error_code EC = readFuncOffsetTable(IsEytzinger, IsNested))
913 return EC;
914 }
915 break;
916 case SecFuncMetadata: {
917 ProfileIsProbeBased =
918 hasSecFlag(Entry, Flag: SecFuncMetadataFlags::SecFlagIsProbeBased);
919 FunctionSamples::ProfileIsProbeBased = ProfileIsProbeBased;
920 ProfileHasAttribute =
921 hasSecFlag(Entry, Flag: SecFuncMetadataFlags::SecFlagHasAttribute);
922 if (std::error_code EC = readFuncMetadata())
923 return EC;
924 break;
925 }
926 case SecProfileSymbolList:
927 if (std::error_code EC = readProfileSymbolList(
928 IsMD5: hasSecFlag(Entry, Flag: SecProfileSymbolListFlags::SecFlagMD5)))
929 return EC;
930 break;
931 default:
932 if (std::error_code EC = readCustomSection(Entry))
933 return EC;
934 break;
935 }
936 return sampleprof_error::success;
937}
938
939bool SampleProfileReaderExtBinaryBase::useFuncOffsetList() const {
940 // If profile is CS, the function offset section is expected to consist of
941 // sequences of contexts in pre-order layout
942 // (e.g. [A, A:1 @ B, A:1 @ B:2.3 @ C] [D, D:1 @ E]), so that when a matched
943 // context in the module is found, the profiles of all its callees are
944 // recursively loaded. A list is needed since the order of profiles matters.
945 if (ProfileIsCS)
946 return true;
947
948 // If the profile is MD5, use the map container to lookup functions in
949 // the module. A remapper has no use on MD5 names.
950 if (useMD5())
951 return false;
952
953 // Profile is not MD5 and if a remapper is present, the remapped name of
954 // every function needed to be matched against the module, so use the list
955 // container since each entry is accessed.
956 if (Remapper)
957 return true;
958
959 // Otherwise use the map container for faster lookup.
960 // TODO: If the cardinality of the function offset section is much smaller
961 // than the number of functions in the module, using the list container can
962 // be always faster, but we need to figure out the constant factor to
963 // determine the cutoff.
964 return false;
965}
966
967std::error_code
968SampleProfileReaderExtBinaryBase::read(const DenseSet<StringRef> &FuncsToUse,
969 SampleProfileMap &Profiles) {
970 if (FuncsToUse.empty())
971 return sampleprof_error::success;
972
973 Data = ProfileSecRange.first;
974 End = ProfileSecRange.second;
975 if (std::error_code EC = readFuncProfiles(FuncsToUse, Profiles))
976 return EC;
977 End = Data;
978 DenseSet<FunctionSamples *> ProfilesToReadMetadata;
979 for (auto FName : FuncsToUse) {
980 auto I = Profiles.find(Ctx: FName);
981 if (I != Profiles.end())
982 ProfilesToReadMetadata.insert(V: &I->second);
983 }
984
985 if (std::error_code EC = readFuncMetadata(Profiles&: ProfilesToReadMetadata))
986 return EC;
987 return sampleprof_error::success;
988}
989
990bool SampleProfileReaderExtBinaryBase::collectFuncsFromModule() {
991 if (!M)
992 return false;
993 FuncsToUse.clear();
994 for (auto &F : *M)
995 FuncsToUse.insert(V: FunctionSamples::getCanonicalFnName(F));
996 return true;
997}
998
999std::error_code
1000SampleProfileReaderExtBinaryBase::readFuncOffsetTable(bool IsEytzinger,
1001 bool IsNested) {
1002 if (IsEytzinger)
1003 return readEytzingerFuncOffsetTable(IsNested);
1004 return readLegacyFuncOffsetTable();
1005}
1006
1007std::error_code
1008SampleProfileReaderExtBinaryBase::readEytzingerFuncOffsetTable(bool IsNested) {
1009 // If there are more than one function offset section, the profile associated
1010 // with the previous section has to be done reading before next one is read.
1011 FuncOffsetTable.reset();
1012
1013 size_t Size = End - Data;
1014 size_t SpanSize = NameTable->getEytzingerSpan(IsNested).size();
1015 if (Size != SpanSize * sizeof(uint32_t))
1016 return sampleprof_error::malformed;
1017
1018 auto *Array = reinterpret_cast<const support::ulittle32_t *>(Data);
1019 ArrayRef<support::ulittle32_t> Offsets(Array, SpanSize);
1020
1021 FuncOffsetTable.emplace(args: EytzingerMode, args: NameTable->getEytzingerSpan(IsNested),
1022 args&: Offsets);
1023
1024 Data = End;
1025 return sampleprof_error::success;
1026}
1027
1028std::error_code SampleProfileReaderExtBinaryBase::readLegacyFuncOffsetTable() {
1029 // If there are more than one function offset section, the profile associated
1030 // with the previous section has to be done reading before next one is read.
1031 FuncOffsetTable.reset();
1032 FuncOffsetList.clear();
1033
1034 auto Size = readNumber<uint64_t>();
1035 if (std::error_code EC = Size.getError())
1036 return EC;
1037
1038 bool UseFuncOffsetList = useFuncOffsetList();
1039 if (UseFuncOffsetList)
1040 FuncOffsetList.reserve(n: *Size);
1041 else
1042 FuncOffsetTable.emplace(args: InMemoryMode, args&: *Size);
1043
1044 for (uint64_t I = 0; I < *Size; ++I) {
1045 auto FContextHash(readSampleContextFromTable());
1046 if (std::error_code EC = FContextHash.getError())
1047 return EC;
1048
1049 auto &[FContext, Hash] = *FContextHash;
1050 auto Offset = readNumber<uint64_t>();
1051 if (std::error_code EC = Offset.getError())
1052 return EC;
1053
1054 if (UseFuncOffsetList)
1055 FuncOffsetList.emplace_back(args&: FContext, args&: *Offset);
1056 else
1057 // Because Porfiles replace existing value with new value if collision
1058 // happens, we also use the latest offset so that they are consistent.
1059 FuncOffsetTable->insert(GUID: Hash, Offset: *Offset);
1060 }
1061
1062 return sampleprof_error::success;
1063}
1064
1065std::error_code SampleProfileReaderExtBinaryBase::readFuncProfiles(
1066 const DenseSet<StringRef> &FuncsToUse, SampleProfileMap &Profiles) {
1067 const uint8_t *Start = Data;
1068
1069 if (Remapper) {
1070 for (auto Name : FuncsToUse) {
1071 Remapper->insert(FunctionName: Name);
1072 }
1073 }
1074
1075 if (FuncOffsetTable && FuncOffsetTable->isEytzinger() &&
1076 useFuncOffsetList()) {
1077 ArrayRef<support::ulittle32_t> Offsets = FuncOffsetTable->getFuncOffsets();
1078 if (Offsets.size() != FuncOffsetTable->getExpectedSize())
1079 return sampleprof_error::malformed;
1080 for (const auto &[LocalIdx, RelOffset] : llvm::enumerate(First&: Offsets)) {
1081 if (RelOffset == UINT32_MAX)
1082 continue;
1083 const uint8_t *FuncProfileAddr = Start + RelOffset;
1084 if (std::error_code EC = readFuncProfile(Start: FuncProfileAddr, Profiles))
1085 return EC;
1086 }
1087 return sampleprof_error::success;
1088 }
1089
1090 if (ProfileIsCS) {
1091 assert(useFuncOffsetList());
1092 DenseSet<uint64_t> FuncGuidsToUse;
1093 if (useMD5()) {
1094 for (auto Name : FuncsToUse)
1095 FuncGuidsToUse.insert(V: Function::getGUIDAssumingExternalLinkage(GlobalName: Name));
1096 }
1097
1098 // For each function in current module, load all context profiles for
1099 // the function as well as their callee contexts which can help profile
1100 // guided importing for ThinLTO. This can be achieved by walking
1101 // through an ordered context container, where contexts are laid out
1102 // as if they were walked in preorder of a context trie. While
1103 // traversing the trie, a link to the highest common ancestor node is
1104 // kept so that all of its decendants will be loaded.
1105 const SampleContext *CommonContext = nullptr;
1106 for (const auto &NameOffset : FuncOffsetList) {
1107 const auto &FContext = NameOffset.first;
1108 FunctionId FName = FContext.getFunction();
1109 StringRef FNameString;
1110 if (!useMD5())
1111 FNameString = FName.stringRef();
1112
1113 // For function in the current module, keep its farthest ancestor
1114 // context. This can be used to load itself and its child and
1115 // sibling contexts.
1116 if ((useMD5() && FuncGuidsToUse.count(V: FName.getHashCode())) ||
1117 (!useMD5() && (FuncsToUse.count(V: FNameString) ||
1118 (Remapper && Remapper->exist(FunctionName: FNameString))))) {
1119 if (!CommonContext || !CommonContext->isPrefixOf(That: FContext))
1120 CommonContext = &FContext;
1121 }
1122
1123 if (CommonContext == &FContext ||
1124 (CommonContext && CommonContext->isPrefixOf(That: FContext))) {
1125 // Load profile for the current context which originated from
1126 // the common ancestor.
1127 const uint8_t *FuncProfileAddr = Start + NameOffset.second;
1128 if (std::error_code EC = readFuncProfile(Start: FuncProfileAddr))
1129 return EC;
1130 }
1131 }
1132 } else if (useMD5()) {
1133 assert(!useFuncOffsetList());
1134 for (auto Name : FuncsToUse) {
1135 auto GUID = MD5Hash(Str: Name);
1136 if (auto Offset = FuncOffsetTable->lookup(GUID)) {
1137 const uint8_t *FuncProfileAddr = Start + *Offset;
1138 if (std::error_code EC = readFuncProfile(Start: FuncProfileAddr, Profiles))
1139 return EC;
1140 }
1141 }
1142 } else if (Remapper) {
1143 assert(useFuncOffsetList());
1144 for (auto NameOffset : FuncOffsetList) {
1145 SampleContext FContext(NameOffset.first);
1146 auto FuncName = FContext.getFunction();
1147 StringRef FuncNameStr = FuncName.stringRef();
1148 if (!FuncsToUse.count(V: FuncNameStr) && !Remapper->exist(FunctionName: FuncNameStr))
1149 continue;
1150 const uint8_t *FuncProfileAddr = Start + NameOffset.second;
1151 if (std::error_code EC = readFuncProfile(Start: FuncProfileAddr, Profiles))
1152 return EC;
1153 }
1154 } else {
1155 assert(!useFuncOffsetList());
1156 for (auto Name : FuncsToUse) {
1157 if (auto Offset = FuncOffsetTable->lookup(GUID: MD5Hash(Str: Name))) {
1158 const uint8_t *FuncProfileAddr = Start + *Offset;
1159 if (std::error_code EC = readFuncProfile(Start: FuncProfileAddr, Profiles))
1160 return EC;
1161 }
1162 }
1163 }
1164
1165 return sampleprof_error::success;
1166}
1167
1168std::error_code SampleProfileReaderExtBinaryBase::readFuncProfiles() {
1169 // Collect functions used by current module if the Reader has been
1170 // given a module.
1171 // collectFuncsFromModule uses FunctionSamples::getCanonicalFnName
1172 // which will query FunctionSamples::HasUniqSuffix, so it has to be
1173 // called after FunctionSamples::HasUniqSuffix is set, i.e. after
1174 // NameTable section is read.
1175 bool LoadFuncsToBeUsed = collectFuncsFromModule();
1176
1177 // When LoadFuncsToBeUsed is false, we are using LLVM tool, need to read all
1178 // profiles.
1179 if (!LoadFuncsToBeUsed) {
1180 while (Data < End) {
1181 if (std::error_code EC = readFuncProfile(Start: Data))
1182 return EC;
1183 }
1184 assert(Data == End && "More data is read than expected");
1185 } else {
1186 // Load function profiles on demand.
1187 if (std::error_code EC = readFuncProfiles(FuncsToUse, Profiles))
1188 return EC;
1189 Data = End;
1190 }
1191 assert((CSProfileCount == 0 || CSProfileCount == Profiles.size()) &&
1192 "Cannot have both context-sensitive and regular profile");
1193 assert((!CSProfileCount || ProfileIsCS) &&
1194 "Section flag should be consistent with actual profile");
1195 return sampleprof_error::success;
1196}
1197
1198std::error_code
1199SampleProfileReaderExtBinaryBase::readProfileSymbolList(bool IsMD5) {
1200 if (IsMD5)
1201 return readMD5ProfileSymbolList();
1202 return readStringBasedProfileSymbolList();
1203}
1204
1205std::error_code SampleProfileReaderExtBinaryBase::readMD5ProfileSymbolList() {
1206 size_t Size = End - Data;
1207 if (Size % sizeof(uint64_t) != 0)
1208 return sampleprof_error::truncated;
1209 const auto *Table = reinterpret_cast<const support::ulittle64_t *>(Data);
1210 size_t NumEntries = Size / sizeof(uint64_t);
1211 if (!ProfSymList)
1212 ProfSymList = std::make_unique<ProfileSymbolList>();
1213 ProfSymList->setColdGUIDTable(
1214 EytzingerTableSpan<support::ulittle64_t>(Table, NumEntries));
1215 Data = End;
1216 return sampleprof_error::success;
1217}
1218
1219std::error_code
1220SampleProfileReaderExtBinaryBase::readStringBasedProfileSymbolList() {
1221 if (!ProfSymList)
1222 ProfSymList = std::make_unique<ProfileSymbolList>();
1223
1224 if (std::error_code EC = ProfSymList->read(Data, ListSize: End - Data))
1225 return EC;
1226
1227 Data = End;
1228 return sampleprof_error::success;
1229}
1230
1231std::error_code SampleProfileReaderExtBinaryBase::decompressSection(
1232 const uint8_t *SecStart, const uint64_t SecSize,
1233 const uint8_t *&DecompressBuf, uint64_t &DecompressBufSize) {
1234 Data = SecStart;
1235 End = SecStart + SecSize;
1236 auto DecompressSize = readNumber<uint64_t>();
1237 if (std::error_code EC = DecompressSize.getError())
1238 return EC;
1239 DecompressBufSize = *DecompressSize;
1240
1241 auto CompressSize = readNumber<uint64_t>();
1242 if (std::error_code EC = CompressSize.getError())
1243 return EC;
1244
1245 if (!llvm::compression::zlib::isAvailable())
1246 return sampleprof_error::zlib_unavailable;
1247
1248 uint8_t *Buffer = Allocator.Allocate<uint8_t>(Num: DecompressBufSize);
1249 size_t UCSize = DecompressBufSize;
1250 llvm::Error E = compression::zlib::decompress(Input: ArrayRef(Data, *CompressSize),
1251 Output: Buffer, UncompressedSize&: UCSize);
1252 if (E)
1253 return sampleprof_error::uncompress_failed;
1254 DecompressBuf = reinterpret_cast<const uint8_t *>(Buffer);
1255 return sampleprof_error::success;
1256}
1257
1258std::error_code SampleProfileReaderExtBinaryBase::readImpl() {
1259 const uint8_t *BufStart =
1260 reinterpret_cast<const uint8_t *>(Buffer->getBufferStart());
1261
1262 for (auto &Entry : SecHdrTable) {
1263 // Skip empty section.
1264 if (!Entry.Size)
1265 continue;
1266
1267 // Skip sections without inlined functions when SkipFlatProf is true.
1268 if (SkipFlatProf && hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagFlat))
1269 continue;
1270
1271 const uint8_t *SecStart = BufStart + Entry.Offset;
1272 uint64_t SecSize = Entry.Size;
1273
1274 // If the section is compressed, decompress it into a buffer
1275 // DecompressBuf before reading the actual data. The pointee of
1276 // 'Data' will be changed to buffer hold by DecompressBuf
1277 // temporarily when reading the actual data.
1278 bool isCompressed = hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagCompress);
1279 if (isCompressed) {
1280 const uint8_t *DecompressBuf;
1281 uint64_t DecompressBufSize;
1282 if (std::error_code EC = decompressSection(
1283 SecStart, SecSize, DecompressBuf, DecompressBufSize))
1284 return EC;
1285 SecStart = DecompressBuf;
1286 SecSize = DecompressBufSize;
1287 }
1288
1289 if (std::error_code EC = readOneSection(Start: SecStart, Size: SecSize, Entry))
1290 return EC;
1291 if (Data != SecStart + SecSize)
1292 return sampleprof_error::malformed;
1293
1294 // Change the pointee of 'Data' from DecompressBuf to original Buffer.
1295 if (isCompressed) {
1296 Data = BufStart + Entry.Offset;
1297 End = BufStart + Buffer->getBufferSize();
1298 }
1299 }
1300
1301 return sampleprof_error::success;
1302}
1303
1304std::error_code SampleProfileReaderRawBinary::verifySPMagic(uint64_t Magic) {
1305 if (Magic == SPMagic())
1306 return sampleprof_error::success;
1307 return sampleprof_error::bad_magic;
1308}
1309
1310std::error_code SampleProfileReaderExtBinary::verifySPMagic(uint64_t Magic) {
1311 if (Magic == SPMagic(Format: SPF_Ext_Binary))
1312 return sampleprof_error::success;
1313 return sampleprof_error::bad_magic;
1314}
1315
1316std::error_code SampleProfileReaderBinary::readNameTable() {
1317 auto Size = readNumber<size_t>();
1318 if (std::error_code EC = Size.getError())
1319 return EC;
1320
1321 // Normally if useMD5 is true, the name table should have MD5 values, not
1322 // strings, however in the case that ExtBinary profile has multiple name
1323 // tables mixing string and MD5, all of them have to be normalized to use MD5,
1324 // because optimization passes can only handle either type.
1325 bool UseMD5 = useMD5();
1326
1327 std::vector<FunctionId> TableVec;
1328 TableVec.reserve(n: *Size);
1329 if (!ProfileIsCS) {
1330 MD5SampleContextTable.clear();
1331 if (UseMD5)
1332 MD5SampleContextTable.reserve(n: *Size);
1333 else
1334 // If we are using strings, delay MD5 computation since only a portion of
1335 // names are used by top level functions. Use 0 to indicate MD5 value is
1336 // to be calculated as no known string has a MD5 value of 0.
1337 MD5SampleContextTable.resize(new_size: *Size);
1338 }
1339 for (size_t I = 0; I < *Size; ++I) {
1340 auto Name(readString());
1341 if (std::error_code EC = Name.getError())
1342 return EC;
1343 if (UseMD5) {
1344 FunctionId FID(*Name);
1345 if (!ProfileIsCS)
1346 MD5SampleContextTable.emplace_back(args: FID.getHashCode());
1347 TableVec.emplace_back(args&: FID);
1348 } else
1349 TableVec.push_back(x: FunctionId(*Name));
1350 }
1351 if (!ProfileIsCS)
1352 MD5SampleContextStart = MD5SampleContextTable.data();
1353 if (UseMD5)
1354 NameTable =
1355 std::make_unique<MD5SampleProfileNameTable>(args: std::move(TableVec));
1356 else
1357 NameTable =
1358 std::make_unique<StringSampleProfileNameTable>(args: std::move(TableVec));
1359 return sampleprof_error::success;
1360}
1361
1362std::error_code SampleProfileReaderExtBinaryBase::readNameTableSec(
1363 bool IsMD5, bool FixedLengthMD5, bool IsEytzinger) {
1364 if (IsEytzinger)
1365 return readNameTableSecEytzinger(IsMD5, FixedLengthMD5);
1366 return readNameTableSecLegacy(IsMD5, FixedLengthMD5);
1367}
1368
1369// Read the Eytzinger layout for SecNameTable from an ExtBinary MD5 profile.
1370//
1371// The section consists of three sequential ULEB128 symbol counts (Nested, Flat,
1372// and Inlinees) followed by their corresponding arrays of 64-bit MD5 hash keys
1373// laid out in Eytzinger order.
1374std::error_code SampleProfileReaderExtBinaryBase::readNameTableSecEytzinger(
1375 bool IsMD5, bool FixedLengthMD5) {
1376 assert(IsMD5 && "Eytzinger name tables require MD5 representation");
1377 if (!IsMD5)
1378 return sampleprof_error::malformed;
1379
1380 // Read the table sizes for Nested, flat, and inlinee symbols.
1381 std::array<uint64_t, static_cast<size_t>(EytzingerSpan::NumSpans)> Counts;
1382 for (uint64_t &Count : Counts) {
1383 auto ValOrErr = readNumber<uint64_t>();
1384 if (std::error_code EC = ValOrErr.getError())
1385 return EC;
1386 Count = *ValOrErr;
1387 }
1388 auto [NumNested, NumFlat, NumInlinees] = Counts;
1389
1390 // Guard against unsigned overflow in total entry computation.
1391 if (NumNested > std::numeric_limits<uint32_t>::max() ||
1392 NumFlat > std::numeric_limits<uint32_t>::max() ||
1393 NumInlinees > std::numeric_limits<uint32_t>::max())
1394 return sampleprof_error::malformed;
1395
1396 uint64_t TotalEntries = NumNested + NumFlat + NumInlinees;
1397 if (static_cast<size_t>(End - Data) < TotalEntries * sizeof(uint64_t))
1398 return sampleprof_error::truncated;
1399
1400 NameTable = std::make_unique<EytzingerSampleProfileNameTable>(
1401 args: reinterpret_cast<const support::ulittle64_t *>(Data), args&: NumNested, args&: NumFlat,
1402 args&: NumInlinees);
1403
1404 if (!ProfileIsCS)
1405 MD5SampleContextStart = reinterpret_cast<const uint64_t *>(Data);
1406 Data = Data + TotalEntries * sizeof(uint64_t);
1407 return sampleprof_error::success;
1408}
1409
1410std::error_code
1411SampleProfileReaderExtBinaryBase::readNameTableSecLegacy(bool IsMD5,
1412 bool FixedLengthMD5) {
1413 if (FixedLengthMD5) {
1414 if (!IsMD5)
1415 errs() << "If FixedLengthMD5 is true, UseMD5 has to be true";
1416 auto Size = readNumber<size_t>();
1417 if (std::error_code EC = Size.getError())
1418 return EC;
1419
1420 assert(Data + (*Size) * sizeof(uint64_t) == End &&
1421 "Fixed length MD5 name table does not contain specified number of "
1422 "entries");
1423 if (Data + (*Size) * sizeof(uint64_t) > End)
1424 return sampleprof_error::truncated;
1425
1426 if (LazyLoadNameTable) {
1427 NameTable = std::make_unique<LazySampleProfileNameTable>(args&: Data, args&: *Size);
1428 } else {
1429 std::vector<FunctionId> TableVec;
1430 TableVec.reserve(n: *Size);
1431 for (size_t I = 0; I < *Size; ++I) {
1432 using namespace support;
1433 uint64_t FID = endian::read<uint64_t, unaligned>(
1434 memory: Data + I * sizeof(uint64_t), endian: endianness::little);
1435 TableVec.emplace_back(args: FunctionId(FID));
1436 }
1437 NameTable =
1438 std::make_unique<MD5SampleProfileNameTable>(args: std::move(TableVec));
1439 }
1440 if (!ProfileIsCS)
1441 MD5SampleContextStart = reinterpret_cast<const uint64_t *>(Data);
1442 Data = Data + (*Size) * sizeof(uint64_t);
1443 return sampleprof_error::success;
1444 }
1445
1446 if (IsMD5) {
1447 assert(!FixedLengthMD5 && "FixedLengthMD5 should be unreachable here");
1448 auto Size = readNumber<size_t>();
1449 if (std::error_code EC = Size.getError())
1450 return EC;
1451
1452 std::vector<FunctionId> TableVec;
1453 TableVec.reserve(n: *Size);
1454 if (!ProfileIsCS)
1455 MD5SampleContextTable.resize(new_size: *Size);
1456 for (size_t I = 0; I < *Size; ++I) {
1457 auto FID = readNumber<uint64_t>();
1458 if (std::error_code EC = FID.getError())
1459 return EC;
1460 if (!ProfileIsCS)
1461 support::endian::write64le(P: &MD5SampleContextTable[I], V: *FID);
1462 TableVec.emplace_back(args: FunctionId(*FID));
1463 }
1464 if (!ProfileIsCS)
1465 MD5SampleContextStart = MD5SampleContextTable.data();
1466 NameTable =
1467 std::make_unique<MD5SampleProfileNameTable>(args: std::move(TableVec));
1468 return sampleprof_error::success;
1469 }
1470
1471 return SampleProfileReaderBinary::readNameTable();
1472}
1473
1474// Read in the CS name table section, which basically contains a list of context
1475// vectors. Each element of a context vector, aka a frame, refers to the
1476// underlying raw function names that are stored in the name table, as well as
1477// a callsite identifier that only makes sense for non-leaf frames.
1478std::error_code SampleProfileReaderExtBinaryBase::readCSNameTableSec() {
1479 auto Size = readNumber<size_t>();
1480 if (std::error_code EC = Size.getError())
1481 return EC;
1482
1483 CSNameTable.clear();
1484 CSNameTable.reserve(n: *Size);
1485 if (ProfileIsCS) {
1486 // Delay MD5 computation of CS context until they are needed. Use 0 to
1487 // indicate MD5 value is to be calculated as no known string has a MD5
1488 // value of 0.
1489 MD5SampleContextTable.clear();
1490 MD5SampleContextTable.resize(new_size: *Size);
1491 MD5SampleContextStart = MD5SampleContextTable.data();
1492 }
1493 for (size_t I = 0; I < *Size; ++I) {
1494 CSNameTable.emplace_back(args: SampleContextFrameVector());
1495 auto ContextSize = readNumber<uint32_t>();
1496 if (std::error_code EC = ContextSize.getError())
1497 return EC;
1498 for (uint32_t J = 0; J < *ContextSize; ++J) {
1499 auto FName(readStringFromTable());
1500 if (std::error_code EC = FName.getError())
1501 return EC;
1502 auto LineOffset = readNumber<uint64_t>();
1503 if (std::error_code EC = LineOffset.getError())
1504 return EC;
1505
1506 if (!isOffsetLegal(L: *LineOffset))
1507 return sampleprof_error::illegal_line_offset;
1508
1509 auto Discriminator = readNumber<uint64_t>();
1510 if (std::error_code EC = Discriminator.getError())
1511 return EC;
1512
1513 CSNameTable.back().emplace_back(
1514 Args&: FName.get(), Args: LineLocation(LineOffset.get(), Discriminator.get()));
1515 }
1516 }
1517
1518 return sampleprof_error::success;
1519}
1520
1521std::error_code
1522SampleProfileReaderExtBinaryBase::readFuncMetadata(FunctionSamples *FProfile) {
1523 if (Data < End) {
1524 if (ProfileIsProbeBased) {
1525 auto Checksum = readNumber<uint64_t>();
1526 if (std::error_code EC = Checksum.getError())
1527 return EC;
1528 if (FProfile)
1529 FProfile->setFunctionHash(*Checksum);
1530 }
1531
1532 if (ProfileHasAttribute) {
1533 auto Attributes = readNumber<uint32_t>();
1534 if (std::error_code EC = Attributes.getError())
1535 return EC;
1536 if (FProfile)
1537 FProfile->getContext().setAllAttributes(*Attributes);
1538 }
1539
1540 if (!ProfileIsCS) {
1541 // Read all the attributes for inlined function calls.
1542 auto NumCallsites = readNumber<uint32_t>();
1543 if (std::error_code EC = NumCallsites.getError())
1544 return EC;
1545
1546 for (uint32_t J = 0; J < *NumCallsites; ++J) {
1547 auto LineOffset = readNumber<uint64_t>();
1548 if (std::error_code EC = LineOffset.getError())
1549 return EC;
1550
1551 auto Discriminator = readNumber<uint64_t>();
1552 if (std::error_code EC = Discriminator.getError())
1553 return EC;
1554
1555 auto FContextHash(readSampleContextFromTable());
1556 if (std::error_code EC = FContextHash.getError())
1557 return EC;
1558
1559 auto &[FContext, Hash] = *FContextHash;
1560 FunctionSamples *CalleeProfile = nullptr;
1561 if (FProfile) {
1562 CalleeProfile = const_cast<FunctionSamples *>(
1563 &FProfile->functionSamplesAt(Loc: LineLocation(
1564 *LineOffset, *Discriminator))[FContext.getFunction()]);
1565 }
1566 if (std::error_code EC = readFuncMetadata(FProfile: CalleeProfile))
1567 return EC;
1568 }
1569 }
1570 }
1571
1572 return sampleprof_error::success;
1573}
1574
1575std::error_code SampleProfileReaderExtBinaryBase::readFuncMetadata(
1576 DenseSet<FunctionSamples *> &Profiles) {
1577 if (FuncMetadataIndex.empty())
1578 return sampleprof_error::success;
1579
1580 for (auto *FProfile : Profiles) {
1581 auto R = FuncMetadataIndex.find(Val: FProfile->getContext().getHashCode());
1582 if (R == FuncMetadataIndex.end())
1583 continue;
1584
1585 Data = R->second.first;
1586 End = R->second.second;
1587 if (std::error_code EC = readFuncMetadata(FProfile))
1588 return EC;
1589 assert(Data == End && "More data is read than expected");
1590 }
1591 return sampleprof_error::success;
1592}
1593
1594std::error_code SampleProfileReaderExtBinaryBase::readFuncMetadata() {
1595 while (Data < End) {
1596 auto FContextHash(readSampleContextFromTable());
1597 if (std::error_code EC = FContextHash.getError())
1598 return EC;
1599 auto &[FContext, Hash] = *FContextHash;
1600 FunctionSamples *FProfile = nullptr;
1601 auto It = Profiles.find(Ctx: FContext);
1602 if (It != Profiles.end())
1603 FProfile = &It->second;
1604
1605 const uint8_t *Start = Data;
1606 if (std::error_code EC = readFuncMetadata(FProfile))
1607 return EC;
1608
1609 FuncMetadataIndex[FContext.getHashCode()] = {Start, Data};
1610 }
1611
1612 assert(Data == End && "More data is read than expected");
1613 return sampleprof_error::success;
1614}
1615
1616std::error_code
1617SampleProfileReaderExtBinaryBase::readSecHdrTableEntry(uint64_t Idx) {
1618 SecHdrTableEntry Entry;
1619 auto Type = readUnencodedNumber<uint64_t>();
1620 if (std::error_code EC = Type.getError())
1621 return EC;
1622 Entry.Type = static_cast<SecType>(*Type);
1623
1624 auto Flags = readUnencodedNumber<uint64_t>();
1625 if (std::error_code EC = Flags.getError())
1626 return EC;
1627 Entry.Flags = *Flags;
1628
1629 auto Offset = readUnencodedNumber<uint64_t>();
1630 if (std::error_code EC = Offset.getError())
1631 return EC;
1632 Entry.Offset = *Offset;
1633
1634 auto Size = readUnencodedNumber<uint64_t>();
1635 if (std::error_code EC = Size.getError())
1636 return EC;
1637 Entry.Size = *Size;
1638
1639 Entry.LayoutIndex = Idx;
1640 SecHdrTable.push_back(x: std::move(Entry));
1641 return sampleprof_error::success;
1642}
1643
1644std::error_code SampleProfileReaderExtBinaryBase::readSecHdrTable() {
1645 auto EntryNum = readUnencodedNumber<uint64_t>();
1646 if (std::error_code EC = EntryNum.getError())
1647 return EC;
1648
1649 for (uint64_t i = 0; i < (*EntryNum); i++)
1650 if (std::error_code EC = readSecHdrTableEntry(Idx: i))
1651 return EC;
1652
1653 return sampleprof_error::success;
1654}
1655
1656std::error_code SampleProfileReaderExtBinaryBase::readHeader() {
1657 const uint8_t *BufStart =
1658 reinterpret_cast<const uint8_t *>(Buffer->getBufferStart());
1659 Data = BufStart;
1660 End = BufStart + Buffer->getBufferSize();
1661
1662 if (std::error_code EC = readMagicIdent())
1663 return EC;
1664
1665 if (std::error_code EC = readSecHdrTable())
1666 return EC;
1667
1668 return sampleprof_error::success;
1669}
1670
1671uint64_t SampleProfileReaderExtBinaryBase::getSectionSize(SecType Type) {
1672 uint64_t Size = 0;
1673 for (auto &Entry : SecHdrTable) {
1674 if (Entry.Type == Type)
1675 Size += Entry.Size;
1676 }
1677 return Size;
1678}
1679
1680uint64_t SampleProfileReaderExtBinaryBase::getFileSize() {
1681 // Sections in SecHdrTable is not necessarily in the same order as
1682 // sections in the profile because section like FuncOffsetTable needs
1683 // to be written after section LBRProfile but needs to be read before
1684 // section LBRProfile, so we cannot simply use the last entry in
1685 // SecHdrTable to calculate the file size.
1686 uint64_t FileSize = 0;
1687 for (auto &Entry : SecHdrTable) {
1688 FileSize = std::max(a: Entry.Offset + Entry.Size, b: FileSize);
1689 }
1690 return FileSize;
1691}
1692
1693static std::string getSecFlagsStr(const SecHdrTableEntry &Entry) {
1694 std::string Flags;
1695 if (hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagCompress))
1696 Flags.append(s: "{compressed,");
1697 else
1698 Flags.append(s: "{");
1699
1700 if (hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagFlat))
1701 Flags.append(s: "flat,");
1702
1703 switch (Entry.Type) {
1704 case SecNameTable:
1705 if (hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagEytzinger))
1706 Flags.append(s: "eytzinger,");
1707 if (hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagFixedLengthMD5))
1708 Flags.append(s: "fixlenmd5,");
1709 else if (hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagMD5Name))
1710 Flags.append(s: "md5,");
1711 if (hasSecFlag(Entry, Flag: SecNameTableFlags::SecFlagUniqSuffix))
1712 Flags.append(s: "uniq,");
1713 break;
1714 case SecProfSummary:
1715 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagPartial))
1716 Flags.append(s: "partial,");
1717 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagFullContext))
1718 Flags.append(s: "context,");
1719 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagIsPreInlined))
1720 Flags.append(s: "preInlined,");
1721 if (hasSecFlag(Entry, Flag: SecProfSummaryFlags::SecFlagFSDiscriminator))
1722 Flags.append(s: "fs-discriminator,");
1723 break;
1724 case SecFuncOffsetTable:
1725 if (hasSecFlag(Entry, Flag: SecFuncOffsetFlags::SecFlagOrdered))
1726 Flags.append(s: "ordered,");
1727 if (hasSecFlag(Entry, Flag: SecFuncOffsetFlags::SecFlagEytzinger))
1728 Flags.append(s: "eytzinger,");
1729 break;
1730 case SecFuncMetadata:
1731 if (hasSecFlag(Entry, Flag: SecFuncMetadataFlags::SecFlagIsProbeBased))
1732 Flags.append(s: "probe,");
1733 if (hasSecFlag(Entry, Flag: SecFuncMetadataFlags::SecFlagHasAttribute))
1734 Flags.append(s: "attr,");
1735 break;
1736 case SecProfileSymbolList:
1737 if (hasSecFlag(Entry, Flag: SecProfileSymbolListFlags::SecFlagMD5))
1738 Flags.append(s: "md5,");
1739 break;
1740 default:
1741 break;
1742 }
1743 char &last = Flags.back();
1744 if (last == ',')
1745 last = '}';
1746 else
1747 Flags.append(s: "}");
1748 return Flags;
1749}
1750
1751bool SampleProfileReaderExtBinaryBase::dumpSectionInfo(raw_ostream &OS) {
1752 uint64_t TotalSecsSize = 0;
1753 for (auto &Entry : SecHdrTable) {
1754 OS << getSecName(Type: Entry.Type) << " - Offset: " << Entry.Offset
1755 << ", Size: " << Entry.Size << ", Flags: " << getSecFlagsStr(Entry)
1756 << "\n";
1757 ;
1758 TotalSecsSize += Entry.Size;
1759 }
1760 uint64_t HeaderSize = SecHdrTable.front().Offset;
1761 assert(HeaderSize + TotalSecsSize == getFileSize() &&
1762 "Size of 'header + sections' doesn't match the total size of profile");
1763
1764 OS << "Header Size: " << HeaderSize << "\n";
1765 OS << "Total Sections Size: " << TotalSecsSize << "\n";
1766 OS << "File Size: " << getFileSize() << "\n";
1767 return true;
1768}
1769
1770std::error_code SampleProfileReaderBinary::readMagicIdent() {
1771 // Read and check the magic identifier.
1772 auto Magic = readNumber<uint64_t>();
1773 if (std::error_code EC = Magic.getError())
1774 return EC;
1775 else if (std::error_code EC = verifySPMagic(Magic: *Magic))
1776 return EC;
1777
1778 // Read the version number.
1779 auto Version = readNumber<uint64_t>();
1780 if (std::error_code EC = Version.getError())
1781 return EC;
1782 else if (!formatVersionIsSupported(Version: *Version))
1783 return sampleprof_error::unsupported_version;
1784 FormatVersion = *Version;
1785
1786 return sampleprof_error::success;
1787}
1788
1789std::error_code SampleProfileReaderBinary::readHeader() {
1790 Data = reinterpret_cast<const uint8_t *>(Buffer->getBufferStart());
1791 End = Data + Buffer->getBufferSize();
1792
1793 if (std::error_code EC = readMagicIdent())
1794 return EC;
1795
1796 if (std::error_code EC = readSummary())
1797 return EC;
1798
1799 if (std::error_code EC = readNameTable())
1800 return EC;
1801 return sampleprof_error::success;
1802}
1803
1804std::error_code SampleProfileReaderBinary::readSummaryEntry(
1805 std::vector<ProfileSummaryEntry> &Entries) {
1806 auto Cutoff = readNumber<uint64_t>();
1807 if (std::error_code EC = Cutoff.getError())
1808 return EC;
1809
1810 auto MinBlockCount = readNumber<uint64_t>();
1811 if (std::error_code EC = MinBlockCount.getError())
1812 return EC;
1813
1814 auto NumBlocks = readNumber<uint64_t>();
1815 if (std::error_code EC = NumBlocks.getError())
1816 return EC;
1817
1818 Entries.emplace_back(args&: *Cutoff, args&: *MinBlockCount, args&: *NumBlocks);
1819 return sampleprof_error::success;
1820}
1821
1822std::error_code SampleProfileReaderBinary::readSummary() {
1823 auto TotalCount = readNumber<uint64_t>();
1824 if (std::error_code EC = TotalCount.getError())
1825 return EC;
1826
1827 auto MaxBlockCount = readNumber<uint64_t>();
1828 if (std::error_code EC = MaxBlockCount.getError())
1829 return EC;
1830
1831 auto MaxFunctionCount = readNumber<uint64_t>();
1832 if (std::error_code EC = MaxFunctionCount.getError())
1833 return EC;
1834
1835 auto NumBlocks = readNumber<uint64_t>();
1836 if (std::error_code EC = NumBlocks.getError())
1837 return EC;
1838
1839 auto NumFunctions = readNumber<uint64_t>();
1840 if (std::error_code EC = NumFunctions.getError())
1841 return EC;
1842
1843 auto NumSummaryEntries = readNumber<uint64_t>();
1844 if (std::error_code EC = NumSummaryEntries.getError())
1845 return EC;
1846
1847 std::vector<ProfileSummaryEntry> Entries;
1848 for (unsigned i = 0; i < *NumSummaryEntries; i++) {
1849 std::error_code EC = readSummaryEntry(Entries);
1850 if (EC != sampleprof_error::success)
1851 return EC;
1852 }
1853 Summary = std::make_unique<ProfileSummary>(
1854 args: ProfileSummary::PSK_Sample, args&: Entries, args&: *TotalCount, args&: *MaxBlockCount, args: 0,
1855 args&: *MaxFunctionCount, args&: *NumBlocks, args&: *NumFunctions);
1856
1857 return sampleprof_error::success;
1858}
1859
1860bool SampleProfileReaderRawBinary::hasFormat(const MemoryBuffer &Buffer) {
1861 const uint8_t *Data =
1862 reinterpret_cast<const uint8_t *>(Buffer.getBufferStart());
1863 uint64_t Magic = decodeULEB128(p: Data);
1864 return Magic == SPMagic();
1865}
1866
1867bool SampleProfileReaderExtBinary::hasFormat(const MemoryBuffer &Buffer) {
1868 const uint8_t *Data =
1869 reinterpret_cast<const uint8_t *>(Buffer.getBufferStart());
1870 uint64_t Magic = decodeULEB128(p: Data);
1871 return Magic == SPMagic(Format: SPF_Ext_Binary);
1872}
1873
1874std::error_code SampleProfileReaderGCC::skipNextWord() {
1875 uint32_t dummy;
1876 if (!GcovBuffer.readInt(Val&: dummy))
1877 return sampleprof_error::truncated;
1878 return sampleprof_error::success;
1879}
1880
1881template <typename T> ErrorOr<T> SampleProfileReaderGCC::readNumber() {
1882 if (sizeof(T) <= sizeof(uint32_t)) {
1883 uint32_t Val;
1884 if (GcovBuffer.readInt(Val) && Val <= std::numeric_limits<T>::max())
1885 return static_cast<T>(Val);
1886 } else if (sizeof(T) <= sizeof(uint64_t)) {
1887 uint64_t Val;
1888 if (GcovBuffer.readInt64(Val) && Val <= std::numeric_limits<T>::max())
1889 return static_cast<T>(Val);
1890 }
1891
1892 std::error_code EC = sampleprof_error::malformed;
1893 reportError(LineNumber: 0, Msg: EC.message());
1894 return EC;
1895}
1896
1897ErrorOr<StringRef> SampleProfileReaderGCC::readString() {
1898 StringRef Str;
1899 if (!GcovBuffer.readString(str&: Str))
1900 return sampleprof_error::truncated;
1901 return Str;
1902}
1903
1904std::error_code SampleProfileReaderGCC::readHeader() {
1905 // Read the magic identifier.
1906 if (!GcovBuffer.readGCDAFormat())
1907 return sampleprof_error::unrecognized_format;
1908
1909 // Read the version number. Note - the GCC reader does not validate this
1910 // version, but the profile creator generates v704.
1911 GCOV::GCOVVersion version;
1912 if (!GcovBuffer.readGCOVVersion(version))
1913 return sampleprof_error::unrecognized_format;
1914
1915 if (version != GCOV::V407)
1916 return sampleprof_error::unsupported_version;
1917
1918 // Skip the empty integer.
1919 if (std::error_code EC = skipNextWord())
1920 return EC;
1921
1922 return sampleprof_error::success;
1923}
1924
1925std::error_code SampleProfileReaderGCC::readSectionTag(uint32_t Expected) {
1926 uint32_t Tag;
1927 if (!GcovBuffer.readInt(Val&: Tag))
1928 return sampleprof_error::truncated;
1929
1930 if (Tag != Expected)
1931 return sampleprof_error::malformed;
1932
1933 if (std::error_code EC = skipNextWord())
1934 return EC;
1935
1936 return sampleprof_error::success;
1937}
1938
1939std::error_code SampleProfileReaderGCC::readNameTable() {
1940 if (std::error_code EC = readSectionTag(Expected: GCOVTagAFDOFileNames))
1941 return EC;
1942
1943 uint32_t Size;
1944 if (!GcovBuffer.readInt(Val&: Size))
1945 return sampleprof_error::truncated;
1946
1947 for (uint32_t I = 0; I < Size; ++I) {
1948 StringRef Str;
1949 if (!GcovBuffer.readString(str&: Str))
1950 return sampleprof_error::truncated;
1951 Names.push_back(x: std::string(Str));
1952 }
1953
1954 return sampleprof_error::success;
1955}
1956
1957std::error_code SampleProfileReaderGCC::readFunctionProfiles() {
1958 if (std::error_code EC = readSectionTag(Expected: GCOVTagAFDOFunction))
1959 return EC;
1960
1961 uint32_t NumFunctions;
1962 if (!GcovBuffer.readInt(Val&: NumFunctions))
1963 return sampleprof_error::truncated;
1964
1965 InlineCallStack Stack;
1966 for (uint32_t I = 0; I < NumFunctions; ++I)
1967 if (std::error_code EC = readOneFunctionProfile(InlineStack: Stack, Update: true, Offset: 0))
1968 return EC;
1969
1970 computeSummary();
1971 return sampleprof_error::success;
1972}
1973
1974std::error_code SampleProfileReaderGCC::readOneFunctionProfile(
1975 const InlineCallStack &InlineStack, bool Update, uint32_t Offset) {
1976 uint64_t HeadCount = 0;
1977 if (InlineStack.size() == 0)
1978 if (!GcovBuffer.readInt64(Val&: HeadCount))
1979 return sampleprof_error::truncated;
1980
1981 uint32_t NameIdx;
1982 if (!GcovBuffer.readInt(Val&: NameIdx))
1983 return sampleprof_error::truncated;
1984
1985 StringRef Name(Names[NameIdx]);
1986
1987 uint32_t NumPosCounts;
1988 if (!GcovBuffer.readInt(Val&: NumPosCounts))
1989 return sampleprof_error::truncated;
1990
1991 uint32_t NumCallsites;
1992 if (!GcovBuffer.readInt(Val&: NumCallsites))
1993 return sampleprof_error::truncated;
1994
1995 FunctionSamples *FProfile = nullptr;
1996 if (InlineStack.size() == 0) {
1997 // If this is a top function that we have already processed, do not
1998 // update its profile again. This happens in the presence of
1999 // function aliases. Since these aliases share the same function
2000 // body, there will be identical replicated profiles for the
2001 // original function. In this case, we simply not bother updating
2002 // the profile of the original function.
2003 FProfile = &Profiles[FunctionId(Name)];
2004 FProfile->addHeadSamples(Num: HeadCount);
2005 if (FProfile->getTotalSamples() > 0)
2006 Update = false;
2007 } else {
2008 // Otherwise, we are reading an inlined instance. The top of the
2009 // inline stack contains the profile of the caller. Insert this
2010 // callee in the caller's CallsiteMap.
2011 FunctionSamples *CallerProfile = InlineStack.front();
2012 uint32_t LineOffset = Offset >> 16;
2013 uint32_t Discriminator = Offset & 0xffff;
2014 FProfile = &CallerProfile->functionSamplesAt(
2015 Loc: LineLocation(LineOffset, Discriminator))[FunctionId(Name)];
2016 }
2017 FProfile->setFunction(FunctionId(Name));
2018 FProfile->reserveBodySamples(NumEntries: NumPosCounts);
2019
2020 for (uint32_t I = 0; I < NumPosCounts; ++I) {
2021 uint32_t Offset;
2022 if (!GcovBuffer.readInt(Val&: Offset))
2023 return sampleprof_error::truncated;
2024
2025 uint32_t NumTargets;
2026 if (!GcovBuffer.readInt(Val&: NumTargets))
2027 return sampleprof_error::truncated;
2028
2029 uint64_t Count;
2030 if (!GcovBuffer.readInt64(Val&: Count))
2031 return sampleprof_error::truncated;
2032
2033 // The line location is encoded in the offset as:
2034 // high 16 bits: line offset to the start of the function.
2035 // low 16 bits: discriminator.
2036 uint32_t LineOffset = Offset >> 16;
2037 uint32_t Discriminator = Offset & 0xffff;
2038
2039 InlineCallStack NewStack;
2040 NewStack.push_back(Elt: FProfile);
2041 llvm::append_range(C&: NewStack, R: InlineStack);
2042 if (Update) {
2043 // Walk up the inline stack, adding the samples on this line to
2044 // the total sample count of the callers in the chain.
2045 for (auto *CallerProfile : NewStack)
2046 CallerProfile->addTotalSamples(Num: Count);
2047
2048 // Update the body samples for the current profile.
2049 FProfile->addBodySamples(LineOffset, Discriminator, Num: Count);
2050 }
2051
2052 // Process the list of functions called at an indirect call site.
2053 // These are all the targets that a function pointer (or virtual
2054 // function) resolved at runtime.
2055 for (uint32_t J = 0; J < NumTargets; J++) {
2056 uint32_t HistVal;
2057 if (!GcovBuffer.readInt(Val&: HistVal))
2058 return sampleprof_error::truncated;
2059
2060 if (HistVal != HIST_TYPE_INDIR_CALL_TOPN)
2061 return sampleprof_error::malformed;
2062
2063 uint64_t TargetIdx;
2064 if (!GcovBuffer.readInt64(Val&: TargetIdx))
2065 return sampleprof_error::truncated;
2066 StringRef TargetName(Names[TargetIdx]);
2067
2068 uint64_t TargetCount;
2069 if (!GcovBuffer.readInt64(Val&: TargetCount))
2070 return sampleprof_error::truncated;
2071
2072 if (Update)
2073 FProfile->addCalledTargetSamples(LineOffset, Discriminator,
2074 Func: FunctionId(TargetName), Num: TargetCount);
2075 }
2076 }
2077
2078 // Process all the inlined callers into the current function. These
2079 // are all the callsites that were inlined into this function.
2080 for (uint32_t I = 0; I < NumCallsites; I++) {
2081 // The offset is encoded as:
2082 // high 16 bits: line offset to the start of the function.
2083 // low 16 bits: discriminator.
2084 uint32_t Offset;
2085 if (!GcovBuffer.readInt(Val&: Offset))
2086 return sampleprof_error::truncated;
2087 InlineCallStack NewStack;
2088 NewStack.push_back(Elt: FProfile);
2089 llvm::append_range(C&: NewStack, R: InlineStack);
2090 if (std::error_code EC = readOneFunctionProfile(InlineStack: NewStack, Update, Offset))
2091 return EC;
2092 }
2093
2094 return sampleprof_error::success;
2095}
2096
2097/// Read a GCC AutoFDO profile.
2098///
2099/// This format is generated by the Linux Perf conversion tool at
2100/// https://github.com/google/autofdo.
2101std::error_code SampleProfileReaderGCC::readImpl() {
2102 assert(!ProfileIsFSDisciminator && "Gcc profiles not support FSDisciminator");
2103 // Read the string table.
2104 if (std::error_code EC = readNameTable())
2105 return EC;
2106
2107 // Read the source profile.
2108 if (std::error_code EC = readFunctionProfiles())
2109 return EC;
2110
2111 return sampleprof_error::success;
2112}
2113
2114bool SampleProfileReaderGCC::hasFormat(const MemoryBuffer &Buffer) {
2115 StringRef Magic(Buffer.getBufferStart());
2116 return Magic == "adcg*704";
2117}
2118
2119void SampleProfileReaderItaniumRemapper::applyRemapping(LLVMContext &Ctx) {
2120 // If the reader uses MD5 to represent string, we can't remap it because
2121 // we don't know what the original function names were.
2122 if (Reader.useMD5()) {
2123 Ctx.diagnose(DI: DiagnosticInfoSampleProfile(
2124 Reader.getBuffer()->getBufferIdentifier(),
2125 "Profile data remapping cannot be applied to profile data "
2126 "using MD5 names (original mangled names are not available).",
2127 DS_Warning));
2128 return;
2129 }
2130
2131 // CSSPGO-TODO: Remapper is not yet supported.
2132 // We will need to remap the entire context string.
2133 assert(Remappings && "should be initialized while creating remapper");
2134 for (auto &Sample : Reader.getProfiles()) {
2135 DenseSet<FunctionId> NamesInSample;
2136 Sample.second.findAllNames(NameSet&: NamesInSample);
2137 for (auto &Name : NamesInSample) {
2138 StringRef NameStr = Name.stringRef();
2139 if (auto Key = Remappings->insert(FunctionName: NameStr))
2140 NameMap.insert(KV: {Key, NameStr});
2141 }
2142 }
2143
2144 RemappingApplied = true;
2145}
2146
2147std::optional<StringRef>
2148SampleProfileReaderItaniumRemapper::lookUpNameInProfile(StringRef Fname) {
2149 if (auto Key = Remappings->lookup(FunctionName: Fname)) {
2150 StringRef Result = NameMap.lookup(Val: Key);
2151 if (!Result.empty())
2152 return Result;
2153 }
2154 return std::nullopt;
2155}
2156
2157/// Prepare a memory buffer for the contents of \p Filename.
2158///
2159/// \returns an error code indicating the status of the buffer.
2160static ErrorOr<std::unique_ptr<MemoryBuffer>>
2161setupMemoryBuffer(const Twine &Filename, vfs::FileSystem &FS) {
2162 auto BufferOrErr = Filename.str() == "-" ? MemoryBuffer::getSTDIN()
2163 : FS.getBufferForFile(Name: Filename);
2164 if (std::error_code EC = BufferOrErr.getError())
2165 return EC;
2166 auto Buffer = std::move(BufferOrErr.get());
2167
2168 return std::move(Buffer);
2169}
2170
2171/// Create a sample profile reader based on the format of the input file.
2172///
2173/// \param Filename The file to open.
2174///
2175/// \param C The LLVM context to use to emit diagnostics.
2176///
2177/// \param P The FSDiscriminatorPass.
2178///
2179/// \param RemapFilename The file used for profile remapping.
2180///
2181/// \returns an error code indicating the status of the created reader.
2182ErrorOr<std::unique_ptr<SampleProfileReader>>
2183SampleProfileReader::create(StringRef Filename, LLVMContext &C,
2184 vfs::FileSystem &FS, FSDiscriminatorPass P,
2185 StringRef RemapFilename) {
2186 auto BufferOrError = setupMemoryBuffer(Filename, FS);
2187 if (std::error_code EC = BufferOrError.getError())
2188 return EC;
2189 return create(B&: BufferOrError.get(), C, FS, P, RemapFilename);
2190}
2191
2192/// Create a sample profile remapper from the given input, to remap the
2193/// function names in the given profile data.
2194///
2195/// \param Filename The file to open.
2196///
2197/// \param Reader The profile reader the remapper is going to be applied to.
2198///
2199/// \param C The LLVM context to use to emit diagnostics.
2200///
2201/// \returns an error code indicating the status of the created reader.
2202ErrorOr<std::unique_ptr<SampleProfileReaderItaniumRemapper>>
2203SampleProfileReaderItaniumRemapper::create(StringRef Filename,
2204 vfs::FileSystem &FS,
2205 SampleProfileReader &Reader,
2206 LLVMContext &C) {
2207 auto BufferOrError = setupMemoryBuffer(Filename, FS);
2208 if (std::error_code EC = BufferOrError.getError())
2209 return EC;
2210 return create(B&: BufferOrError.get(), Reader, C);
2211}
2212
2213/// Create a sample profile remapper from the given input, to remap the
2214/// function names in the given profile data.
2215///
2216/// \param B The memory buffer to create the reader from (assumes ownership).
2217///
2218/// \param C The LLVM context to use to emit diagnostics.
2219///
2220/// \param Reader The profile reader the remapper is going to be applied to.
2221///
2222/// \returns an error code indicating the status of the created reader.
2223ErrorOr<std::unique_ptr<SampleProfileReaderItaniumRemapper>>
2224SampleProfileReaderItaniumRemapper::create(std::unique_ptr<MemoryBuffer> &B,
2225 SampleProfileReader &Reader,
2226 LLVMContext &C) {
2227 auto Remappings = std::make_unique<SymbolRemappingReader>();
2228 if (Error E = Remappings->read(B&: *B)) {
2229 handleAllErrors(
2230 E: std::move(E), Handlers: [&](const SymbolRemappingParseError &ParseError) {
2231 C.diagnose(DI: DiagnosticInfoSampleProfile(B->getBufferIdentifier(),
2232 ParseError.getLineNum(),
2233 ParseError.getMessage()));
2234 });
2235 return sampleprof_error::malformed;
2236 }
2237
2238 return std::make_unique<SampleProfileReaderItaniumRemapper>(
2239 args: std::move(B), args: std::move(Remappings), args&: Reader);
2240}
2241
2242/// Create a sample profile reader based on the format of the input data.
2243///
2244/// \param B The memory buffer to create the reader from (assumes ownership).
2245///
2246/// \param C The LLVM context to use to emit diagnostics.
2247///
2248/// \param P The FSDiscriminatorPass.
2249///
2250/// \param RemapFilename The file used for profile remapping.
2251///
2252/// \returns an error code indicating the status of the created reader.
2253ErrorOr<std::unique_ptr<SampleProfileReader>>
2254SampleProfileReader::create(std::unique_ptr<MemoryBuffer> &B, LLVMContext &C,
2255 vfs::FileSystem &FS, FSDiscriminatorPass P,
2256 StringRef RemapFilename) {
2257 std::unique_ptr<SampleProfileReader> Reader;
2258 if (SampleProfileReaderRawBinary::hasFormat(Buffer: *B))
2259 Reader.reset(p: new SampleProfileReaderRawBinary(std::move(B), C));
2260 else if (SampleProfileReaderExtBinary::hasFormat(Buffer: *B))
2261 Reader.reset(p: new SampleProfileReaderExtBinary(std::move(B), C));
2262 else if (SampleProfileReaderGCC::hasFormat(Buffer: *B))
2263 Reader.reset(p: new SampleProfileReaderGCC(std::move(B), C));
2264 else if (SampleProfileReaderText::hasFormat(Buffer: *B))
2265 Reader.reset(p: new SampleProfileReaderText(std::move(B), C));
2266 else
2267 return sampleprof_error::unrecognized_format;
2268
2269 if (!RemapFilename.empty()) {
2270 auto ReaderOrErr = SampleProfileReaderItaniumRemapper::create(
2271 Filename: RemapFilename, FS, Reader&: *Reader, C);
2272 if (std::error_code EC = ReaderOrErr.getError()) {
2273 std::string Msg = "Could not create remapper: " + EC.message();
2274 C.diagnose(DI: DiagnosticInfoSampleProfile(RemapFilename, Msg));
2275 return EC;
2276 }
2277 Reader->Remapper = std::move(ReaderOrErr.get());
2278 }
2279
2280 if (std::error_code EC = Reader->readHeader()) {
2281 return EC;
2282 }
2283
2284 Reader->setDiscriminatorMaskedBitFrom(P);
2285
2286 return std::move(Reader);
2287}
2288
2289// For text and GCC file formats, we compute the summary after reading the
2290// profile. Binary format has the profile summary in its header.
2291void SampleProfileReader::computeSummary() {
2292 SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs);
2293 Summary = Builder.computeSummaryForProfiles(Profiles);
2294}
2295