1//===- InstrProfReader.cpp - Instrumented profiling reader ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains support for reading profiling data for clang's
10// instrumentation based PGO and coverage.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/ProfileData/InstrProfReader.h"
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/ADT/DenseMap.h"
17#include "llvm/ADT/StringExtras.h"
18#include "llvm/ADT/StringRef.h"
19#include "llvm/IR/ProfileSummary.h"
20#include "llvm/ProfileData/InstrProf.h"
21// #include "llvm/ProfileData/MemProf.h"
22#include "llvm/ProfileData/MemProfRadixTree.h"
23#include "llvm/ProfileData/ProfileCommon.h"
24#include "llvm/ProfileData/SymbolRemappingReader.h"
25#include "llvm/Support/Endian.h"
26#include "llvm/Support/Error.h"
27#include "llvm/Support/ErrorOr.h"
28#include "llvm/Support/FormatVariadic.h"
29#include "llvm/Support/MemoryBuffer.h"
30#include "llvm/Support/VirtualFileSystem.h"
31#include <algorithm>
32#include <cstddef>
33#include <cstdint>
34#include <limits>
35#include <memory>
36#include <optional>
37#include <system_error>
38#include <utility>
39#include <vector>
40
41using namespace llvm;
42
43// Extracts the variant information from the top 32 bits in the version and
44// returns an enum specifying the variants present.
45static InstrProfKind getProfileKindFromVersion(uint64_t Version) {
46 InstrProfKind ProfileKind = InstrProfKind::Unknown;
47 if (Version & VARIANT_MASK_IR_PROF) {
48 ProfileKind |= InstrProfKind::IRInstrumentation;
49 }
50 if (Version & VARIANT_MASK_CSIR_PROF) {
51 ProfileKind |= InstrProfKind::ContextSensitive;
52 }
53 if (Version & VARIANT_MASK_INSTR_ENTRY) {
54 ProfileKind |= InstrProfKind::FunctionEntryInstrumentation;
55 }
56 if (Version & VARIANT_MASK_INSTR_LOOP_ENTRIES) {
57 ProfileKind |= InstrProfKind::LoopEntriesInstrumentation;
58 }
59 if (Version & VARIANT_MASK_BYTE_COVERAGE) {
60 ProfileKind |= InstrProfKind::SingleByteCoverage;
61 }
62 if (Version & VARIANT_MASK_FUNCTION_ENTRY_ONLY) {
63 ProfileKind |= InstrProfKind::FunctionEntryOnly;
64 }
65 if (Version & VARIANT_MASK_MEMPROF) {
66 ProfileKind |= InstrProfKind::MemProf;
67 }
68 if (Version & VARIANT_MASK_TEMPORAL_PROF) {
69 ProfileKind |= InstrProfKind::TemporalProfile;
70 }
71 return ProfileKind;
72}
73
74static Expected<std::unique_ptr<MemoryBuffer>>
75setupMemoryBuffer(const Twine &Filename, vfs::FileSystem &FS) {
76 auto BufferOrErr = Filename.str() == "-" ? MemoryBuffer::getSTDIN()
77 : FS.getBufferForFile(Name: Filename);
78 if (std::error_code EC = BufferOrErr.getError())
79 return errorCodeToError(EC);
80 return std::move(BufferOrErr.get());
81}
82
83static Error initializeReader(InstrProfReader &Reader) {
84 return Reader.readHeader();
85}
86
87/// Read a list of binary ids from a profile that consist of
88/// a. uint64_t binary id length
89/// b. uint8_t binary id data
90/// c. uint8_t padding (if necessary)
91/// This function is shared between raw and indexed profiles.
92/// Raw profiles are in host-endian format, and indexed profiles are in
93/// little-endian format. So, this function takes an argument indicating the
94/// associated endian format to read the binary ids correctly.
95static Error
96readBinaryIdsInternal(const MemoryBuffer &DataBuffer,
97 ArrayRef<uint8_t> BinaryIdsBuffer,
98 std::vector<llvm::object::BuildID> &BinaryIds,
99 const llvm::endianness Endian) {
100 using namespace support;
101
102 const uint64_t BinaryIdsSize = BinaryIdsBuffer.size();
103 const uint8_t *BinaryIdsStart = BinaryIdsBuffer.data();
104
105 if (BinaryIdsSize == 0)
106 return Error::success();
107
108 const uint8_t *BI = BinaryIdsStart;
109 const uint8_t *BIEnd = BinaryIdsStart + BinaryIdsSize;
110 const uint8_t *End =
111 reinterpret_cast<const uint8_t *>(DataBuffer.getBufferEnd());
112
113 while (BI < BIEnd) {
114 size_t Remaining = BIEnd - BI;
115 // There should be enough left to read the binary id length.
116 if (Remaining < sizeof(uint64_t))
117 return make_error<InstrProfError>(
118 Args: instrprof_error::malformed,
119 Args: "not enough data to read binary id length");
120
121 uint64_t BILen = endian::readNext<uint64_t>(memory&: BI, endian: Endian);
122 if (BILen == 0)
123 return make_error<InstrProfError>(Args: instrprof_error::malformed,
124 Args: "binary id length is 0");
125
126 Remaining = BIEnd - BI;
127 // There should be enough left to read the binary id data.
128 if (Remaining < alignToPowerOf2(Value: BILen, Align: sizeof(uint64_t)))
129 return make_error<InstrProfError>(
130 Args: instrprof_error::malformed, Args: "not enough data to read binary id data");
131
132 // Add binary id to the binary ids list.
133 BinaryIds.push_back(x: object::BuildID(BI, BI + BILen));
134
135 // Increment by binary id data length, which aligned to the size of uint64.
136 BI += alignToPowerOf2(Value: BILen, Align: sizeof(uint64_t));
137 if (BI > End)
138 return make_error<InstrProfError>(
139 Args: instrprof_error::malformed,
140 Args: "binary id section is greater than buffer size");
141 }
142
143 return Error::success();
144}
145
146static void printBinaryIdsInternal(raw_ostream &OS,
147 ArrayRef<llvm::object::BuildID> BinaryIds) {
148 OS << "Binary IDs: \n";
149 for (const auto &BI : BinaryIds) {
150 for (auto I : BI)
151 OS << format(Fmt: "%02x", Vals: I);
152 OS << "\n";
153 }
154}
155
156Expected<std::unique_ptr<InstrProfReader>> InstrProfReader::create(
157 const Twine &Path, vfs::FileSystem &FS,
158 const InstrProfCorrelator *Correlator,
159 const object::BuildIDFetcher *BIDFetcher,
160 const InstrProfCorrelator::ProfCorrelatorKind BIDFetcherCorrelatorKind,
161 std::function<void(Error)> Warn) {
162 // Set up the buffer to read.
163 auto BufferOrError = setupMemoryBuffer(Filename: Path, FS);
164 if (Error E = BufferOrError.takeError())
165 return std::move(E);
166 return InstrProfReader::create(Buffer: std::move(BufferOrError.get()), Correlator,
167 BIDFetcher, BIDFetcherCorrelatorKind, Warn);
168}
169
170Expected<std::unique_ptr<InstrProfReader>> InstrProfReader::create(
171 std::unique_ptr<MemoryBuffer> Buffer, const InstrProfCorrelator *Correlator,
172 const object::BuildIDFetcher *BIDFetcher,
173 const InstrProfCorrelator::ProfCorrelatorKind BIDFetcherCorrelatorKind,
174 std::function<void(Error)> Warn) {
175 if (Buffer->getBufferSize() == 0)
176 return make_error<InstrProfError>(Args: instrprof_error::empty_raw_profile);
177
178 std::unique_ptr<InstrProfReader> Result;
179 // Create the reader.
180 if (IndexedInstrProfReader::hasFormat(DataBuffer: *Buffer))
181 Result.reset(p: new IndexedInstrProfReader(std::move(Buffer)));
182 else if (RawInstrProfReader64::hasFormat(DataBuffer: *Buffer))
183 Result.reset(p: new RawInstrProfReader64(std::move(Buffer), Correlator,
184 BIDFetcher, BIDFetcherCorrelatorKind,
185 Warn));
186 else if (RawInstrProfReader32::hasFormat(DataBuffer: *Buffer))
187 Result.reset(p: new RawInstrProfReader32(std::move(Buffer), Correlator,
188 BIDFetcher, BIDFetcherCorrelatorKind,
189 Warn));
190 else if (TextInstrProfReader::hasFormat(Buffer: *Buffer))
191 Result.reset(p: new TextInstrProfReader(std::move(Buffer)));
192 else
193 return make_error<InstrProfError>(Args: instrprof_error::unrecognized_format);
194
195 // Initialize the reader and return the result.
196 if (Error E = initializeReader(Reader&: *Result))
197 return std::move(E);
198
199 return std::move(Result);
200}
201
202Expected<std::unique_ptr<IndexedInstrProfReader>>
203IndexedInstrProfReader::create(const Twine &Path, vfs::FileSystem &FS,
204 const Twine &RemappingPath) {
205 // Set up the buffer to read.
206 auto BufferOrError = setupMemoryBuffer(Filename: Path, FS);
207 if (Error E = BufferOrError.takeError())
208 return std::move(E);
209
210 // Set up the remapping buffer if requested.
211 std::unique_ptr<MemoryBuffer> RemappingBuffer;
212 std::string RemappingPathStr = RemappingPath.str();
213 if (!RemappingPathStr.empty()) {
214 auto RemappingBufferOrError = setupMemoryBuffer(Filename: RemappingPathStr, FS);
215 if (Error E = RemappingBufferOrError.takeError())
216 return std::move(E);
217 RemappingBuffer = std::move(RemappingBufferOrError.get());
218 }
219
220 return IndexedInstrProfReader::create(Buffer: std::move(BufferOrError.get()),
221 RemappingBuffer: std::move(RemappingBuffer));
222}
223
224Expected<std::unique_ptr<IndexedInstrProfReader>>
225IndexedInstrProfReader::create(std::unique_ptr<MemoryBuffer> Buffer,
226 std::unique_ptr<MemoryBuffer> RemappingBuffer) {
227 // Create the reader.
228 if (!IndexedInstrProfReader::hasFormat(DataBuffer: *Buffer))
229 return make_error<InstrProfError>(Args: instrprof_error::bad_magic);
230 auto Result = std::make_unique<IndexedInstrProfReader>(
231 args: std::move(Buffer), args: std::move(RemappingBuffer));
232
233 // Initialize the reader and return the result.
234 if (Error E = initializeReader(Reader&: *Result))
235 return std::move(E);
236
237 return std::move(Result);
238}
239
240bool TextInstrProfReader::hasFormat(const MemoryBuffer &Buffer) {
241 // Verify that this really looks like plain ASCII text by checking a
242 // 'reasonable' number of characters (up to profile magic size).
243 size_t count = std::min(a: Buffer.getBufferSize(), b: sizeof(uint64_t));
244 StringRef buffer = Buffer.getBufferStart();
245 return count == 0 ||
246 std::all_of(first: buffer.begin(), last: buffer.begin() + count,
247 pred: [](char c) { return isPrint(C: c) || isSpace(C: c); });
248}
249
250// Read the profile variant flag from the header: ":FE" means this is a FE
251// generated profile. ":IR" means this is an IR level profile. Other strings
252// with a leading ':' will be reported an error format.
253Error TextInstrProfReader::readHeader() {
254 Symtab.reset(p: new InstrProfSymtab());
255
256 while (Line->starts_with(Prefix: ":")) {
257 StringRef Str = Line->substr(Start: 1);
258 if (Str.equals_insensitive(RHS: "ir"))
259 ProfileKind |= InstrProfKind::IRInstrumentation;
260 else if (Str.equals_insensitive(RHS: "fe"))
261 ProfileKind |= InstrProfKind::FrontendInstrumentation;
262 else if (Str.equals_insensitive(RHS: "csir")) {
263 ProfileKind |= InstrProfKind::IRInstrumentation;
264 ProfileKind |= InstrProfKind::ContextSensitive;
265 } else if (Str.equals_insensitive(RHS: "entry_first"))
266 ProfileKind |= InstrProfKind::FunctionEntryInstrumentation;
267 else if (Str.equals_insensitive(RHS: "not_entry_first"))
268 ProfileKind &= ~InstrProfKind::FunctionEntryInstrumentation;
269 else if (Str.equals_insensitive(RHS: "instrument_loop_entries"))
270 ProfileKind |= InstrProfKind::LoopEntriesInstrumentation;
271 else if (Str.equals_insensitive(RHS: "single_byte_coverage"))
272 ProfileKind |= InstrProfKind::SingleByteCoverage;
273 else if (Str.equals_insensitive(RHS: "temporal_prof_traces")) {
274 ProfileKind |= InstrProfKind::TemporalProfile;
275 if (auto Err = readTemporalProfTraceData())
276 return error(E: std::move(Err));
277 } else
278 return error(Err: instrprof_error::bad_header);
279 ++Line;
280 }
281 return success();
282}
283
284/// Temporal profile trace data is stored in the header immediately after
285/// ":temporal_prof_traces". The first integer is the number of traces, the
286/// second integer is the stream size, then the following lines are the actual
287/// traces which consist of a weight and a comma separated list of function
288/// names.
289Error TextInstrProfReader::readTemporalProfTraceData() {
290 if ((++Line).is_at_end())
291 return error(Err: instrprof_error::eof);
292
293 uint32_t NumTraces;
294 if (Line->getAsInteger(Radix: 0, Result&: NumTraces))
295 return error(Err: instrprof_error::malformed);
296
297 if ((++Line).is_at_end())
298 return error(Err: instrprof_error::eof);
299
300 if (Line->getAsInteger(Radix: 0, Result&: TemporalProfTraceStreamSize))
301 return error(Err: instrprof_error::malformed);
302
303 for (uint32_t i = 0; i < NumTraces; i++) {
304 if ((++Line).is_at_end())
305 return error(Err: instrprof_error::eof);
306
307 TemporalProfTraceTy Trace;
308 if (Line->getAsInteger(Radix: 0, Result&: Trace.Weight))
309 return error(Err: instrprof_error::malformed);
310
311 if ((++Line).is_at_end())
312 return error(Err: instrprof_error::eof);
313
314 SmallVector<StringRef> FuncNames;
315 Line->split(A&: FuncNames, Separator: ",", /*MaxSplit=*/-1, /*KeepEmpty=*/false);
316 for (auto &FuncName : FuncNames)
317 Trace.FunctionNameRefs.push_back(
318 x: IndexedInstrProf::ComputeHash(K: FuncName.trim()));
319 TemporalProfTraces.push_back(Elt: std::move(Trace));
320 }
321 return success();
322}
323
324Error
325TextInstrProfReader::readValueProfileData(InstrProfRecord &Record) {
326
327#define CHECK_LINE_END(Line) \
328 if (Line.is_at_end()) \
329 return error(instrprof_error::truncated);
330#define READ_NUM(Str, Dst) \
331 if ((Str).getAsInteger(10, (Dst))) \
332 return error(instrprof_error::malformed);
333#define VP_READ_ADVANCE(Val) \
334 CHECK_LINE_END(Line); \
335 uint32_t Val; \
336 READ_NUM((*Line), (Val)); \
337 Line++;
338
339 if (Line.is_at_end())
340 return success();
341
342 uint32_t NumValueKinds;
343 if (Line->getAsInteger(Radix: 10, Result&: NumValueKinds)) {
344 // No value profile data
345 return success();
346 }
347 if (NumValueKinds == 0 || NumValueKinds > IPVK_Last + 1)
348 return error(Err: instrprof_error::malformed,
349 ErrMsg: "number of value kinds is invalid");
350 Line++;
351
352 for (uint32_t VK = 0; VK < NumValueKinds; VK++) {
353 VP_READ_ADVANCE(ValueKind);
354 if (ValueKind > IPVK_Last)
355 return error(Err: instrprof_error::malformed, ErrMsg: "value kind is invalid");
356 ;
357 VP_READ_ADVANCE(NumValueSites);
358 if (!NumValueSites)
359 continue;
360
361 Record.reserveSites(ValueKind: VK, NumValueSites);
362 for (uint32_t S = 0; S < NumValueSites; S++) {
363 VP_READ_ADVANCE(NumValueData);
364
365 std::vector<InstrProfValueData> CurrentValues;
366 for (uint32_t V = 0; V < NumValueData; V++) {
367 CHECK_LINE_END(Line);
368 std::pair<StringRef, StringRef> VD = Line->rsplit(Separator: ':');
369 uint64_t TakenCount, Value;
370 if (ValueKind == IPVK_IndirectCallTarget) {
371 if (InstrProfSymtab::isExternalSymbol(Symbol: VD.first)) {
372 Value = 0;
373 } else {
374 if (Error E = Symtab->addFuncName(FuncName: VD.first))
375 return E;
376 Value = IndexedInstrProf::ComputeHash(K: VD.first);
377 }
378 } else if (ValueKind == IPVK_VTableTarget) {
379 if (InstrProfSymtab::isExternalSymbol(Symbol: VD.first))
380 Value = 0;
381 else {
382 if (Error E = Symtab->addVTableName(VTableName: VD.first))
383 return E;
384 Value = IndexedInstrProf::ComputeHash(K: VD.first);
385 }
386 } else {
387 READ_NUM(VD.first, Value);
388 }
389 READ_NUM(VD.second, TakenCount);
390 CurrentValues.push_back(x: {.Value: Value, .Count: TakenCount});
391 Line++;
392 }
393 assert(CurrentValues.size() == NumValueData);
394 Record.addValueData(ValueKind, Site: S, VData: CurrentValues, SymTab: nullptr);
395 }
396 }
397 return success();
398
399#undef CHECK_LINE_END
400#undef READ_NUM
401#undef VP_READ_ADVANCE
402}
403
404Error TextInstrProfReader::readNextRecord(NamedInstrProfRecord &Record) {
405 // Skip empty lines and comments.
406 while (!Line.is_at_end() && (Line->empty() || Line->starts_with(Prefix: "#")))
407 ++Line;
408 // If we hit EOF while looking for a name, we're done.
409 if (Line.is_at_end()) {
410 return error(Err: instrprof_error::eof);
411 }
412
413 // Read the function name.
414 Record.Name = *Line++;
415 if (Error E = Symtab->addFuncName(FuncName: Record.Name))
416 return error(E: std::move(E));
417
418 // Read the function hash.
419 if (Line.is_at_end())
420 return error(Err: instrprof_error::truncated);
421 if ((Line++)->getAsInteger(Radix: 0, Result&: Record.Hash))
422 return error(Err: instrprof_error::malformed,
423 ErrMsg: "function hash is not a valid integer");
424
425 // Read the number of counters.
426 uint64_t NumCounters;
427 if (Line.is_at_end())
428 return error(Err: instrprof_error::truncated);
429 if ((Line++)->getAsInteger(Radix: 10, Result&: NumCounters))
430 return error(Err: instrprof_error::malformed,
431 ErrMsg: "number of counters is not a valid integer");
432 if (NumCounters == 0)
433 return error(Err: instrprof_error::malformed, ErrMsg: "number of counters is zero");
434
435 // Read each counter and fill our internal storage with the values.
436 Record.Clear();
437 Record.Counts.reserve(n: NumCounters);
438 for (uint64_t I = 0; I < NumCounters; ++I) {
439 if (Line.is_at_end())
440 return error(Err: instrprof_error::truncated);
441 uint64_t Count;
442 if ((Line++)->getAsInteger(Radix: 10, Result&: Count))
443 return error(Err: instrprof_error::malformed, ErrMsg: "count is invalid");
444 Record.Counts.push_back(x: Count);
445 }
446
447 // Bitmap byte information is indicated by '$' followed by an integer. Only
448 // treat numeric-looking lines as bitmap records so function names such as
449 // Swift manglings beginning with "$s" remain unambiguous.
450 StringRef BitmapSize =
451 Line->starts_with(Prefix: "$") ? Line->drop_front(N: 1).trim() : StringRef();
452 if (!BitmapSize.empty() && isDigit(C: BitmapSize.front())) {
453 Record.BitmapBytes.clear();
454 // Read the number of bitmap bytes.
455 uint64_t NumBitmapBytes;
456 if ((Line++)->drop_front(N: 1).trim().getAsInteger(Radix: 0, Result&: NumBitmapBytes))
457 return error(Err: instrprof_error::malformed,
458 ErrMsg: "number of bitmap bytes is not a valid integer");
459 if (NumBitmapBytes != 0) {
460 // Read each bitmap and fill our internal storage with the values.
461 Record.BitmapBytes.reserve(n: NumBitmapBytes);
462 for (uint64_t I = 0; I < NumBitmapBytes; ++I) {
463 if (Line.is_at_end())
464 return error(Err: instrprof_error::truncated);
465 uint8_t BitmapByte;
466 if ((Line++)->getAsInteger(Radix: 0, Result&: BitmapByte))
467 return error(Err: instrprof_error::malformed,
468 ErrMsg: "bitmap byte is not a valid integer");
469 Record.BitmapBytes.push_back(x: BitmapByte);
470 }
471 }
472 }
473
474 // Check if value profile data exists and read it if so.
475 if (Error E = readValueProfileData(Record))
476 return error(E: std::move(E));
477
478 return success();
479}
480
481template <class IntPtrT>
482InstrProfKind RawInstrProfReader<IntPtrT>::getProfileKind() const {
483 return getProfileKindFromVersion(Version);
484}
485
486template <class IntPtrT>
487SmallVector<TemporalProfTraceTy> &
488RawInstrProfReader<IntPtrT>::getTemporalProfTraces(
489 std::optional<uint64_t> Weight) {
490 if (TemporalProfTimestamps.empty()) {
491 assert(TemporalProfTraces.empty());
492 return TemporalProfTraces;
493 }
494 // Sort functions by their timestamps to build the trace.
495 std::sort(first: TemporalProfTimestamps.begin(), last: TemporalProfTimestamps.end());
496 TemporalProfTraceTy Trace;
497 if (Weight)
498 Trace.Weight = *Weight;
499 for (auto &[TimestampValue, NameRef] : TemporalProfTimestamps)
500 Trace.FunctionNameRefs.push_back(x: NameRef);
501 TemporalProfTraces = {std::move(Trace)};
502 return TemporalProfTraces;
503}
504
505template <class IntPtrT>
506bool RawInstrProfReader<IntPtrT>::hasFormat(const MemoryBuffer &DataBuffer) {
507 if (DataBuffer.getBufferSize() < sizeof(uint64_t))
508 return false;
509 uint64_t Magic =
510 *reinterpret_cast<const uint64_t *>(DataBuffer.getBufferStart());
511 return RawInstrProf::getMagic<IntPtrT>() == Magic ||
512 llvm::byteswap(RawInstrProf::getMagic<IntPtrT>()) == Magic;
513}
514
515template <class IntPtrT>
516Error RawInstrProfReader<IntPtrT>::readHeader() {
517 if (!hasFormat(DataBuffer: *DataBuffer))
518 return error(instrprof_error::bad_magic);
519 if (DataBuffer->getBufferSize() < sizeof(RawInstrProf::Header))
520 return error(instrprof_error::bad_header,
521 std::string("profile file header is truncated"));
522 auto *Header = reinterpret_cast<const RawInstrProf::Header *>(
523 DataBuffer->getBufferStart());
524 ShouldSwapBytes = Header->Magic != RawInstrProf::getMagic<IntPtrT>();
525 return readHeader(*Header);
526}
527
528template <class IntPtrT>
529Error RawInstrProfReader<IntPtrT>::readNextHeader(const char *CurrentPos) {
530 const char *End = DataBuffer->getBufferEnd();
531 // Skip zero padding between profiles.
532 while (CurrentPos != End && *CurrentPos == 0)
533 ++CurrentPos;
534 // If there's nothing left, we're done.
535 if (CurrentPos == End)
536 return make_error<InstrProfError>(Args: instrprof_error::eof);
537 // If there isn't enough space for another header, this is probably just
538 // garbage at the end of the file.
539 if (CurrentPos + sizeof(RawInstrProf::Header) > End)
540 return make_error<InstrProfError>(Args: instrprof_error::malformed,
541 Args: "not enough space for another header");
542 // The writer ensures each profile is padded to start at an aligned address.
543 if (reinterpret_cast<size_t>(CurrentPos) % alignof(uint64_t))
544 return make_error<InstrProfError>(Args: instrprof_error::malformed,
545 Args: "insufficient padding");
546 // The magic should have the same byte order as in the previous header.
547 uint64_t Magic = *reinterpret_cast<const uint64_t *>(CurrentPos);
548 if (Magic != swap(RawInstrProf::getMagic<IntPtrT>()))
549 return make_error<InstrProfError>(Args: instrprof_error::bad_magic);
550
551 // There's another profile to read, so we need to process the header.
552 auto *Header = reinterpret_cast<const RawInstrProf::Header *>(CurrentPos);
553 return readHeader(*Header);
554}
555
556template <class IntPtrT>
557Error RawInstrProfReader<IntPtrT>::createSymtab(InstrProfSymtab &Symtab) {
558 if (Error E = Symtab.create(FuncNameStrings: StringRef(NamesStart, NamesEnd - NamesStart),
559 VTableNameStrings: StringRef(VNamesStart, VNamesEnd - VNamesStart)))
560 return error(std::move(E));
561 for (const RawInstrProf::ProfileData<IntPtrT> *I = Data; I != DataEnd; ++I) {
562 const IntPtrT FPtr = swap(I->FunctionPointer);
563 if (!FPtr)
564 continue;
565 Symtab.mapAddress(Addr: FPtr, MD5Val: swap(I->NameRef));
566 }
567
568 if (VTableBegin != nullptr && VTableEnd != nullptr) {
569 for (const RawInstrProf::VTableProfileData<IntPtrT> *I = VTableBegin;
570 I != VTableEnd; ++I) {
571 const IntPtrT VPtr = swap(I->VTablePointer);
572 if (!VPtr)
573 continue;
574 // Map both begin and end address to the name hash, since the instrumented
575 // address could be somewhere in the middle.
576 // VPtr is of type uint32_t or uint64_t so 'VPtr + I->VTableSize' marks
577 // the end of vtable address.
578 Symtab.mapVTableAddress(StartAddr: VPtr, EndAddr: VPtr + swap(I->VTableSize),
579 MD5Val: swap(I->VTableNameHash));
580 }
581 }
582 return success();
583}
584
585template <class IntPtrT>
586Error RawInstrProfReader<IntPtrT>::readHeader(
587 const RawInstrProf::Header &Header) {
588 Version = swap(Header.Version);
589 if (GET_VERSION(Version) != RawInstrProf::Version)
590 return error(instrprof_error::raw_profile_version_mismatch,
591 ("Profile uses raw profile format version = " +
592 Twine(GET_VERSION(Version)) +
593 "; expected version = " + Twine(RawInstrProf::Version) +
594 "\nPLEASE update this tool to version in the raw profile, or "
595 "regenerate raw profile with expected version.")
596 .str());
597 uint64_t BinaryIdSize = swap(Header.BinaryIdsSize);
598 // Binary id start just after the header if exists.
599 const uint8_t *BinaryIdStart =
600 reinterpret_cast<const uint8_t *>(&Header) + sizeof(RawInstrProf::Header);
601 const uint8_t *BinaryIdEnd = BinaryIdStart + BinaryIdSize;
602 const uint8_t *BufferEnd = (const uint8_t *)DataBuffer->getBufferEnd();
603 if (BinaryIdSize % sizeof(uint64_t))
604 return error(
605 instrprof_error::bad_header,
606 ("BinaryIdSize (" + Twine(BinaryIdSize) + ") is not a multiple of 8")
607 .str());
608 if (BinaryIdEnd > BufferEnd)
609 return error(instrprof_error::header_size_mismatch,
610 ("Header.BinaryIdSize = " + Twine(BinaryIdSize) + " bytes; " +
611 Twine(BufferEnd - BinaryIdStart) + " bytes available")
612 .str());
613
614 ArrayRef<uint8_t> BinaryIdsBuffer(BinaryIdStart, BinaryIdSize);
615 if (!BinaryIdsBuffer.empty()) {
616 if (Error Err = readBinaryIdsInternal(*DataBuffer, BinaryIdsBuffer,
617 BinaryIds, getDataEndianness()))
618 return Err;
619 }
620
621 CountersDelta = swap(Header.CountersDelta);
622 BitmapDelta = swap(Header.BitmapDelta);
623 UniformCountersDelta = swap(Header.UniformCountersDelta);
624 NamesDelta = swap(Header.NamesDelta);
625 auto NumData = swap(Header.NumData);
626 auto PaddingBytesBeforeCounters = swap(Header.PaddingBytesBeforeCounters);
627 auto CountersSize = swap(Header.NumCounters) * getCounterTypeSize();
628 auto PaddingBytesAfterCounters = swap(Header.PaddingBytesAfterCounters);
629 auto NumBitmapBytes = swap(Header.NumBitmapBytes);
630 auto PaddingBytesAfterBitmapBytes = swap(Header.PaddingBytesAfterBitmapBytes);
631 auto NumUniformCounters = swap(Header.NumUniformCounters);
632 auto PaddingBytesAfterUniformCounters =
633 swap(Header.PaddingBytesAfterUniformCounters);
634 auto NamesSize = swap(Header.NamesSize);
635 auto VTableNameSize = swap(Header.VNamesSize);
636 auto NumVTables = swap(Header.NumVTables);
637 ValueKindLast = swap(Header.ValueKindLast);
638
639 auto DataSize = NumData * sizeof(RawInstrProf::ProfileData<IntPtrT>);
640 auto PaddingBytesAfterNames = getNumPaddingBytes(SizeInBytes: NamesSize);
641 auto PaddingBytesAfterVTableNames = getNumPaddingBytes(SizeInBytes: VTableNameSize);
642
643 auto VTableSectionSize =
644 NumVTables * sizeof(RawInstrProf::VTableProfileData<IntPtrT>);
645 auto PaddingBytesAfterVTableProfData = getNumPaddingBytes(SizeInBytes: VTableSectionSize);
646 auto UniformCountersSectionSize = NumUniformCounters * sizeof(uint64_t);
647
648 // Profile data starts after profile header and binary ids if exist.
649 ptrdiff_t DataOffset = sizeof(RawInstrProf::Header) + BinaryIdSize;
650 ptrdiff_t CountersOffset = DataOffset + DataSize + PaddingBytesBeforeCounters;
651 ptrdiff_t BitmapOffset =
652 CountersOffset + CountersSize + PaddingBytesAfterCounters;
653 ptrdiff_t UniformCountersOffset =
654 BitmapOffset + NumBitmapBytes + PaddingBytesAfterBitmapBytes;
655 ptrdiff_t NamesOffset = UniformCountersOffset + UniformCountersSectionSize +
656 PaddingBytesAfterUniformCounters;
657 ptrdiff_t VTableProfDataOffset =
658 NamesOffset + NamesSize + PaddingBytesAfterNames;
659 ptrdiff_t VTableNameOffset = VTableProfDataOffset + VTableSectionSize +
660 PaddingBytesAfterVTableProfData;
661 ptrdiff_t ValueDataOffset =
662 VTableNameOffset + VTableNameSize + PaddingBytesAfterVTableNames;
663
664 auto *Start = reinterpret_cast<const char *>(&Header);
665 if (Start + ValueDataOffset > DataBuffer->getBufferEnd())
666 // clang-format off
667 return error(
668 instrprof_error::header_size_mismatch,
669 ("profile file size (" + Twine(DataBuffer->getBufferSize()) +
670 " bytes) smaller than expected (at least " + Twine(ValueDataOffset) +
671 " bytes: " +
672 Twine(sizeof(RawInstrProf::Header)) + "(Header) + " +
673 Twine(BinaryIdSize) + "(BinaryIdSize) + " +
674 Twine(DataSize) + "(DataSize) + " +
675 Twine(CountersSize) + "(CountersSize) + " +
676 Twine(NumBitmapBytes) + "(NumBitmapBytes) + " +
677 Twine(UniformCountersSectionSize) + "(UniformCountersSectionSize) + " +
678 Twine(NamesSize) + "(NamesSize) + " +
679 Twine(VTableSectionSize) + "(VTableSectionSize) + " +
680 Twine(VTableNameSize) + "(VTableNameSize) + " +
681 Twine(PaddingBytesBeforeCounters + PaddingBytesAfterCounters +
682 PaddingBytesAfterBitmapBytes + PaddingBytesAfterUniformCounters +
683 PaddingBytesAfterNames + PaddingBytesAfterVTableProfData +
684 PaddingBytesAfterVTableNames) +
685 "(Padding))")
686 .str());
687 // clang-format on
688
689 if (BIDFetcher) {
690 std::vector<object::BuildID> BinaryIDs;
691 if (Error E = readBinaryIds(BinaryIds&: BinaryIDs))
692 return E;
693 if (auto E = InstrProfCorrelator::get(Filename: "", FileKind: BIDFetcherCorrelatorKind,
694 BIDFetcher, BIs: BinaryIDs)
695 .moveInto(Value&: BIDFetcherCorrelator)) {
696 return E;
697 }
698 if (auto Err = BIDFetcherCorrelator->correlateProfileData(MaxWarnings: 0))
699 return Err;
700 }
701
702 if (Correlator) {
703 // These sizes in the raw file are zero because we constructed them in the
704 // Correlator.
705 if (!(DataSize == 0 && NamesSize == 0 && CountersDelta == 0 &&
706 BitmapDelta == 0 && NamesDelta == 0))
707 return error(instrprof_error::unexpected_correlation_info);
708 Data = Correlator->getDataPointer();
709 DataEnd = Data + Correlator->getDataSize();
710 NamesStart = Correlator->getNamesPointer();
711 NamesEnd = NamesStart + Correlator->getNamesSize();
712 } else if (BIDFetcherCorrelator) {
713 InstrProfCorrelatorImpl<IntPtrT> *BIDFetcherCorrelatorImpl =
714 dyn_cast_or_null<InstrProfCorrelatorImpl<IntPtrT>>(
715 BIDFetcherCorrelator.get());
716 Data = BIDFetcherCorrelatorImpl->getDataPointer();
717 DataEnd = Data + BIDFetcherCorrelatorImpl->getDataSize();
718 NamesStart = BIDFetcherCorrelatorImpl->getNamesPointer();
719 NamesEnd = NamesStart + BIDFetcherCorrelatorImpl->getNamesSize();
720 } else {
721 Data = reinterpret_cast<const RawInstrProf::ProfileData<IntPtrT> *>(
722 Start + DataOffset);
723 DataEnd = Data + NumData;
724 VTableBegin =
725 reinterpret_cast<const RawInstrProf::VTableProfileData<IntPtrT> *>(
726 Start + VTableProfDataOffset);
727 VTableEnd = VTableBegin + NumVTables;
728 NamesStart = Start + NamesOffset;
729 NamesEnd = NamesStart + NamesSize;
730 VNamesStart = Start + VTableNameOffset;
731 VNamesEnd = VNamesStart + VTableNameSize;
732 }
733
734 CountersStart = Start + CountersOffset;
735 CountersEnd = CountersStart + CountersSize;
736 BitmapStart = Start + BitmapOffset;
737 BitmapEnd = BitmapStart + NumBitmapBytes;
738 UniformCountersStart = Start + UniformCountersOffset;
739 UniformCountersEnd = UniformCountersStart + UniformCountersSectionSize;
740 ValueDataStart = reinterpret_cast<const uint8_t *>(Start + ValueDataOffset);
741
742 std::unique_ptr<InstrProfSymtab> NewSymtab = std::make_unique<InstrProfSymtab>();
743 if (Error E = createSymtab(Symtab&: *NewSymtab))
744 return E;
745
746 Symtab = std::move(NewSymtab);
747 return success();
748}
749
750template <class IntPtrT>
751Error RawInstrProfReader<IntPtrT>::readName(NamedInstrProfRecord &Record) {
752 Record.Name = getName(NameRef: Data->NameRef);
753 return success();
754}
755
756template <class IntPtrT>
757Error RawInstrProfReader<IntPtrT>::readFuncHash(NamedInstrProfRecord &Record) {
758 Record.Hash = swap(Data->FuncHash);
759 return success();
760}
761
762template <class IntPtrT>
763Error RawInstrProfReader<IntPtrT>::readRawCounts(
764 InstrProfRecord &Record) {
765 uint32_t NumCounters = swap(Data->NumCounters);
766 if (NumCounters == 0)
767 return error(instrprof_error::malformed, "number of counters is zero");
768
769 ptrdiff_t CounterBaseOffset = swap(Data->CounterPtr) - CountersDelta;
770 if (CounterBaseOffset < 0)
771 return error(
772 instrprof_error::malformed,
773 ("counter offset " + Twine(CounterBaseOffset) + " is negative").str());
774
775 if (CounterBaseOffset >= CountersEnd - CountersStart)
776 return error(instrprof_error::malformed,
777 ("counter offset " + Twine(CounterBaseOffset) +
778 " is greater than the maximum counter offset " +
779 Twine(CountersEnd - CountersStart - 1))
780 .str());
781
782 uint64_t MaxNumCounters =
783 (CountersEnd - (CountersStart + CounterBaseOffset)) /
784 getCounterTypeSize();
785 if (NumCounters > MaxNumCounters)
786 return error(instrprof_error::malformed,
787 ("number of counters " + Twine(NumCounters) +
788 " is greater than the maximum number of counters " +
789 Twine(MaxNumCounters))
790 .str());
791
792 Record.Counts.clear();
793 Record.Counts.reserve(n: NumCounters);
794 for (uint32_t I = 0; I < NumCounters; I++) {
795 const char *Ptr =
796 CountersStart + CounterBaseOffset + I * getCounterTypeSize();
797 if (I == 0 && hasTemporalProfile()) {
798 uint64_t TimestampValue = swap(*reinterpret_cast<const uint64_t *>(Ptr));
799 if (TimestampValue != 0 &&
800 TimestampValue != std::numeric_limits<uint64_t>::max()) {
801 TemporalProfTimestamps.emplace_back(TimestampValue,
802 swap(Data->NameRef));
803 TemporalProfTraceStreamSize = 1;
804 }
805 if (hasSingleByteCoverage()) {
806 // In coverage mode, getCounterTypeSize() returns 1 byte but our
807 // timestamp field has size uint64_t. Increment I so that the next
808 // iteration of this for loop points to the byte after the timestamp
809 // field, i.e., I += 8.
810 I += 7;
811 }
812 continue;
813 }
814 if (hasSingleByteCoverage()) {
815 // A value of zero signifies the block is covered.
816 Record.Counts.push_back(x: *Ptr == 0 ? 1 : 0);
817 } else {
818 uint64_t CounterValue = swap(*reinterpret_cast<const uint64_t *>(Ptr));
819 if (CounterValue > MaxCounterValue && Warn)
820 Warn(make_error<InstrProfError>(
821 Args: instrprof_error::counter_value_too_large, Args: Twine(CounterValue)));
822
823 Record.Counts.push_back(x: CounterValue);
824 }
825 }
826
827 return success();
828}
829
830template <class IntPtrT>
831Error RawInstrProfReader<IntPtrT>::readRawBitmapBytes(InstrProfRecord &Record) {
832 uint32_t NumBitmapBytes = swap(Data->NumBitmapBytes);
833
834 Record.BitmapBytes.clear();
835 Record.BitmapBytes.reserve(n: NumBitmapBytes);
836
837 // It's possible MCDC is either not enabled or only used for some functions
838 // and not others. So if we record 0 bytes, just move on.
839 if (NumBitmapBytes == 0)
840 return success();
841
842 // BitmapDelta decreases as we advance to the next data record.
843 ptrdiff_t BitmapOffset = swap(Data->BitmapPtr) - BitmapDelta;
844 if (BitmapOffset < 0)
845 return error(
846 instrprof_error::malformed,
847 ("bitmap offset " + Twine(BitmapOffset) + " is negative").str());
848
849 if (BitmapOffset >= BitmapEnd - BitmapStart)
850 return error(instrprof_error::malformed,
851 ("bitmap offset " + Twine(BitmapOffset) +
852 " is greater than the maximum bitmap offset " +
853 Twine(BitmapEnd - BitmapStart - 1))
854 .str());
855
856 uint64_t MaxNumBitmapBytes =
857 (BitmapEnd - (BitmapStart + BitmapOffset)) / sizeof(uint8_t);
858 if (NumBitmapBytes > MaxNumBitmapBytes)
859 return error(instrprof_error::malformed,
860 ("number of bitmap bytes " + Twine(NumBitmapBytes) +
861 " is greater than the maximum number of bitmap bytes " +
862 Twine(MaxNumBitmapBytes))
863 .str());
864
865 for (uint32_t I = 0; I < NumBitmapBytes; I++) {
866 const char *Ptr = BitmapStart + BitmapOffset + I;
867 Record.BitmapBytes.push_back(swap(*Ptr));
868 }
869
870 return success();
871}
872
873template <class IntPtrT>
874Error RawInstrProfReader<IntPtrT>::readRawUniformCounters(
875 InstrProfRecord &Record) {
876 Record.UniformCounts.clear();
877
878 if (UniformCountersStart == UniformCountersEnd)
879 return success();
880
881 uint32_t NumCounters = swap(Data->NumCounters);
882
883 ptrdiff_t UniformCounterOffset =
884 swap(Data->UniformCounterPtr) - UniformCountersDelta;
885 if (UniformCounterOffset < 0)
886 return error(instrprof_error::malformed,
887 ("uniform counter offset " + Twine(UniformCounterOffset) +
888 " is negative")
889 .str());
890
891 if (UniformCounterOffset >= UniformCountersEnd - UniformCountersStart)
892 return error(instrprof_error::malformed,
893 ("uniform counter offset " + Twine(UniformCounterOffset) +
894 " is greater than the maximum uniform counter offset " +
895 Twine(UniformCountersEnd - UniformCountersStart - 1))
896 .str());
897
898 uint64_t MaxNumCounters =
899 (UniformCountersEnd - (UniformCountersStart + UniformCounterOffset)) /
900 sizeof(uint64_t);
901 if (NumCounters > MaxNumCounters)
902 return error(instrprof_error::malformed,
903 ("number of uniform counters " + Twine(NumCounters) +
904 " is greater than the maximum number of uniform counters " +
905 Twine(MaxNumCounters))
906 .str());
907
908 Record.UniformCounts.reserve(n: NumCounters);
909 for (uint32_t I = 0; I < NumCounters; I++) {
910 const char *Ptr =
911 UniformCountersStart + UniformCounterOffset + I * sizeof(uint64_t);
912 uint64_t CounterValue = swap(*reinterpret_cast<const uint64_t *>(Ptr));
913 Record.UniformCounts.push_back(x: CounterValue);
914 }
915
916 return success();
917}
918
919template <class IntPtrT>
920Error RawInstrProfReader<IntPtrT>::readValueProfilingData(
921 InstrProfRecord &Record) {
922 Record.clearValueData();
923 CurValueDataSize = 0;
924 // Need to match the logic in value profile dumper code in compiler-rt:
925 uint32_t NumValueKinds = 0;
926 for (uint32_t I = 0; I < IPVK_Last + 1; I++)
927 NumValueKinds += (Data->NumValueSites[I] != 0);
928
929 if (!NumValueKinds)
930 return success();
931
932 Expected<std::unique_ptr<ValueProfData>> VDataPtrOrErr =
933 ValueProfData::getValueProfData(
934 SrcBuffer: ValueDataStart, SrcBufferEnd: (const unsigned char *)DataBuffer->getBufferEnd(),
935 SrcDataEndianness: getDataEndianness());
936
937 if (Error E = VDataPtrOrErr.takeError())
938 return E;
939
940 // Note that besides deserialization, this also performs the conversion for
941 // indirect call targets. The function pointers from the raw profile are
942 // remapped into function name hashes.
943 VDataPtrOrErr.get()->deserializeTo(Record, SymTab: Symtab.get());
944 CurValueDataSize = VDataPtrOrErr.get()->getSize();
945 return success();
946}
947
948template <class IntPtrT>
949Error RawInstrProfReader<IntPtrT>::readNextRecord(NamedInstrProfRecord &Record) {
950 // Keep reading profiles that consist of only headers and no profile data and
951 // counters.
952 while (atEnd())
953 // At this point, ValueDataStart field points to the next header.
954 if (Error E = readNextHeader(CurrentPos: getNextHeaderPos()))
955 return error(std::move(E));
956
957 // Read name and set it in Record.
958 if (Error E = readName(Record))
959 return error(std::move(E));
960
961 // Read FuncHash and set it in Record.
962 if (Error E = readFuncHash(Record))
963 return error(std::move(E));
964
965 Record.OffloadDeviceWaveSize = swap(Data->OffloadDeviceWaveSize);
966
967 // Read raw counts and set Record.
968 if (Error E = readRawCounts(Record))
969 return error(std::move(E));
970
971 // Read raw bitmap bytes and set Record.
972 if (Error E = readRawBitmapBytes(Record))
973 return error(std::move(E));
974
975 // Read raw uniform counters and set Record.
976 if (Error E = readRawUniformCounters(Record))
977 return error(std::move(E));
978
979 // Read value data and set Record.
980 if (Error E = readValueProfilingData(Record))
981 return error(std::move(E));
982
983 // Iterate.
984 advanceData();
985 return success();
986}
987
988template <class IntPtrT>
989Error RawInstrProfReader<IntPtrT>::readBinaryIds(
990 std::vector<llvm::object::BuildID> &BinaryIds) {
991 BinaryIds.insert(position: BinaryIds.begin(), first: this->BinaryIds.begin(),
992 last: this->BinaryIds.end());
993 return Error::success();
994}
995
996template <class IntPtrT>
997Error RawInstrProfReader<IntPtrT>::printBinaryIds(raw_ostream &OS) {
998 if (!BinaryIds.empty())
999 printBinaryIdsInternal(OS, BinaryIds);
1000 return Error::success();
1001}
1002
1003namespace llvm {
1004
1005template class RawInstrProfReader<uint32_t>;
1006template class RawInstrProfReader<uint64_t>;
1007
1008} // end namespace llvm
1009
1010InstrProfLookupTrait::hash_value_type
1011InstrProfLookupTrait::ComputeHash(StringRef K) {
1012 return IndexedInstrProf::ComputeHash(Type: HashType, K);
1013}
1014
1015using data_type = InstrProfLookupTrait::data_type;
1016using offset_type = InstrProfLookupTrait::offset_type;
1017
1018bool InstrProfLookupTrait::readValueProfilingData(
1019 const unsigned char *&D, const unsigned char *const End) {
1020 Expected<std::unique_ptr<ValueProfData>> VDataPtrOrErr =
1021 ValueProfData::getValueProfData(SrcBuffer: D, SrcBufferEnd: End, SrcDataEndianness: ValueProfDataEndianness);
1022
1023 if (VDataPtrOrErr.takeError())
1024 return false;
1025
1026 VDataPtrOrErr.get()->deserializeTo(Record&: DataBuffer.back(), SymTab: nullptr);
1027 D += VDataPtrOrErr.get()->TotalSize;
1028
1029 return true;
1030}
1031
1032data_type InstrProfLookupTrait::ReadData(StringRef K, const unsigned char *D,
1033 offset_type N) {
1034 using namespace support;
1035
1036 // Check if the data is corrupt. If so, don't try to read it.
1037 if (N % sizeof(uint64_t))
1038 return data_type();
1039
1040 DataBuffer.clear();
1041 std::vector<uint64_t> CounterBuffer;
1042 std::vector<uint8_t> BitmapByteBuffer;
1043 std::vector<uint8_t> UniformityBitsBuffer;
1044
1045 const unsigned char *End = D + N;
1046 while (D < End) {
1047 // Read hash.
1048 if (D + sizeof(uint64_t) > End)
1049 return data_type();
1050 uint64_t Hash = endian::readNext<uint64_t, llvm::endianness::little>(memory&: D);
1051
1052 // Initialize number of counters for GET_VERSION(FormatVersion) == 1.
1053 uint64_t CountsSize = N / sizeof(uint64_t) - 1;
1054 // If format version is different then read the number of counters.
1055 if (GET_VERSION(FormatVersion) != IndexedInstrProf::ProfVersion::Version1) {
1056 if (D + sizeof(uint64_t) > End)
1057 return data_type();
1058 CountsSize = endian::readNext<uint64_t, llvm::endianness::little>(memory&: D);
1059 }
1060 // Read counter values.
1061 if (D + CountsSize * sizeof(uint64_t) > End)
1062 return data_type();
1063
1064 CounterBuffer.clear();
1065 CounterBuffer.reserve(n: CountsSize);
1066 for (uint64_t J = 0; J < CountsSize; ++J)
1067 CounterBuffer.push_back(
1068 x: endian::readNext<uint64_t, llvm::endianness::little>(memory&: D));
1069
1070 // Read bitmap bytes for GET_VERSION(FormatVersion) > 10.
1071 if (GET_VERSION(FormatVersion) > IndexedInstrProf::ProfVersion::Version10) {
1072 uint64_t BitmapBytes = 0;
1073 if (D + sizeof(uint64_t) > End)
1074 return data_type();
1075 BitmapBytes = endian::readNext<uint64_t, llvm::endianness::little>(memory&: D);
1076 BitmapByteBuffer.clear();
1077 BitmapByteBuffer.reserve(n: BitmapBytes);
1078
1079 if (GET_VERSION(FormatVersion) >=
1080 IndexedInstrProf::ProfVersion::Version14) {
1081 // Version 14+: bitmap bytes stored as uint8_t with padding.
1082 uint64_t PaddedSize = alignTo(Value: BitmapBytes, Align: sizeof(uint64_t));
1083 if (D + PaddedSize > End)
1084 return data_type();
1085 for (uint64_t J = 0; J < BitmapBytes; ++J)
1086 BitmapByteBuffer.push_back(
1087 x: endian::readNext<uint8_t, llvm::endianness::little>(memory&: D));
1088 for (uint64_t J = BitmapBytes; J < PaddedSize; ++J)
1089 (void)endian::readNext<uint8_t, llvm::endianness::little>(memory&: D);
1090
1091 // Read uniformity bits (AMDGPU offload profiling).
1092 uint64_t UniformityBitsSize = 0;
1093 if (D + sizeof(uint64_t) > End)
1094 return data_type();
1095 UniformityBitsSize =
1096 endian::readNext<uint64_t, llvm::endianness::little>(memory&: D);
1097 uint64_t PaddedUniformitySize =
1098 alignTo(Value: UniformityBitsSize, Align: sizeof(uint64_t));
1099 if (D + PaddedUniformitySize > End)
1100 return data_type();
1101 UniformityBitsBuffer.clear();
1102 UniformityBitsBuffer.reserve(n: UniformityBitsSize);
1103 for (uint64_t J = 0; J < UniformityBitsSize; ++J)
1104 UniformityBitsBuffer.push_back(
1105 x: endian::readNext<uint8_t, llvm::endianness::little>(memory&: D));
1106 for (uint64_t J = UniformityBitsSize; J < PaddedUniformitySize; ++J)
1107 (void)endian::readNext<uint8_t, llvm::endianness::little>(memory&: D);
1108 } else {
1109 // Version 11-13: each bitmap byte stored as a uint64_t.
1110 if (D + BitmapBytes * sizeof(uint64_t) > End)
1111 return data_type();
1112 for (uint64_t J = 0; J < BitmapBytes; ++J)
1113 BitmapByteBuffer.push_back(x: static_cast<uint8_t>(
1114 endian::readNext<uint64_t, llvm::endianness::little>(memory&: D)));
1115 }
1116 }
1117
1118 DataBuffer.emplace_back(args&: K, args&: Hash, args: std::move(CounterBuffer),
1119 args: std::move(BitmapByteBuffer),
1120 args: std::move(UniformityBitsBuffer));
1121
1122 // Read value profiling data.
1123 if (GET_VERSION(FormatVersion) > IndexedInstrProf::ProfVersion::Version2 &&
1124 !readValueProfilingData(D, End)) {
1125 DataBuffer.clear();
1126 return data_type();
1127 }
1128 }
1129 return DataBuffer;
1130}
1131
1132template <typename HashTableImpl>
1133Error InstrProfReaderIndex<HashTableImpl>::getRecords(
1134 StringRef FuncName, ArrayRef<NamedInstrProfRecord> &Data) {
1135 auto Iter = HashTable->find(FuncName);
1136 if (Iter == HashTable->end())
1137 return make_error<InstrProfError>(Args: instrprof_error::unknown_function);
1138
1139 Data = (*Iter);
1140 if (Data.empty())
1141 return make_error<InstrProfError>(Args: instrprof_error::malformed,
1142 Args: "profile data is empty");
1143
1144 return Error::success();
1145}
1146
1147template <typename HashTableImpl>
1148Error InstrProfReaderIndex<HashTableImpl>::getRecords(
1149 ArrayRef<NamedInstrProfRecord> &Data) {
1150 if (atEnd())
1151 return make_error<InstrProfError>(Args: instrprof_error::eof);
1152
1153 Data = *RecordIterator;
1154
1155 if (Data.empty())
1156 return make_error<InstrProfError>(Args: instrprof_error::malformed,
1157 Args: "profile data is empty");
1158
1159 return Error::success();
1160}
1161
1162template <typename HashTableImpl>
1163InstrProfReaderIndex<HashTableImpl>::InstrProfReaderIndex(
1164 const unsigned char *Buckets, const unsigned char *const Payload,
1165 const unsigned char *const Base, IndexedInstrProf::HashT HashType,
1166 uint64_t Version) {
1167 FormatVersion = Version;
1168 HashTable.reset(HashTableImpl::Create(
1169 Buckets, Payload, Base,
1170 typename HashTableImpl::InfoType(HashType, Version)));
1171 RecordIterator = HashTable->data_begin();
1172}
1173
1174template <typename HashTableImpl>
1175InstrProfKind InstrProfReaderIndex<HashTableImpl>::getProfileKind() const {
1176 return getProfileKindFromVersion(Version: FormatVersion);
1177}
1178
1179namespace {
1180/// A remapper that does not apply any remappings.
1181class InstrProfReaderNullRemapper : public InstrProfReaderRemapper {
1182 InstrProfReaderIndexBase &Underlying;
1183
1184public:
1185 InstrProfReaderNullRemapper(InstrProfReaderIndexBase &Underlying)
1186 : Underlying(Underlying) {}
1187
1188 Error getRecords(StringRef FuncName,
1189 ArrayRef<NamedInstrProfRecord> &Data) override {
1190 return Underlying.getRecords(FuncName, Data);
1191 }
1192};
1193} // namespace
1194
1195/// A remapper that applies remappings based on a symbol remapping file.
1196template <typename HashTableImpl>
1197class llvm::InstrProfReaderItaniumRemapper
1198 : public InstrProfReaderRemapper {
1199public:
1200 InstrProfReaderItaniumRemapper(
1201 std::unique_ptr<MemoryBuffer> RemapBuffer,
1202 InstrProfReaderIndex<HashTableImpl> &Underlying)
1203 : RemapBuffer(std::move(RemapBuffer)), Underlying(Underlying) {
1204 }
1205
1206 /// Extract the original function name from a PGO function name.
1207 static StringRef extractName(StringRef Name) {
1208 // We can have multiple pieces separated by kGlobalIdentifierDelimiter (
1209 // semicolon now and colon in older profiles); there can be pieces both
1210 // before and after the mangled name. Find the first part that starts with
1211 // '_Z'; we'll assume that's the mangled name we want.
1212 std::pair<StringRef, StringRef> Parts = {StringRef(), Name};
1213 while (true) {
1214 Parts = Parts.second.split(Separator: GlobalIdentifierDelimiter);
1215 if (Parts.first.starts_with(Prefix: "_Z"))
1216 return Parts.first;
1217 if (Parts.second.empty())
1218 return Name;
1219 }
1220 }
1221
1222 /// Given a mangled name extracted from a PGO function name, and a new
1223 /// form for that mangled name, reconstitute the name.
1224 static void reconstituteName(StringRef OrigName, StringRef ExtractedName,
1225 StringRef Replacement,
1226 SmallVectorImpl<char> &Out) {
1227 Out.reserve(N: OrigName.size() + Replacement.size() - ExtractedName.size());
1228 Out.insert(I: Out.end(), From: OrigName.begin(), To: ExtractedName.begin());
1229 llvm::append_range(C&: Out, R&: Replacement);
1230 Out.insert(I: Out.end(), From: ExtractedName.end(), To: OrigName.end());
1231 }
1232
1233 Error populateRemappings() override {
1234 if (Error E = Remappings.read(B&: *RemapBuffer))
1235 return E;
1236 for (StringRef Name : Underlying.HashTable->keys()) {
1237 StringRef RealName = extractName(Name);
1238 if (auto Key = Remappings.insert(FunctionName: RealName)) {
1239 // FIXME: We could theoretically map the same equivalence class to
1240 // multiple names in the profile data. If that happens, we should
1241 // return NamedInstrProfRecords from all of them.
1242 MappedNames.insert(KV: {Key, RealName});
1243 }
1244 }
1245 return Error::success();
1246 }
1247
1248 Error getRecords(StringRef FuncName,
1249 ArrayRef<NamedInstrProfRecord> &Data) override {
1250 StringRef RealName = extractName(Name: FuncName);
1251 if (auto Key = Remappings.lookup(FunctionName: RealName)) {
1252 StringRef Remapped = MappedNames.lookup(Val: Key);
1253 if (!Remapped.empty()) {
1254 if (RealName.begin() == FuncName.begin() &&
1255 RealName.end() == FuncName.end())
1256 FuncName = Remapped;
1257 else {
1258 // Try rebuilding the name from the given remapping.
1259 SmallString<256> Reconstituted;
1260 reconstituteName(OrigName: FuncName, ExtractedName: RealName, Replacement: Remapped, Out&: Reconstituted);
1261 Error E = Underlying.getRecords(Reconstituted, Data);
1262 if (!E)
1263 return E;
1264
1265 // If we failed because the name doesn't exist, fall back to asking
1266 // about the original name.
1267 if (Error Unhandled = handleErrors(
1268 std::move(E), [](std::unique_ptr<InstrProfError> Err) {
1269 return Err->get() == instrprof_error::unknown_function
1270 ? Error::success()
1271 : Error(std::move(Err));
1272 }))
1273 return Unhandled;
1274 }
1275 }
1276 }
1277 return Underlying.getRecords(FuncName, Data);
1278 }
1279
1280private:
1281 /// The memory buffer containing the remapping configuration. Remappings
1282 /// holds pointers into this buffer.
1283 std::unique_ptr<MemoryBuffer> RemapBuffer;
1284
1285 /// The mangling remapper.
1286 SymbolRemappingReader Remappings;
1287
1288 /// Mapping from mangled name keys to the name used for the key in the
1289 /// profile data.
1290 /// FIXME: Can we store a location within the on-disk hash table instead of
1291 /// redoing lookup?
1292 DenseMap<SymbolRemappingReader::Key, StringRef> MappedNames;
1293
1294 /// The real profile data reader.
1295 InstrProfReaderIndex<HashTableImpl> &Underlying;
1296};
1297
1298bool IndexedInstrProfReader::hasFormat(const MemoryBuffer &DataBuffer) {
1299 using namespace support;
1300
1301 if (DataBuffer.getBufferSize() < 8)
1302 return false;
1303 uint64_t Magic = endian::read<uint64_t, aligned>(memory: DataBuffer.getBufferStart(),
1304 endian: llvm::endianness::little);
1305 // Verify that it's magical.
1306 return Magic == IndexedInstrProf::Magic;
1307}
1308
1309const unsigned char *
1310IndexedInstrProfReader::readSummary(IndexedInstrProf::ProfVersion Version,
1311 const unsigned char *Cur, bool UseCS) {
1312 using namespace IndexedInstrProf;
1313 using namespace support;
1314
1315 if (Version >= IndexedInstrProf::Version4) {
1316 const IndexedInstrProf::Summary *SummaryInLE =
1317 reinterpret_cast<const IndexedInstrProf::Summary *>(Cur);
1318 uint64_t NFields = endian::byte_swap<uint64_t>(
1319 value: SummaryInLE->NumSummaryFields, endian: llvm::endianness::little);
1320 uint64_t NEntries = endian::byte_swap<uint64_t>(
1321 value: SummaryInLE->NumCutoffEntries, endian: llvm::endianness::little);
1322 uint32_t SummarySize =
1323 IndexedInstrProf::Summary::getSize(NumSumFields: NFields, NumCutoffEntries: NEntries);
1324 std::unique_ptr<IndexedInstrProf::Summary> SummaryData =
1325 IndexedInstrProf::allocSummary(TotalSize: SummarySize);
1326
1327 const uint64_t *Src = reinterpret_cast<const uint64_t *>(SummaryInLE);
1328 uint64_t *Dst = reinterpret_cast<uint64_t *>(SummaryData.get());
1329 for (unsigned I = 0; I < SummarySize / sizeof(uint64_t); I++)
1330 Dst[I] = endian::byte_swap<uint64_t>(value: Src[I], endian: llvm::endianness::little);
1331
1332 SummaryEntryVector DetailedSummary;
1333 for (unsigned I = 0; I < SummaryData->NumCutoffEntries; I++) {
1334 const IndexedInstrProf::Summary::Entry &Ent = SummaryData->getEntry(I);
1335 DetailedSummary.emplace_back(args: (uint32_t)Ent.Cutoff, args: Ent.MinBlockCount,
1336 args: Ent.NumBlocks);
1337 }
1338 std::unique_ptr<llvm::ProfileSummary> &Summary =
1339 UseCS ? this->CS_Summary : this->Summary;
1340
1341 // initialize InstrProfSummary using the SummaryData from disk.
1342 Summary = std::make_unique<ProfileSummary>(
1343 args: UseCS ? ProfileSummary::PSK_CSInstr : ProfileSummary::PSK_Instr,
1344 args&: DetailedSummary, args: SummaryData->get(K: Summary::TotalBlockCount),
1345 args: SummaryData->get(K: Summary::MaxBlockCount),
1346 args: SummaryData->get(K: Summary::MaxInternalBlockCount),
1347 args: SummaryData->get(K: Summary::MaxFunctionCount),
1348 args: SummaryData->get(K: Summary::TotalNumBlocks),
1349 args: SummaryData->get(K: Summary::TotalNumFunctions));
1350 return Cur + SummarySize;
1351 } else {
1352 // The older versions do not support a profile summary. This just computes
1353 // an empty summary, which will not result in accurate hot/cold detection.
1354 // We would need to call addRecord for all NamedInstrProfRecords to get the
1355 // correct summary. However, this version is old (prior to early 2016) and
1356 // has not been supporting an accurate summary for several years.
1357 InstrProfSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs);
1358 Summary = Builder.getSummary();
1359 return Cur;
1360 }
1361}
1362
1363Error IndexedInstrProfReader::readHeader() {
1364 using namespace support;
1365
1366 const unsigned char *Start =
1367 (const unsigned char *)DataBuffer->getBufferStart();
1368 const unsigned char *Cur = Start;
1369 if ((const unsigned char *)DataBuffer->getBufferEnd() - Cur < 24)
1370 return error(Err: instrprof_error::truncated);
1371
1372 auto HeaderOr = IndexedInstrProf::Header::readFromBuffer(Buffer: Start);
1373 if (!HeaderOr)
1374 return HeaderOr.takeError();
1375
1376 const IndexedInstrProf::Header *Header = &HeaderOr.get();
1377 Cur += Header->size();
1378
1379 Cur = readSummary(Version: (IndexedInstrProf::ProfVersion)Header->Version, Cur,
1380 /* UseCS */ false);
1381 if (Header->Version & VARIANT_MASK_CSIR_PROF)
1382 Cur = readSummary(Version: (IndexedInstrProf::ProfVersion)Header->Version, Cur,
1383 /* UseCS */ true);
1384 // Read the hash type and start offset.
1385 IndexedInstrProf::HashT HashType =
1386 static_cast<IndexedInstrProf::HashT>(Header->HashType);
1387 if (HashType > IndexedInstrProf::HashT::Last)
1388 return error(Err: instrprof_error::unsupported_hash_type);
1389
1390 // The hash table with profile counts comes next.
1391 auto IndexPtr = std::make_unique<InstrProfReaderIndex<OnDiskHashTableImplV3>>(
1392 args: Start + Header->HashOffset, args&: Cur, args&: Start, args&: HashType, args: Header->Version);
1393
1394 // The MemProfOffset field in the header is only valid when the format
1395 // version is higher than 8 (when it was introduced).
1396 if (Header->getIndexedProfileVersion() >= 8 &&
1397 Header->Version & VARIANT_MASK_MEMPROF) {
1398 if (Error E = MemProfReader.deserialize(Start, MemProfOffset: Header->MemProfOffset))
1399 return E;
1400 }
1401
1402 // BinaryIdOffset field in the header is only valid when the format version
1403 // is higher than 9 (when it was introduced).
1404 if (Header->getIndexedProfileVersion() >= 9) {
1405 const unsigned char *Ptr = Start + Header->BinaryIdOffset;
1406 // Read binary ids size.
1407 uint64_t BinaryIdsSize =
1408 support::endian::readNext<uint64_t, llvm::endianness::little>(memory&: Ptr);
1409 if (BinaryIdsSize % sizeof(uint64_t))
1410 return error(
1411 Err: instrprof_error::bad_header,
1412 ErrMsg: ("BinaryIdSize (" + Twine(BinaryIdsSize) + ") is not a multiple of 8")
1413 .str());
1414 // Set the binary ids start.
1415 BinaryIdsBuffer = ArrayRef<uint8_t>(Ptr, BinaryIdsSize);
1416 if (Ptr > (const unsigned char *)DataBuffer->getBufferEnd())
1417 return make_error<InstrProfError>(Args: instrprof_error::malformed,
1418 Args: "corrupted binary ids");
1419 }
1420
1421 if (Header->getIndexedProfileVersion() >= 12) {
1422 const unsigned char *Ptr = Start + Header->VTableNamesOffset;
1423
1424 uint64_t CompressedVTableNamesLen =
1425 support::endian::readNext<uint64_t, llvm::endianness::little>(memory&: Ptr);
1426
1427 // Writer first writes the length of compressed string, and then the actual
1428 // content.
1429 const char *VTableNamePtr = (const char *)Ptr;
1430 if (VTableNamePtr > DataBuffer->getBufferEnd())
1431 return make_error<InstrProfError>(Args: instrprof_error::truncated);
1432
1433 VTableName = StringRef(VTableNamePtr, CompressedVTableNamesLen);
1434 }
1435
1436 if (Header->getIndexedProfileVersion() >= 10 &&
1437 Header->Version & VARIANT_MASK_TEMPORAL_PROF) {
1438 const unsigned char *Ptr = Start + Header->TemporalProfTracesOffset;
1439 const auto *PtrEnd = (const unsigned char *)DataBuffer->getBufferEnd();
1440 // Expect at least two 64 bit fields: NumTraces, and TraceStreamSize
1441 if (Ptr + 2 * sizeof(uint64_t) > PtrEnd)
1442 return error(Err: instrprof_error::truncated);
1443 const uint64_t NumTraces =
1444 support::endian::readNext<uint64_t, llvm::endianness::little>(memory&: Ptr);
1445 TemporalProfTraceStreamSize =
1446 support::endian::readNext<uint64_t, llvm::endianness::little>(memory&: Ptr);
1447 for (unsigned i = 0; i < NumTraces; i++) {
1448 // Expect at least two 64 bit fields: Weight and NumFunctions
1449 if (Ptr + 2 * sizeof(uint64_t) > PtrEnd)
1450 return error(Err: instrprof_error::truncated);
1451 TemporalProfTraceTy Trace;
1452 Trace.Weight =
1453 support::endian::readNext<uint64_t, llvm::endianness::little>(memory&: Ptr);
1454 const uint64_t NumFunctions =
1455 support::endian::readNext<uint64_t, llvm::endianness::little>(memory&: Ptr);
1456 // Expect at least NumFunctions 64 bit fields
1457 if (Ptr + NumFunctions * sizeof(uint64_t) > PtrEnd)
1458 return error(Err: instrprof_error::truncated);
1459 for (unsigned j = 0; j < NumFunctions; j++) {
1460 const uint64_t NameRef =
1461 support::endian::readNext<uint64_t, llvm::endianness::little>(memory&: Ptr);
1462 Trace.FunctionNameRefs.push_back(x: NameRef);
1463 }
1464 TemporalProfTraces.push_back(Elt: std::move(Trace));
1465 }
1466 }
1467
1468 // Load the remapping table now if requested.
1469 if (RemappingBuffer) {
1470 Remapper =
1471 std::make_unique<InstrProfReaderItaniumRemapper<OnDiskHashTableImplV3>>(
1472 args: std::move(RemappingBuffer), args&: *IndexPtr);
1473 if (Error E = Remapper->populateRemappings())
1474 return E;
1475 } else {
1476 Remapper = std::make_unique<InstrProfReaderNullRemapper>(args&: *IndexPtr);
1477 }
1478 Index = std::move(IndexPtr);
1479
1480 return success();
1481}
1482
1483InstrProfSymtab &IndexedInstrProfReader::getSymtab() {
1484 if (Symtab)
1485 return *Symtab;
1486
1487 auto NewSymtab = std::make_unique<InstrProfSymtab>();
1488
1489 if (Error E = NewSymtab->initVTableNamesFromCompressedStrings(CompressedVTableNames: VTableName)) {
1490 auto [ErrCode, Msg] = InstrProfError::take(E: std::move(E));
1491 consumeError(Err: error(Err: ErrCode, ErrMsg: Msg));
1492 }
1493
1494 // finalizeSymtab is called inside populateSymtab.
1495 if (Error E = Index->populateSymtab(*NewSymtab)) {
1496 auto [ErrCode, Msg] = InstrProfError::take(E: std::move(E));
1497 consumeError(Err: error(Err: ErrCode, ErrMsg: Msg));
1498 }
1499
1500 Symtab = std::move(NewSymtab);
1501 return *Symtab;
1502}
1503
1504Expected<NamedInstrProfRecord> IndexedInstrProfReader::getInstrProfRecord(
1505 StringRef FuncName, uint64_t FuncHash, StringRef DeprecatedFuncName,
1506 uint64_t *MismatchedFuncSum) {
1507 ArrayRef<NamedInstrProfRecord> Data;
1508 uint64_t FuncSum = 0;
1509 auto Err = Remapper->getRecords(FuncName, Data);
1510 if (Err) {
1511 // If we don't find FuncName, try DeprecatedFuncName to handle profiles
1512 // built by older compilers.
1513 auto Err2 =
1514 handleErrors(E: std::move(Err), Hs: [&](const InstrProfError &IE) -> Error {
1515 if (IE.get() != instrprof_error::unknown_function)
1516 return make_error<InstrProfError>(Args: IE);
1517 if (auto Err = Remapper->getRecords(FuncName: DeprecatedFuncName, Data))
1518 return Err;
1519 return Error::success();
1520 });
1521 if (Err2)
1522 return std::move(Err2);
1523 }
1524 // Found it. Look for counters with the right hash.
1525
1526 // A flag to indicate if the records are from the same type
1527 // of profile (i.e cs vs nocs).
1528 bool CSBitMatch = false;
1529 auto getFuncSum = [](ArrayRef<uint64_t> Counts) {
1530 uint64_t ValueSum = 0;
1531 for (uint64_t CountValue : Counts) {
1532 if (CountValue == (uint64_t)-1)
1533 continue;
1534 // Handle overflow -- if that happens, return max.
1535 if (std::numeric_limits<uint64_t>::max() - CountValue <= ValueSum)
1536 return std::numeric_limits<uint64_t>::max();
1537 ValueSum += CountValue;
1538 }
1539 return ValueSum;
1540 };
1541
1542 for (const NamedInstrProfRecord &I : Data) {
1543 // Check for a match and fill the vector if there is one.
1544 if (I.Hash == FuncHash)
1545 return std::move(I);
1546 if (NamedInstrProfRecord::hasCSFlagInHash(FuncHash: I.Hash) ==
1547 NamedInstrProfRecord::hasCSFlagInHash(FuncHash)) {
1548 CSBitMatch = true;
1549 if (MismatchedFuncSum == nullptr)
1550 continue;
1551 FuncSum = std::max(a: FuncSum, b: getFuncSum(I.Counts));
1552 }
1553 }
1554 if (CSBitMatch) {
1555 if (MismatchedFuncSum != nullptr)
1556 *MismatchedFuncSum = FuncSum;
1557 return error(Err: instrprof_error::hash_mismatch);
1558 }
1559 return error(Err: instrprof_error::unknown_function);
1560}
1561
1562Expected<memprof::MemProfRecord>
1563IndexedMemProfReader::getMemProfRecord(const uint64_t FuncNameHash) const {
1564 // TODO: Add memprof specific errors.
1565 if (MemProfRecordTable == nullptr)
1566 return make_error<InstrProfError>(Args: instrprof_error::invalid_prof,
1567 Args: "no memprof data available in profile");
1568 auto Iter = MemProfRecordTable->find(EKey: FuncNameHash);
1569 if (Iter == MemProfRecordTable->end())
1570 return make_error<InstrProfError>(
1571 Args: instrprof_error::unknown_function,
1572 Args: "memprof record not found for function hash " + Twine(FuncNameHash));
1573
1574 const memprof::IndexedMemProfRecord &IndexedRecord = *Iter;
1575 switch (Version) {
1576 // Combine V3 and V4 cases as the record conversion logic is the same.
1577 case memprof::Version3:
1578 case memprof::Version4:
1579 assert(FrameBase && "FrameBase must be available");
1580 assert(CallStackBase && "CallStackBase must be available");
1581 {
1582 memprof::LinearFrameIdConverter FrameIdConv(FrameBase);
1583 memprof::LinearCallStackIdConverter CSIdConv(CallStackBase, FrameIdConv);
1584 memprof::MemProfRecord Record = IndexedRecord.toMemProfRecord(Callback: CSIdConv);
1585 return Record;
1586 }
1587 }
1588
1589 return make_error<InstrProfError>(
1590 Args: instrprof_error::unsupported_version,
1591 Args: formatv(Fmt: "MemProf version {} not supported; "
1592 "requires version between {} and {}, inclusive",
1593 Vals: Version, Vals: memprof::MinimumSupportedVersion,
1594 Vals: memprof::MaximumSupportedVersion));
1595}
1596
1597DenseMap<uint64_t, SmallVector<memprof::CallEdgeTy, 0>>
1598IndexedMemProfReader::getMemProfCallerCalleePairs() const {
1599 assert(MemProfRecordTable);
1600 assert(Version == memprof::Version3 || Version == memprof::Version4);
1601
1602 memprof::LinearFrameIdConverter FrameIdConv(FrameBase);
1603 memprof::CallerCalleePairExtractor Extractor(CallStackBase, FrameIdConv,
1604 RadixTreeSize);
1605
1606 // The set of linear call stack IDs that we need to traverse from. We expect
1607 // the set to be dense, so we use a BitVector.
1608 BitVector Worklist(RadixTreeSize);
1609
1610 // Collect the set of linear call stack IDs. Since we expect a lot of
1611 // duplicates, we first collect them in the form of a bit vector before
1612 // processing them.
1613 for (const memprof::IndexedMemProfRecord &IndexedRecord :
1614 MemProfRecordTable->data()) {
1615 for (const memprof::IndexedAllocationInfo &IndexedAI :
1616 IndexedRecord.AllocSites)
1617 Worklist.set(IndexedAI.CSId);
1618 }
1619
1620 // Collect caller-callee pairs for each linear call stack ID in Worklist.
1621 for (unsigned CS : Worklist.set_bits())
1622 Extractor(CS);
1623
1624 DenseMap<uint64_t, SmallVector<memprof::CallEdgeTy, 0>> Pairs =
1625 std::move(Extractor.CallerCalleePairs);
1626
1627 // Sort each call list by the source location.
1628 for (auto &[CallerGUID, CallList] : Pairs) {
1629 llvm::sort(C&: CallList);
1630 CallList.erase(CS: llvm::unique(R&: CallList), CE: CallList.end());
1631 }
1632
1633 return Pairs;
1634}
1635
1636memprof::AllMemProfData IndexedMemProfReader::getAllMemProfData() const {
1637 memprof::AllMemProfData AllMemProfData;
1638 AllMemProfData.HeapProfileRecords.reserve(
1639 n: MemProfRecordTable->getNumEntries());
1640 for (uint64_t Key : MemProfRecordTable->keys()) {
1641 auto Record = getMemProfRecord(FuncNameHash: Key);
1642 if (Record.takeError())
1643 continue;
1644 memprof::GUIDMemProfRecordPair Pair;
1645 Pair.GUID = Key;
1646 Pair.Record = std::move(*Record);
1647 AllMemProfData.HeapProfileRecords.push_back(x: std::move(Pair));
1648 }
1649 // Populate the data access profiles for yaml output.
1650 if (DataAccessProfileData != nullptr) {
1651 AllMemProfData.YamlifiedDataAccessProfiles.Records.reserve(
1652 n: DataAccessProfileData->getRecords().size());
1653 AllMemProfData.YamlifiedDataAccessProfiles.KnownColdSymbols.reserve(
1654 n: DataAccessProfileData->getKnownColdSymbols().size());
1655 AllMemProfData.YamlifiedDataAccessProfiles.KnownColdStrHashes.reserve(
1656 n: DataAccessProfileData->getKnownColdHashes().size());
1657 for (const auto &[SymHandleRef, RecordRef] :
1658 DataAccessProfileData->getRecords())
1659 AllMemProfData.YamlifiedDataAccessProfiles.Records.push_back(
1660 x: memprof::DataAccessProfRecord(SymHandleRef, RecordRef.AccessCount,
1661 RecordRef.Locations));
1662 for (StringRef ColdSymbol : DataAccessProfileData->getKnownColdSymbols())
1663 AllMemProfData.YamlifiedDataAccessProfiles.KnownColdSymbols.push_back(
1664 x: ColdSymbol.str());
1665 for (uint64_t Hash : DataAccessProfileData->getKnownColdHashes())
1666 AllMemProfData.YamlifiedDataAccessProfiles.KnownColdStrHashes.push_back(
1667 x: Hash);
1668 llvm::stable_sort(Range&: AllMemProfData.YamlifiedDataAccessProfiles.Records,
1669 C: [](const llvm::memprof::DataAccessProfRecord &lhs,
1670 const llvm::memprof::DataAccessProfRecord &rhs) {
1671 return lhs.AccessCount > rhs.AccessCount;
1672 });
1673 llvm::stable_sort(
1674 Range&: AllMemProfData.YamlifiedDataAccessProfiles.KnownColdSymbols,
1675 C: [](const std::string &lhs, const std::string &rhs) {
1676 return lhs < rhs;
1677 });
1678 llvm::stable_sort(
1679 Range&: AllMemProfData.YamlifiedDataAccessProfiles.KnownColdStrHashes,
1680 C: [](const uint64_t &lhs, const uint64_t &rhs) { return lhs < rhs; });
1681 }
1682 return AllMemProfData;
1683}
1684
1685Error IndexedInstrProfReader::getFunctionCounts(StringRef FuncName,
1686 uint64_t FuncHash,
1687 std::vector<uint64_t> &Counts) {
1688 auto Record = getInstrProfRecord(FuncName, FuncHash);
1689 if (Error E = Record.takeError())
1690 return error(E: std::move(E));
1691
1692 Counts = Record.get().Counts;
1693 return success();
1694}
1695
1696Error IndexedInstrProfReader::getFunctionBitmap(StringRef FuncName,
1697 uint64_t FuncHash,
1698 BitVector &Bitmap) {
1699 auto Record = getInstrProfRecord(FuncName, FuncHash);
1700 if (Error E = Record.takeError())
1701 return error(E: std::move(E));
1702
1703 const auto &BitmapBytes = Record.get().BitmapBytes;
1704 size_t I = 0, E = BitmapBytes.size();
1705 Bitmap.resize(N: E * CHAR_BIT);
1706 BitVector::apply(
1707 f: [&](auto X) {
1708 using XTy = decltype(X);
1709 alignas(XTy) uint8_t W[sizeof(X)];
1710 size_t N = std::min(a: E - I, b: sizeof(W));
1711 std::memset(s: W, c: 0, n: sizeof(W));
1712 std::memcpy(dest: W, src: &BitmapBytes[I], n: N);
1713 I += N;
1714 return support::endian::read<XTy, support::aligned>(
1715 W, llvm::endianness::little);
1716 },
1717 Out&: Bitmap, Arg: Bitmap);
1718 assert(I == E);
1719
1720 return success();
1721}
1722
1723Error IndexedInstrProfReader::readNextRecord(NamedInstrProfRecord &Record) {
1724 ArrayRef<NamedInstrProfRecord> Data;
1725
1726 Error E = Index->getRecords(Data);
1727 if (E)
1728 return error(E: std::move(E));
1729
1730 Record = Data[RecordIndex++];
1731 if (RecordIndex >= Data.size()) {
1732 Index->advanceToNextKey();
1733 RecordIndex = 0;
1734 }
1735 return success();
1736}
1737
1738Error IndexedInstrProfReader::readBinaryIds(
1739 std::vector<llvm::object::BuildID> &BinaryIds) {
1740 return readBinaryIdsInternal(DataBuffer: *DataBuffer, BinaryIdsBuffer, BinaryIds,
1741 Endian: llvm::endianness::little);
1742}
1743
1744Error IndexedInstrProfReader::printBinaryIds(raw_ostream &OS) {
1745 std::vector<llvm::object::BuildID> BinaryIds;
1746 if (Error E = readBinaryIds(BinaryIds))
1747 return E;
1748 printBinaryIdsInternal(OS, BinaryIds);
1749 return Error::success();
1750}
1751
1752void InstrProfReader::accumulateCounts(CountSumOrPercent &Sum, bool IsCS) {
1753 uint64_t NumFuncs = 0;
1754 for (const auto &Func : *this) {
1755 if (isIRLevelProfile()) {
1756 bool FuncIsCS = NamedInstrProfRecord::hasCSFlagInHash(FuncHash: Func.Hash);
1757 if (FuncIsCS != IsCS)
1758 continue;
1759 }
1760 Func.accumulateCounts(Sum);
1761 ++NumFuncs;
1762 }
1763 Sum.NumEntries = NumFuncs;
1764}
1765