1//===- SampleProfWriter.cpp - Write LLVM sample profile data --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the class that writes LLVM sample profiles. It
10// supports two file formats: text and binary. The textual representation
11// is useful for debugging and testing purposes. The binary representation
12// is more compact, resulting in smaller file sizes. However, they can
13// both be used interchangeably.
14//
15// See lib/ProfileData/SampleProfReader.cpp for documentation on each of the
16// supported formats.
17//
18//===----------------------------------------------------------------------===//
19
20#include "llvm/ProfileData/SampleProfWriter.h"
21#include "llvm/ADT/Eytzinger.h"
22#include "llvm/ADT/StringRef.h"
23#include "llvm/ProfileData/ProfileCommon.h"
24#include "llvm/ProfileData/SampleProf.h"
25#include "llvm/Support/Compression.h"
26#include "llvm/Support/EndianStream.h"
27#include "llvm/Support/ErrorOr.h"
28#include "llvm/Support/FileSystem.h"
29#include "llvm/Support/LEB128.h"
30#include "llvm/Support/MD5.h"
31#include "llvm/Support/SaveAndRestore.h"
32#include "llvm/Support/raw_ostream.h"
33#include <array>
34#include <cmath>
35#include <cstdint>
36#include <memory>
37#include <system_error>
38#include <utility>
39#include <vector>
40
41#define DEBUG_TYPE "llvm-profdata"
42
43using namespace llvm;
44using namespace sampleprof;
45
46// To begin with, make this option off by default.
47static cl::opt<bool> ExtBinaryWriteVTableTypeProf(
48 "extbinary-write-vtable-type-prof", cl::init(Val: false), cl::Hidden,
49 cl::desc("Write vtable type profile in ext-binary sample profile writer"));
50
51static cl::opt<uint64_t> RequestedVersion(
52 "sample-profile-format-version", cl::init(Val: DefaultVersion), cl::Hidden,
53 cl::desc("Format version to write for extensible binary profiles"));
54
55static cl::opt<bool>
56 ExtBinaryCompositeProf("extbinary-composite-prof", cl::init(Val: false),
57 cl::Hidden,
58 cl::desc("Use the composite profile format"));
59
60namespace llvm {
61namespace support {
62namespace endian {
63namespace {
64
65// Adapter class to llvm::support::endian::Writer for pwrite().
66struct SeekableWriter {
67 raw_pwrite_stream &OS;
68 endianness Endian;
69 SeekableWriter(raw_pwrite_stream &OS, endianness Endian)
70 : OS(OS), Endian(Endian) {}
71
72 template <typename ValueType> void pwrite(ValueType Val, size_t Offset) {
73 std::string StringBuf;
74 raw_string_ostream SStream(StringBuf);
75 Writer(SStream, Endian).write(Val);
76 OS.pwrite(Ptr: StringBuf.data(), Size: StringBuf.size(), Offset);
77 }
78};
79
80} // namespace
81} // namespace endian
82} // namespace support
83} // namespace llvm
84
85DefaultFunctionPruningStrategy::DefaultFunctionPruningStrategy(
86 SampleProfileMap &ProfileMap, size_t OutputSizeLimit)
87 : FunctionPruningStrategy(ProfileMap, OutputSizeLimit) {
88 sortFuncProfiles(ProfileMap, SortedProfiles&: SortedFunctions);
89}
90
91void DefaultFunctionPruningStrategy::Erase(size_t CurrentOutputSize) {
92 double D = (double)OutputSizeLimit / CurrentOutputSize;
93 size_t NewSize = (size_t)round(x: ProfileMap.size() * D * D);
94 size_t NumToRemove = ProfileMap.size() - NewSize;
95 if (NumToRemove < 1)
96 NumToRemove = 1;
97
98 assert(NumToRemove <= SortedFunctions.size());
99 for (const NameFunctionSamples &E :
100 llvm::drop_begin(RangeOrContainer&: SortedFunctions, N: SortedFunctions.size() - NumToRemove))
101 ProfileMap.erase(Key: E.first);
102 SortedFunctions.resize(new_size: SortedFunctions.size() - NumToRemove);
103}
104
105std::error_code SampleProfileWriter::writeWithSizeLimitInternal(
106 SampleProfileMap &ProfileMap, size_t OutputSizeLimit,
107 FunctionPruningStrategy *Strategy) {
108 if (OutputSizeLimit == 0)
109 return write(ProfileMap);
110
111 size_t OriginalFunctionCount = ProfileMap.size();
112
113 std::unique_ptr<raw_ostream> OriginalOutputStream;
114 OutputStream.swap(u&: OriginalOutputStream);
115
116 size_t IterationCount = 0;
117 size_t TotalSize;
118
119 SmallVector<char> StringBuffer;
120 do {
121 StringBuffer.clear();
122 OutputStream.reset(p: new raw_svector_ostream(StringBuffer));
123 if (std::error_code EC = write(ProfileMap))
124 return EC;
125
126 TotalSize = StringBuffer.size();
127 // On Windows every "\n" is actually written as "\r\n" to disk but not to
128 // memory buffer, this difference should be added when considering the total
129 // output size.
130#ifdef _WIN32
131 if (Format == SPF_Text)
132 TotalSize += LineCount;
133#endif
134 if (TotalSize <= OutputSizeLimit)
135 break;
136
137 Strategy->Erase(CurrentOutputSize: TotalSize);
138 IterationCount++;
139 } while (ProfileMap.size() != 0);
140
141 if (ProfileMap.size() == 0)
142 return sampleprof_error::too_large;
143
144 OutputStream.swap(u&: OriginalOutputStream);
145 OutputStream->write(Ptr: StringBuffer.data(), Size: StringBuffer.size());
146 LLVM_DEBUG(dbgs() << "Profile originally has " << OriginalFunctionCount
147 << " functions, reduced to " << ProfileMap.size() << " in "
148 << IterationCount << " iterations\n");
149 // Silence warning on Release build.
150 (void)OriginalFunctionCount;
151 (void)IterationCount;
152 return sampleprof_error::success;
153}
154
155std::error_code
156SampleProfileWriter::writeFuncProfiles(const SampleProfileMap &ProfileMap) {
157 std::vector<NameFunctionSamples> V;
158 sortFuncProfiles(ProfileMap, SortedProfiles&: V);
159 for (const auto &I : V) {
160 if (std::error_code EC = writeSample(S: *I.second))
161 return EC;
162 }
163 return sampleprof_error::success;
164}
165
166std::error_code SampleProfileWriter::write(const SampleProfileMap &ProfileMap) {
167 if (std::error_code EC = writeHeader(ProfileMap))
168 return EC;
169
170 if (std::error_code EC = writeFuncProfiles(ProfileMap))
171 return EC;
172
173 return sampleprof_error::success;
174}
175
176SmallVector<SecHdrTableEntry, 8>
177SampleProfileWriterExtBinaryBase::getExtBinaryHdrLayout(SectionLayout SL) {
178 // Note that SecFuncOffsetTable section is written after SecLBRProfile in the
179 // profile, but is put before SecLBRProfile in SectionHdrLayout. This is
180 // because sample reader follows the order in SectionHdrLayout to read each
181 // section. To read function profiles on demand, sample reader need to get the
182 // offset of each function profile first.
183 switch (SL) {
184 case DefaultLayout:
185 return {
186 {.Type: SecProfSummary, .Flags: 0, .Offset: 0, .Size: 0, .LayoutIndex: 0}, {.Type: SecNameTable, .Flags: 0, .Offset: 0, .Size: 0, .LayoutIndex: 0},
187 {.Type: SecCSNameTable, .Flags: 0, .Offset: 0, .Size: 0, .LayoutIndex: 0}, {.Type: SecFuncOffsetTable, .Flags: 0, .Offset: 0, .Size: 0, .LayoutIndex: 0},
188 {.Type: SecLBRProfile, .Flags: 0, .Offset: 0, .Size: 0, .LayoutIndex: 0}, {.Type: SecProfileSymbolList, .Flags: 0, .Offset: 0, .Size: 0, .LayoutIndex: 0},
189 {.Type: SecFuncMetadata, .Flags: 0, .Offset: 0, .Size: 0, .LayoutIndex: 0},
190 };
191 case CtxSplitLayout:
192 return {
193 {.Type: SecProfSummary, .Flags: 0, .Offset: 0, .Size: 0, .LayoutIndex: 0},
194 {.Type: SecNameTable, .Flags: 0, .Offset: 0, .Size: 0, .LayoutIndex: 0},
195 // profile with inlined functions
196 // for next two sections
197 {.Type: SecFuncOffsetTable, .Flags: 0, .Offset: 0, .Size: 0, .LayoutIndex: 0},
198 {.Type: SecLBRProfile, .Flags: 0, .Offset: 0, .Size: 0, .LayoutIndex: 0},
199 // profile without inlined functions
200 // for next two sections
201 {.Type: SecFuncOffsetTable, .Flags: static_cast<uint64_t>(SecCommonFlags::SecFlagFlat),
202 .Offset: 0, .Size: 0, .LayoutIndex: 0},
203 {.Type: SecLBRProfile, .Flags: static_cast<uint64_t>(SecCommonFlags::SecFlagFlat), .Offset: 0,
204 .Size: 0, .LayoutIndex: 0},
205 {.Type: SecProfileSymbolList, .Flags: 0, .Offset: 0, .Size: 0, .LayoutIndex: 0},
206 {.Type: SecFuncMetadata, .Flags: 0, .Offset: 0, .Size: 0, .LayoutIndex: 0},
207 };
208 case NumOfLayout:
209 break;
210 }
211 llvm_unreachable("Unsupported layout");
212}
213
214/// Return the current position and prepare to use it as the start
215/// position of a section given the section type \p Type and its position
216/// \p LayoutIdx in SectionHdrLayout.
217uint64_t
218SampleProfileWriterExtBinaryBase::markSectionStart(SecType Type,
219 uint32_t LayoutIdx) {
220 uint64_t SectionStart = OutputStream->tell();
221 assert(LayoutIdx < SectionHdrLayout.size() && "LayoutIdx out of range");
222 const auto &Entry = SectionHdrLayout[LayoutIdx];
223 assert(Entry.Type == Type && "Unexpected section type");
224 // Use LocalBuf as a temporary output for writing data.
225 if (hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagCompress))
226 LocalBufStream.swap(u&: OutputStream);
227 return SectionStart;
228}
229
230std::error_code SampleProfileWriterExtBinaryBase::compressAndOutput() {
231 if (!llvm::compression::zlib::isAvailable())
232 return sampleprof_error::zlib_unavailable;
233 std::string &UncompressedStrings =
234 static_cast<raw_string_ostream *>(LocalBufStream.get())->str();
235 if (UncompressedStrings.empty())
236 return sampleprof_error::success;
237 auto &OS = *OutputStream;
238 SmallVector<uint8_t, 128> CompressedStrings;
239 compression::zlib::compress(Input: arrayRefFromStringRef(Input: UncompressedStrings),
240 CompressedBuffer&: CompressedStrings,
241 Level: compression::zlib::BestSizeCompression);
242 encodeULEB128(Value: UncompressedStrings.size(), OS);
243 encodeULEB128(Value: CompressedStrings.size(), OS);
244 OS << toStringRef(Input: CompressedStrings);
245 UncompressedStrings.clear();
246 return sampleprof_error::success;
247}
248
249/// Add a new section into section header table given the section type
250/// \p Type, its position \p LayoutIdx in SectionHdrLayout and the
251/// location \p SectionStart where the section should be written to.
252std::error_code SampleProfileWriterExtBinaryBase::addNewSection(
253 SecType Type, uint32_t LayoutIdx, uint64_t SectionStart) {
254 assert(LayoutIdx < SectionHdrLayout.size() && "LayoutIdx out of range");
255 const auto &Entry = SectionHdrLayout[LayoutIdx];
256 assert(Entry.Type == Type && "Unexpected section type");
257 if (hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagCompress)) {
258 LocalBufStream.swap(u&: OutputStream);
259 if (std::error_code EC = compressAndOutput())
260 return EC;
261 }
262 SecHdrTable.push_back(x: {.Type: Type, .Flags: Entry.Flags, .Offset: SectionStart - FileStart,
263 .Size: OutputStream->tell() - SectionStart, .LayoutIndex: LayoutIdx});
264 return sampleprof_error::success;
265}
266
267std::error_code
268SampleProfileWriterExtBinaryBase::write(const SampleProfileMap &ProfileMap) {
269 // When calling write on a different profile map, existing states should be
270 // cleared.
271 NameTable.clear();
272 CSNameTable.clear();
273 SecHdrTable.clear();
274
275 if (std::error_code EC = writeHeader(ProfileMap))
276 return EC;
277
278 std::string LocalBuf;
279 LocalBufStream = std::make_unique<raw_string_ostream>(args&: LocalBuf);
280 if (std::error_code EC = writeSections(ProfileMap))
281 return EC;
282
283 if (std::error_code EC = writeSecHdrTable())
284 return EC;
285
286 return sampleprof_error::success;
287}
288
289std::error_code SampleProfileWriterExtBinaryBase::writeContextIdx(
290 const SampleContext &Context) {
291 if (Context.hasContext())
292 return writeCSNameIdx(Context);
293 else
294 return SampleProfileWriterBinary::writeNameIdx(FName: Context.getFunction());
295}
296
297std::error_code
298SampleProfileWriterExtBinaryBase::writeCSNameIdx(const SampleContext &Context) {
299 const auto &Ret = CSNameTable.find(Key: Context);
300 if (Ret == CSNameTable.end())
301 return sampleprof_error::truncated_name_table;
302 encodeULEB128(Value: Ret->second, OS&: *OutputStream);
303 return sampleprof_error::success;
304}
305
306std::error_code
307SampleProfileWriterExtBinaryBase::writeSample(const FunctionSamples &S) {
308 uint64_t Offset = OutputStream->tell();
309 auto &Context = S.getContext();
310 FuncOffsetTable[Context] = Offset - SecLBRProfileStart;
311 if (!WriteCompositeProf)
312 encodeULEB128(Value: S.getHeadSamples(), OS&: *OutputStream);
313 return writeBody(S, /*IsNested=*/false);
314}
315
316std::error_code
317SampleProfileWriterExtBinaryBase::writeFuncOffsetTable(SecType Type,
318 bool IsNested) {
319 if (UseMD5IndexedTables) {
320 // Eytzinger layout requires MD5 representation and does not support
321 // multi-context Context-Sensitive profiles.
322 if (!UseMD5 || FunctionSamples::ProfileIsCS)
323 return sampleprof_error::unsupported_writing_format;
324 return writeEytzingerFuncOffsetTable(Type, IsNested);
325 }
326 return writeLegacyFuncOffsetTable(Type);
327}
328
329std::error_code
330SampleProfileWriterExtBinaryBase::writeEytzingerFuncOffsetTable(SecType Type,
331 bool IsNested) {
332 assert((NumNested + NumFlat > 0 || FuncOffsetTable.empty()) &&
333 "SecNameTable must be written before SecFuncOffsetTable to establish "
334 "Eytzinger indices!");
335
336 size_t SpanSize = IsNested ? NumNested : NumFlat;
337 size_t BaseIdx = IsNested ? 0 : NumNested;
338
339 std::vector<support::ulittle32_t> FuncOffsets(
340 SpanSize, support::ulittle32_t(UINT32_MAX));
341
342 // Populate the function offset array parallel to the Eytzinger span.
343 for (const auto &[Context, RelativeOffset] : FuncOffsetTable) {
344 if (RelativeOffset >= UINT32_MAX)
345 return sampleprof_error::too_large;
346
347 FunctionId FId = Context.getFunction();
348 auto It = NameTable.find(Key: FId);
349 if (It == NameTable.end())
350 continue;
351
352 size_t GlobalIdx = It->second;
353 if (GlobalIdx < BaseIdx || (GlobalIdx - BaseIdx) >= SpanSize)
354 continue;
355
356 size_t LocalIdx = GlobalIdx - BaseIdx;
357 assert(
358 FuncOffsets[LocalIdx] == UINT32_MAX &&
359 "Function offset slot already populated; duplicate GUID or collision!");
360 FuncOffsets[LocalIdx] = static_cast<uint32_t>(RelativeOffset);
361 }
362
363 assert(!llvm::is_contained(FuncOffsets, support::ulittle32_t(UINT32_MAX)) &&
364 "Unpopulated slot in Eytzinger function offset array!");
365
366 OutputStream->write(Ptr: reinterpret_cast<const char *>(FuncOffsets.data()),
367 Size: SpanSize * sizeof(support::ulittle32_t));
368 // Type is SecFuncOffsetTable or SecCompositeFuncOffsetTable.
369 addSectionFlag(Type, Flag: SecFuncOffsetFlags::SecFlagEytzinger);
370 FuncOffsetTable.clear();
371 return sampleprof_error::success;
372}
373
374std::error_code
375SampleProfileWriterExtBinaryBase::writeLegacyFuncOffsetTable(SecType Type) {
376 auto &OS = *OutputStream;
377
378 // Write out the table size.
379 encodeULEB128(Value: FuncOffsetTable.size(), OS);
380
381 // Write out FuncOffsetTable.
382 auto WriteItem = [&](const SampleContext &Context, uint64_t Offset) {
383 if (std::error_code EC = writeContextIdx(Context))
384 return EC;
385 encodeULEB128(Value: Offset, OS);
386 return (std::error_code)sampleprof_error::success;
387 };
388
389 if (FunctionSamples::ProfileIsCS) {
390 // Sort the contexts before writing them out. This is to help fast load all
391 // context profiles for a function as well as their callee contexts which
392 // can help profile-guided importing for ThinLTO.
393 std::map<SampleContext, uint64_t> OrderedFuncOffsetTable(
394 FuncOffsetTable.begin(), FuncOffsetTable.end());
395 for (const auto &Entry : OrderedFuncOffsetTable) {
396 if (std::error_code EC = WriteItem(Entry.first, Entry.second))
397 return EC;
398 }
399 addSectionFlag(Type, Flag: SecFuncOffsetFlags::SecFlagOrdered);
400 } else {
401 for (const auto &Entry : FuncOffsetTable) {
402 if (std::error_code EC = WriteItem(Entry.first, Entry.second))
403 return EC;
404 }
405 }
406
407 FuncOffsetTable.clear();
408 return sampleprof_error::success;
409}
410
411std::error_code SampleProfileWriterExtBinaryBase::writeFuncMetadata(
412 const FunctionSamples &FunctionProfile) {
413 auto &OS = *OutputStream;
414 if (std::error_code EC = writeContextIdx(Context: FunctionProfile.getContext()))
415 return EC;
416
417 if (FunctionSamples::ProfileIsProbeBased)
418 encodeULEB128(Value: FunctionProfile.getFunctionHash(), OS);
419 if (FunctionSamples::ProfileIsCS || FunctionSamples::ProfileIsPreInlined) {
420 encodeULEB128(Value: FunctionProfile.getContext().getAllAttributes(), OS);
421 }
422
423 if (!FunctionSamples::ProfileIsCS) {
424 // Recursively emit attributes for all callee samples.
425 uint64_t NumCallsites = 0;
426 for (const auto &J : FunctionProfile.getCallsiteSamples())
427 NumCallsites += J.second.size();
428 encodeULEB128(Value: NumCallsites, OS);
429 for (const auto &J : FunctionProfile.getCallsiteSamples()) {
430 for (const auto &FS : J.second) {
431 LineLocation Loc = J.first;
432 encodeULEB128(Value: Loc.LineOffset, OS);
433 encodeULEB128(Value: Loc.Discriminator, OS);
434 if (std::error_code EC = writeFuncMetadata(FunctionProfile: FS.second))
435 return EC;
436 }
437 }
438 }
439
440 return sampleprof_error::success;
441}
442
443std::error_code SampleProfileWriterExtBinaryBase::writeFuncMetadata(
444 const SampleProfileMap &Profiles) {
445 if (!FunctionSamples::ProfileIsProbeBased && !FunctionSamples::ProfileIsCS &&
446 !FunctionSamples::ProfileIsPreInlined)
447 return sampleprof_error::success;
448 for (const auto &Entry : Profiles) {
449 if (std::error_code EC = writeFuncMetadata(FunctionProfile: Entry.second))
450 return EC;
451 }
452 return sampleprof_error::success;
453}
454
455template <class KeyT, class ValT>
456static SmallVector<std::pair<KeyT, ValT> *, 0>
457stabilizeTable(MapVector<KeyT, ValT> &Table) {
458 SmallVector<std::pair<KeyT, ValT> *, 0> Entries(
459 llvm::make_pointer_range(Table));
460
461 llvm::sort(Entries,
462 [](const auto *L, const auto *R) { return L->first < R->first; });
463
464 for (const auto &[I, Entry] : llvm::enumerate(Entries))
465 Entry->second = I;
466
467 return Entries;
468}
469
470std::error_code SampleProfileWriterExtBinaryBase::writeNameTable() {
471 if (!UseMD5)
472 return SampleProfileWriterBinary::writeNameTable();
473
474 auto &OS = *OutputStream;
475
476 // Write out the MD5 name table. We wrote unencoded MD5 so reader can
477 // retrieve the name using the name index without having to read the
478 // whole name table.
479 encodeULEB128(Value: NameTable.size(), OS);
480 support::endian::Writer Writer(OS, llvm::endianness::little);
481 for (const auto *Entry : stabilizeTable(Table&: NameTable))
482 Writer.write(Val: Entry->first.getHashCode());
483 return sampleprof_error::success;
484}
485
486std::error_code SampleProfileWriterExtBinaryBase::writeNameTableSection(
487 const SampleProfileMap &ProfileMap) {
488 for (const auto &I : ProfileMap) {
489 addContext(Context: I.second.getContext());
490 addNames(S: I.second);
491 }
492
493 // If NameTable contains ".__uniq." suffix, set SecFlagUniqSuffix flag
494 // so compiler won't strip the suffix during profile matching after
495 // seeing the flag in the profile.
496 // Original names are unavailable if using MD5, so this option has no use.
497 if (!UseMD5) {
498 for (const auto &I : NameTable) {
499 if (I.first.stringRef().contains(Other: FunctionSamples::UniqSuffix)) {
500 addSectionFlag(Type: SecNameTable, Flag: SecNameTableFlags::SecFlagUniqSuffix);
501 break;
502 }
503 }
504 }
505
506 if (UseMD5 && UseMD5IndexedTables) {
507 // Eytzinger name tables do not support CSSPGO profiles
508 // (FunctionSamples::ProfileIsCS).
509 if (FunctionSamples::ProfileIsCS)
510 return sampleprof_error::unsupported_writing_format;
511 if (auto EC = writeEytzingerNameTableSection(ProfileMap))
512 return EC;
513 return sampleprof_error::success;
514 }
515
516 if (auto EC = writeNameTable())
517 return EC;
518 return sampleprof_error::success;
519}
520
521namespace {
522
523// Helper class to construct and write the SecNameTable section in Eytzinger
524// layout for ExtBinary MD5 profiles.
525//
526// The on-disk layout of the Eytzinger name table section consists of symbol
527// counts followed by three contiguous Eytzinger hash arrays:
528// - ULEB128 count of Nested top-level profile symbol keys
529// - ULEB128 count of Flat top-level profile symbol keys
530// - ULEB128 count of Inlinee and auxiliary profile symbol keys
531// - Array of 64-bit little-endian MD5 hash keys for Nested profiles in
532// Eytzinger order
533// - Array of 64-bit little-endian MD5 hash keys for Flat profiles in Eytzinger
534// order
535// - Array of 64-bit little-endian MD5 hash keys for Inlinees in Eytzinger order
536class EytzingerNameTable {
537 using TableT = llvm::EytzingerTable<support::ulittle64_t>;
538 std::array<TableT, static_cast<size_t>(EytzingerSpan::NumSpans)> Spans;
539
540public:
541 EytzingerNameTable(std::vector<support::ulittle64_t> NestedKeys,
542 std::vector<support::ulittle64_t> FlatKeys,
543 std::vector<support::ulittle64_t> InlineeKeys)
544 : Spans{TableT::create(Keys: std::move(NestedKeys)),
545 TableT::create(Keys: std::move(FlatKeys)),
546 TableT::create(Keys: std::move(InlineeKeys))} {}
547
548 // Find the global index of GUID across the three Eytzinger table spans.
549 uint64_t findGlobalIdx(uint64_t GUID) const {
550 uint64_t BaseIdx = 0;
551 for (const auto &Table : Spans) {
552 if (std::optional<size_t> LocalIdx = Table.findIndex(Target: GUID))
553 return BaseIdx + *LocalIdx;
554 BaseIdx += Table.size();
555 }
556 llvm_unreachable("Symbol in NameTable missing from Eytzinger spans");
557 }
558
559 void write(raw_ostream &OS) const {
560 for (const auto &Table : Spans)
561 encodeULEB128(Value: uint64_t(Table.size()), OS);
562 for (const auto &Table : Spans)
563 OS.write(Ptr: reinterpret_cast<const char *>(Table.data()),
564 Size: Table.size() * sizeof(support::ulittle64_t));
565 }
566
567 size_t size(EytzingerSpan S) const {
568 return Spans[static_cast<size_t>(S)].size();
569 }
570};
571
572} // end anonymous namespace
573
574std::error_code
575SampleProfileWriterExtBinaryBase::writeEytzingerNameTableSection(
576 const SampleProfileMap &ProfileMap) {
577 DenseSet<uint64_t> TopLevelGUIDs;
578 std::vector<support::ulittle64_t> NestedKeys, FlatKeys, InlineeKeys;
579
580 // Collect top-level Nested and Flat keys directly from ProfileMap.
581 for (const auto &I : ProfileMap) {
582 const SampleContext &Ctx = I.second.getContext();
583 uint64_t GUID = Ctx.getFunction().getHashCode();
584 if (TopLevelGUIDs.insert(V: GUID).second) {
585 // In single-table default layouts, unify all top-level symbols in the
586 // Nested partition so they match the single unflagged function offset
587 // table.
588 if (SecLayout != CtxSplitLayout || I.second.hasCallsiteSamples())
589 NestedKeys.emplace_back(args&: GUID);
590 else
591 FlatKeys.emplace_back(args&: GUID);
592 }
593 }
594
595 // Collect remaining non-top-level symbols (inlinees, targets, vtables) from
596 // NameTable.
597 for (const auto &Entry : NameTable) {
598 uint64_t GUID = Entry.first.getHashCode();
599 if (!TopLevelGUIDs.contains(V: GUID))
600 InlineeKeys.emplace_back(args&: GUID);
601 }
602
603 EytzingerNameTable Tables(std::move(NestedKeys), std::move(FlatKeys),
604 std::move(InlineeKeys));
605
606 // Assign each symbol its corresponding index in the Eytzinger layout.
607 for (auto &[FId, Idx] : NameTable)
608 Idx = Tables.findGlobalIdx(GUID: FId.getHashCode());
609
610 Tables.write(OS&: *OutputStream);
611 NumNested = Tables.size(S: EytzingerSpan::Nested);
612 NumFlat = Tables.size(S: EytzingerSpan::Flat);
613
614 return sampleprof_error::success;
615}
616
617std::error_code SampleProfileWriterExtBinaryBase::writeCSNameTableSection() {
618 auto &OS = *OutputStream;
619 encodeULEB128(Value: CSNameTable.size(), OS);
620 support::endian::Writer Writer(OS, llvm::endianness::little);
621 for (const auto *Entry : stabilizeTable(Table&: CSNameTable)) {
622 auto Frames = Entry->first.getContextFrames();
623 encodeULEB128(Value: Frames.size(), OS);
624 for (auto &Callsite : Frames) {
625 if (std::error_code EC = writeNameIdx(FName: Callsite.Func))
626 return EC;
627 encodeULEB128(Value: Callsite.Location.LineOffset, OS);
628 encodeULEB128(Value: Callsite.Location.Discriminator, OS);
629 }
630 }
631
632 return sampleprof_error::success;
633}
634
635std::error_code
636SampleProfileWriterExtBinaryBase::writeProfileSymbolListSection() {
637 if (UseMD5ProfSymList)
638 return writeMD5ProfileSymbolListSection();
639 return writeStringBasedProfileSymbolListSection();
640}
641
642std::error_code
643SampleProfileWriterExtBinaryBase::writeStringBasedProfileSymbolListSection() {
644 assert((!ProfSymList || !ProfSymList->isMD5()) &&
645 "Writing string-based ProfileSymbolListSection from MD5 table "
646 "not yet implemented");
647 if (ProfSymList && ProfSymList->size() > 0)
648 if (std::error_code EC = ProfSymList->write(OS&: *OutputStream))
649 return EC;
650
651 return sampleprof_error::success;
652}
653
654std::error_code
655SampleProfileWriterExtBinaryBase::writeMD5ProfileSymbolListSection() {
656 if (!ProfSymList || ProfSymList->size() == 0)
657 return sampleprof_error::success;
658
659 auto &OS = *OutputStream;
660 std::vector<uint64_t> Keys = ProfSymList->collectGUIDs();
661
662 auto Table =
663 llvm::EytzingerTable<support::ulittle64_t>::create(Keys: std::move(Keys));
664
665 OS.write(Ptr: reinterpret_cast<const char *>(Table.data()),
666 Size: Table.size() * sizeof(support::ulittle64_t));
667 return sampleprof_error::success;
668}
669
670unsigned SampleProfileWriterExtBinaryBase::findUnwrittenEntry(SecType Type) {
671 auto WrittenIndices =
672 llvm::map_range(C&: SecHdrTable, F: &SecHdrTableEntry::LayoutIndex);
673 for (auto [I, Entry] : llvm::enumerate(First&: SectionHdrLayout))
674 if (Entry.Type == Type && !llvm::is_contained(Range&: WrittenIndices, Element: I))
675 return I;
676 llvm_unreachable("Matching section not found in SectionHdrLayout");
677}
678
679std::error_code SampleProfileWriterExtBinaryBase::writeOneSection(
680 SecType Type, const SampleProfileMap &ProfileMap) {
681 unsigned LayoutIdx = findUnwrittenEntry(Type);
682 SecHdrTableEntry &Entry = SectionHdrLayout[LayoutIdx];
683
684 // The setting of SecFlagCompress should happen before markSectionStart.
685 if (Type == SecFuncMetadata && FunctionSamples::ProfileIsProbeBased)
686 addSectionFlag(Type: SecFuncMetadata, Flag: SecFuncMetadataFlags::SecFlagIsProbeBased);
687 if (Type == SecFuncMetadata &&
688 (FunctionSamples::ProfileIsCS || FunctionSamples::ProfileIsPreInlined))
689 addSectionFlag(Type: SecFuncMetadata, Flag: SecFuncMetadataFlags::SecFlagHasAttribute);
690 if (Type == SecProfSummary && FunctionSamples::ProfileIsCS)
691 addSectionFlag(Type: SecProfSummary, Flag: SecProfSummaryFlags::SecFlagFullContext);
692 if (Type == SecProfSummary && FunctionSamples::ProfileIsPreInlined)
693 addSectionFlag(Type: SecProfSummary, Flag: SecProfSummaryFlags::SecFlagIsPreInlined);
694 if (Type == SecProfSummary && FunctionSamples::ProfileIsFS)
695 addSectionFlag(Type: SecProfSummary, Flag: SecProfSummaryFlags::SecFlagFSDiscriminator);
696 if (Type == SecProfSummary && ExtBinaryWriteVTableTypeProf)
697 addSectionFlag(Type: SecProfSummary,
698 Flag: SecProfSummaryFlags::SecFlagHasVTableTypeProf);
699 if (Type == SecProfileSymbolList && UseMD5ProfSymList)
700 addSectionFlag(Type: SecProfileSymbolList, Flag: SecProfileSymbolListFlags::SecFlagMD5);
701 if (Type == SecNameTable && UseMD5IndexedTables && UseMD5)
702 addSectionFlag(Type: SecNameTable, Flag: SecNameTableFlags::SecFlagEytzinger);
703
704 uint64_t SectionStart = markSectionStart(Type, LayoutIdx);
705 switch (Type) {
706 case SecProfSummary:
707 computeSummary(ProfileMap);
708 if (auto EC = writeSummary())
709 return EC;
710 break;
711 case SecNameTable:
712 if (auto EC = writeNameTableSection(ProfileMap))
713 return EC;
714 break;
715 case SecCSNameTable:
716 if (auto EC = writeCSNameTableSection())
717 return EC;
718 break;
719 case SecLBRProfile:
720 case SecCompositeProfile:
721 SecLBRProfileStart = OutputStream->tell();
722 if (std::error_code EC = writeFuncProfiles(ProfileMap))
723 return EC;
724 break;
725 case SecFuncOffsetTable:
726 case SecCompositeFuncOffsetTable: {
727 bool IsFlat = hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagFlat);
728 // An unflagged function offset table inherently indexes the primary
729 // Nested symbol span.
730 bool IsNested = !IsFlat;
731 if (auto EC = writeFuncOffsetTable(Type, IsNested))
732 return EC;
733 break;
734 }
735 case SecFuncMetadata:
736 if (std::error_code EC = writeFuncMetadata(Profiles: ProfileMap))
737 return EC;
738 break;
739 case SecProfileSymbolList:
740 if (auto EC = writeProfileSymbolListSection())
741 return EC;
742 break;
743 default:
744 if (auto EC = writeCustomSection(Type))
745 return EC;
746 break;
747 }
748 if (std::error_code EC = addNewSection(Type, LayoutIdx, SectionStart))
749 return EC;
750 return sampleprof_error::success;
751}
752
753SampleProfileWriterExtBinary::SampleProfileWriterExtBinary(
754 std::unique_ptr<raw_ostream> &OS)
755 : SampleProfileWriterExtBinaryBase(OS) {
756 WriteVTableProf = ExtBinaryWriteVTableTypeProf;
757}
758
759std::error_code SampleProfileWriterExtBinary::writeDefaultLayout(
760 const SampleProfileMap &ProfileMap) {
761 // ProfSection / FuncOffsetSection are SecLBR* or SecComposite* after
762 // configureCompositeProfile.
763 const SecType Sections[] = {
764 SecProfSummary, SecNameTable, SecCSNameTable, ProfSection,
765 SecProfileSymbolList, FuncOffsetSection, SecFuncMetadata,
766 };
767 for (SecType Type : Sections)
768 if (std::error_code EC = writeOneSection(Type, ProfileMap))
769 return EC;
770 return sampleprof_error::success;
771}
772
773static void splitProfileMapToTwo(const SampleProfileMap &ProfileMap,
774 SampleProfileMap &NestedProfileMap,
775 SampleProfileMap &FlatProfileMap) {
776 for (const auto &I : ProfileMap) {
777 if (I.second.hasCallsiteSamples())
778 NestedProfileMap.insert(x: {I.first, I.second});
779 else
780 FlatProfileMap.insert(x: {I.first, I.second});
781 }
782}
783
784std::error_code SampleProfileWriterExtBinary::writeCtxSplitLayout(
785 const SampleProfileMap &ProfileMap) {
786 SampleProfileMap NestedProfileMap, FlatProfileMap;
787 splitProfileMapToTwo(ProfileMap, NestedProfileMap, FlatProfileMap);
788
789 // Flat SecFlag is pre-set in getExtBinaryHdrLayout; findUnwrittenEntry
790 // picks the matching unwritten ProfSection / FuncOffsetSection slot.
791 const std::pair<SecType, const SampleProfileMap &> Sections[] = {
792 {SecProfSummary, ProfileMap}, {SecNameTable, ProfileMap},
793 {ProfSection, NestedProfileMap}, {FuncOffsetSection, NestedProfileMap},
794 {ProfSection, FlatProfileMap}, {FuncOffsetSection, FlatProfileMap},
795 {SecProfileSymbolList, ProfileMap}, {SecFuncMetadata, ProfileMap},
796 };
797 for (const auto &[Type, Map] : Sections)
798 if (std::error_code EC = writeOneSection(Type, ProfileMap: Map))
799 return EC;
800
801 return sampleprof_error::success;
802}
803
804void SampleProfileWriterExtBinary::configureCompositeProfile() {
805 ProfSection = WriteCompositeProf ? SecCompositeProfile : SecLBRProfile;
806 FuncOffsetSection =
807 WriteCompositeProf ? SecCompositeFuncOffsetTable : SecFuncOffsetTable;
808
809 // Change the section types in place to avoid duplicating the whole layout and
810 // its handling. Rewrite both legacy and composite entries so repeated writes
811 // can switch formats without losing configured flags.
812 for (auto &Entry : SectionHdrLayout) {
813 if (Entry.Type == SecFuncOffsetTable ||
814 Entry.Type == SecCompositeFuncOffsetTable)
815 Entry.Type = FuncOffsetSection;
816 else if (Entry.Type == SecLBRProfile || Entry.Type == SecCompositeProfile)
817 Entry.Type = ProfSection;
818 }
819}
820
821std::error_code SampleProfileWriterExtBinary::writeSections(
822 const SampleProfileMap &ProfileMap) {
823 // Rewrite the final configured layout immediately before its section types
824 // are consumed. Earlier layout configuration may replace SectionHdrLayout.
825 configureCompositeProfile();
826
827 std::error_code EC;
828 if (SecLayout == DefaultLayout)
829 EC = writeDefaultLayout(ProfileMap);
830 else if (SecLayout == CtxSplitLayout)
831 EC = writeCtxSplitLayout(ProfileMap);
832 else
833 llvm_unreachable("Unsupported layout");
834 return EC;
835}
836
837/// Write samples to a text file.
838///
839/// Note: it may be tempting to implement this in terms of
840/// FunctionSamples::print(). Please don't. The dump functionality is intended
841/// for debugging and has no specified form.
842///
843/// The format used here is more structured and deliberate because
844/// it needs to be parsed by the SampleProfileReaderText class.
845std::error_code SampleProfileWriterText::writeSample(const FunctionSamples &S) {
846 auto &OS = *OutputStream;
847 if (FunctionSamples::ProfileIsCS)
848 OS << "[" << S.getContext().toString() << "]:" << S.getTotalSamples();
849 else
850 OS << S.getFunction() << ":" << S.getTotalSamples();
851
852 if (Indent == 0)
853 OS << ":" << S.getHeadSamples();
854 OS << "\n";
855 LineCount++;
856
857 for (const auto &[Loc, Sample] : S.getBodySamples()) {
858 OS.indent(NumSpaces: Indent + 1);
859 Loc.print(OS);
860 OS << ": " << Sample.getSamples();
861
862 for (const auto &J : Sample.getSortedCallTargets())
863 OS << " " << J.first << ":" << J.second;
864 OS << "\n";
865 LineCount++;
866
867 if (const TypeCountMap *Map = S.findCallsiteTypeSamplesAt(Loc);
868 Map && !Map->empty()) {
869 OS.indent(NumSpaces: Indent + 1);
870 Loc.print(OS);
871 OS << ": ";
872 OS << kVTableProfPrefix;
873 for (const auto &[TypeName, Count] : *Map) {
874 OS << TypeName << ":" << Count << " ";
875 }
876 OS << "\n";
877 LineCount++;
878 }
879 }
880
881 Indent += 1;
882 for (const auto &[Loc, FunctionSamplesMap] : S.getCallsiteSamples()) {
883 for (const FunctionSamples &CalleeSamples :
884 make_second_range(c: FunctionSamplesMap)) {
885 OS.indent(NumSpaces: Indent);
886 Loc.print(OS);
887 OS << ": ";
888 if (std::error_code EC = writeSample(S: CalleeSamples))
889 return EC;
890 }
891
892 if (const TypeCountMap *Map = S.findCallsiteTypeSamplesAt(Loc);
893 Map && !Map->empty()) {
894 OS.indent(NumSpaces: Indent);
895 Loc.print(OS);
896 OS << ": ";
897 OS << kVTableProfPrefix;
898 for (const auto &[TypeId, Count] : *Map) {
899 OS << TypeId << ":" << Count << " ";
900 }
901 OS << "\n";
902 LineCount++;
903 }
904 }
905
906 Indent -= 1;
907
908 if (FunctionSamples::ProfileIsProbeBased) {
909 OS.indent(NumSpaces: Indent + 1);
910 OS << "!CFGChecksum: " << S.getFunctionHash() << "\n";
911 LineCount++;
912 }
913
914 if (S.getContext().getAllAttributes()) {
915 OS.indent(NumSpaces: Indent + 1);
916 OS << "!Attributes: " << S.getContext().getAllAttributes() << "\n";
917 LineCount++;
918 }
919
920 if (Indent == 0 && MarkFlatProfiles && S.getCallsiteSamples().size() == 0)
921 OS << " !Flat\n";
922
923 return sampleprof_error::success;
924}
925
926std::error_code
927SampleProfileWriterBinary::writeContextIdx(const SampleContext &Context) {
928 assert(!Context.hasContext() && "cs profile is not supported");
929 return writeNameIdx(FName: Context.getFunction());
930}
931
932std::error_code SampleProfileWriterBinary::writeNameIdx(FunctionId FName) {
933 auto &NTable = getNameTable();
934 const auto &Ret = NTable.find(Key: FName);
935 if (Ret == NTable.end())
936 return sampleprof_error::truncated_name_table;
937 encodeULEB128(Value: Ret->second, OS&: *OutputStream);
938 return sampleprof_error::success;
939}
940
941void SampleProfileWriterBinary::addName(FunctionId FName) {
942 auto &NTable = getNameTable();
943 NTable.insert(KV: std::make_pair(x&: FName, y: 0));
944}
945
946void SampleProfileWriterBinary::addContext(const SampleContext &Context) {
947 addName(FName: Context.getFunction());
948}
949
950void SampleProfileWriterBinary::addNames(const FunctionSamples &S) {
951 // Add all the names in indirect call targets.
952 for (const auto &I : S.getBodySamples()) {
953 const SampleRecord &Sample = I.second;
954 for (const auto &J : Sample.getCallTargets())
955 addName(FName: J.first);
956 }
957
958 // Recursively add all the names for inlined callsites.
959 for (const auto &J : S.getCallsiteSamples())
960 for (const auto &FS : J.second) {
961 const FunctionSamples &CalleeSamples = FS.second;
962 addName(FName: CalleeSamples.getFunction());
963 addNames(S: CalleeSamples);
964 }
965
966 if (!WriteVTableProf)
967 return;
968 // Add all the vtable names to NameTable.
969 for (const auto &VTableAccessCountMap :
970 llvm::make_second_range(c: S.getCallsiteTypeCounts())) {
971 // Add type name to NameTable.
972 for (const auto Type : llvm::make_first_range(c: VTableAccessCountMap)) {
973 addName(FName: Type);
974 }
975 }
976}
977
978void SampleProfileWriterExtBinaryBase::addContext(
979 const SampleContext &Context) {
980 if (Context.hasContext()) {
981 for (auto &Callsite : Context.getContextFrames())
982 SampleProfileWriterBinary::addName(FName: Callsite.Func);
983 CSNameTable.insert(KV: std::make_pair(x: Context, y: 0));
984 } else {
985 SampleProfileWriterBinary::addName(FName: Context.getFunction());
986 }
987}
988
989std::error_code SampleProfileWriterBinary::writeNameTable() {
990 auto &OS = *OutputStream;
991
992 // Write out the name table.
993 encodeULEB128(Value: NameTable.size(), OS);
994 for (const auto *Entry : stabilizeTable(Table&: NameTable)) {
995 OS << Entry->first;
996 encodeULEB128(Value: 0, OS);
997 }
998 return sampleprof_error::success;
999}
1000
1001std::error_code
1002SampleProfileWriterBinary::writeMagicIdent(SampleProfileFormat Format) {
1003 auto &OS = *OutputStream;
1004 // Write file magic identifier.
1005 encodeULEB128(Value: SPMagic(Format), OS);
1006 encodeULEB128(Value: FormatVersion, OS);
1007 return sampleprof_error::success;
1008}
1009
1010std::error_code
1011SampleProfileWriterBinary::writeHeader(const SampleProfileMap &ProfileMap) {
1012 // When calling write on a different profile map, existing names should be
1013 // cleared.
1014 NameTable.clear();
1015
1016 writeMagicIdent(Format);
1017
1018 computeSummary(ProfileMap);
1019 if (auto EC = writeSummary())
1020 return EC;
1021
1022 // Generate the name table for all the functions referenced in the profile.
1023 for (const auto &I : ProfileMap) {
1024 addContext(Context: I.second.getContext());
1025 addNames(S: I.second);
1026 }
1027
1028 writeNameTable();
1029 return sampleprof_error::success;
1030}
1031
1032void SampleProfileWriterExtBinaryBase::setToCompressAllSections() {
1033 for (auto &Entry : SectionHdrLayout)
1034 addSecFlag(Entry, Flag: SecCommonFlags::SecFlagCompress);
1035}
1036
1037void SampleProfileWriterExtBinaryBase::setToCompressSection(SecType Type) {
1038 addSectionFlag(Type, Flag: SecCommonFlags::SecFlagCompress);
1039}
1040
1041void SampleProfileWriterExtBinaryBase::allocSecHdrTable() {
1042 support::endian::Writer Writer(*OutputStream, llvm::endianness::little);
1043
1044 Writer.write(Val: static_cast<uint64_t>(SectionHdrLayout.size()));
1045 SecHdrTableOffset = OutputStream->tell();
1046 for (uint32_t i = 0; i < SectionHdrLayout.size(); i++) {
1047 Writer.write(Val: static_cast<uint64_t>(-1));
1048 Writer.write(Val: static_cast<uint64_t>(-1));
1049 Writer.write(Val: static_cast<uint64_t>(-1));
1050 Writer.write(Val: static_cast<uint64_t>(-1));
1051 }
1052}
1053
1054std::error_code SampleProfileWriterExtBinaryBase::writeSecHdrTable() {
1055 assert(SecHdrTable.size() == SectionHdrLayout.size() &&
1056 "SecHdrTable entries doesn't match SectionHdrLayout");
1057 SmallVector<uint32_t, 16> IndexMap(SecHdrTable.size(), -1);
1058 for (uint32_t TableIdx = 0; TableIdx < SecHdrTable.size(); TableIdx++) {
1059 IndexMap[SecHdrTable[TableIdx].LayoutIndex] = TableIdx;
1060 }
1061
1062 // Write the section header table in the order specified in
1063 // SectionHdrLayout. SectionHdrLayout specifies the sections
1064 // order in which profile reader expect to read, so the section
1065 // header table should be written in the order in SectionHdrLayout.
1066 // Note that the section order in SecHdrTable may be different
1067 // from the order in SectionHdrLayout, for example, SecFuncOffsetTable
1068 // needs to be computed after SecLBRProfile (the order in SecHdrTable),
1069 // but it needs to be read before SecLBRProfile (the order in
1070 // SectionHdrLayout). So we use IndexMap above to switch the order.
1071 support::endian::SeekableWriter Writer(
1072 static_cast<raw_pwrite_stream &>(*OutputStream),
1073 llvm::endianness::little);
1074 for (uint32_t LayoutIdx = 0; LayoutIdx < SectionHdrLayout.size();
1075 LayoutIdx++) {
1076 assert(IndexMap[LayoutIdx] < SecHdrTable.size() &&
1077 "Incorrect LayoutIdx in SecHdrTable");
1078 auto Entry = SecHdrTable[IndexMap[LayoutIdx]];
1079 Writer.pwrite(Val: static_cast<uint64_t>(Entry.Type),
1080 Offset: SecHdrTableOffset + 4 * LayoutIdx * sizeof(uint64_t));
1081 Writer.pwrite(Val: static_cast<uint64_t>(Entry.Flags),
1082 Offset: SecHdrTableOffset + (4 * LayoutIdx + 1) * sizeof(uint64_t));
1083 Writer.pwrite(Val: static_cast<uint64_t>(Entry.Offset),
1084 Offset: SecHdrTableOffset + (4 * LayoutIdx + 2) * sizeof(uint64_t));
1085 Writer.pwrite(Val: static_cast<uint64_t>(Entry.Size),
1086 Offset: SecHdrTableOffset + (4 * LayoutIdx + 3) * sizeof(uint64_t));
1087 }
1088
1089 return sampleprof_error::success;
1090}
1091
1092std::error_code SampleProfileWriterExtBinaryBase::writeHeader(
1093 const SampleProfileMap &ProfileMap) {
1094 // Reject a version that cannot describe the selected profile encoding before
1095 // emitting any part of the header.
1096 if (WriteCompositeProf && FormatVersion < CompositeProfileVersion)
1097 return sampleprof_error::unsupported_version;
1098
1099 auto &OS = *OutputStream;
1100 FileStart = OS.tell();
1101 writeMagicIdent(Format);
1102
1103 allocSecHdrTable();
1104 return sampleprof_error::success;
1105}
1106
1107std::error_code SampleProfileWriterBinary::writeCallsiteVTableProf(
1108 const CallsiteTypeMap &CallsiteTypeMap, raw_ostream &OS) {
1109 assert(WriteVTableProf &&
1110 "writeCallsiteVTableProf should not be called if WriteVTableProf is "
1111 "false");
1112
1113 encodeULEB128(Value: CallsiteTypeMap.size(), OS);
1114 for (const auto &[Loc, TypeMap] : CallsiteTypeMap) {
1115 Loc.serialize(OS);
1116 if (std::error_code EC = serializeTypeMap(Map: TypeMap, NameTable: getNameTable(), OS))
1117 return EC;
1118 }
1119
1120 return sampleprof_error::success;
1121}
1122
1123std::error_code SampleProfileWriterBinary::writeSummary() {
1124 auto &OS = *OutputStream;
1125 encodeULEB128(Value: Summary->getTotalCount(), OS);
1126 encodeULEB128(Value: Summary->getMaxCount(), OS);
1127 encodeULEB128(Value: Summary->getMaxFunctionCount(), OS);
1128 encodeULEB128(Value: Summary->getNumCounts(), OS);
1129 encodeULEB128(Value: Summary->getNumFunctions(), OS);
1130 ArrayRef<ProfileSummaryEntry> Entries = Summary->getDetailedSummary();
1131 encodeULEB128(Value: Entries.size(), OS);
1132 for (auto Entry : Entries) {
1133 encodeULEB128(Value: Entry.Cutoff, OS);
1134 encodeULEB128(Value: Entry.MinCount, OS);
1135 encodeULEB128(Value: Entry.NumCounts, OS);
1136 }
1137 return sampleprof_error::success;
1138}
1139
1140std::error_code
1141SampleProfileWriterBinary::writeLBRProfile(const FunctionSamples &S,
1142 bool IsNested) {
1143 auto &OS = *OutputStream;
1144 if (WriteCompositeProf && !IsNested)
1145 encodeULEB128(Value: S.getHeadSamples(), OS);
1146 encodeULEB128(Value: S.getTotalSamples(), OS);
1147 encodeULEB128(Value: S.getBodySamples().size(), OS);
1148 for (const auto &I : S.getBodySamples()) {
1149 LineLocation Loc = I.first;
1150 const SampleRecord &Sample = I.second;
1151 Loc.serialize(OS);
1152 if (std::error_code EC = Sample.serialize(OS, NameTable: getNameTable()))
1153 return EC;
1154 }
1155 return sampleprof_error::success;
1156}
1157
1158namespace {
1159
1160/// A reusable stream that discards payload bytes while counting their size.
1161class PayloadSizeCountingStream final : public raw_ostream {
1162public:
1163 /// Avoid retaining payload data in raw_ostream's internal buffer.
1164 PayloadSizeCountingStream() { SetUnbuffered(); }
1165
1166 /// Prepare the stream to count another payload.
1167 void resetPayload() {
1168 PayloadSize = 0;
1169 Overflowed = false;
1170 }
1171
1172 /// Return whether the payload size exceeded the representable range.
1173 bool overflowed() const { return Overflowed; }
1174
1175 /// Return the complete payload size when overflowed() is false.
1176 uint64_t payloadSize() const { return PayloadSize; }
1177
1178private:
1179 /// Count incoming bytes without retaining their contents.
1180 void write_impl(const char *, size_t Size) override {
1181 if (Overflowed)
1182 return;
1183
1184 // Fail closed if the payload cannot be represented by its uint64_t size.
1185 if (Size > UINT64_MAX - PayloadSize) {
1186 Overflowed = true;
1187 return;
1188 }
1189 PayloadSize += Size;
1190 }
1191
1192 /// Report the number of bytes accepted from the current payload.
1193 uint64_t current_pos() const override { return PayloadSize; }
1194
1195 /// Number of bytes observed during the counting pass.
1196 uint64_t PayloadSize = 0;
1197 /// Whether the counted size no longer fits in uint64_t.
1198 bool Overflowed = false;
1199};
1200
1201} // namespace
1202
1203std::error_code SampleProfileWriterBinary::writeProfileType(
1204 ProfTypes Type, function_ref<std::error_code()> WritePayload) {
1205 // PayloadSizeStream temporarily owns the real output while the callback
1206 // writes through OutputStream. A nested call would therefore mistake the
1207 // real output for PayloadSizeCountingStream.
1208 if (WritingProfileType)
1209 return sampleprof_error::malformed;
1210 SaveAndRestore RestoreWritingProfileType(WritingProfileType, true);
1211
1212 // A profile block stores its payload size before the payload, but that size
1213 // is not known until it has been serialized. Count one complete serialization
1214 // without retaining its bytes, then emit the header and serialize it again.
1215 // TODO: Avoid serializing each payload twice while retaining bounded memory
1216 // use and compatibility with compressed section output.
1217 if (!PayloadSizeStream)
1218 PayloadSizeStream = std::make_unique<PayloadSizeCountingStream>();
1219 auto *SizeStream =
1220 static_cast<PayloadSizeCountingStream *>(PayloadSizeStream.get());
1221 SizeStream->resetPayload();
1222 OutputStream.swap(u&: PayloadSizeStream);
1223 std::error_code EC = WritePayload();
1224 OutputStream.swap(u&: PayloadSizeStream);
1225 if (EC)
1226 return EC;
1227 if (SizeStream->overflowed())
1228 return sampleprof_error::too_large;
1229
1230 // Emit the compact header followed by the second, materialized pass.
1231 auto &OS = *OutputStream;
1232 encodeULEB128(Value: Type, OS);
1233 encodeULEB128(Value: SizeStream->payloadSize(), OS);
1234 uint64_t PayloadStart = OS.tell();
1235 if (std::error_code SecondPassEC = WritePayload())
1236 return SecondPassEC;
1237
1238 // Reject a stateful callback that did not reproduce the counted payload.
1239 if (OS.tell() - PayloadStart != SizeStream->payloadSize())
1240 return sampleprof_error::malformed;
1241 return sampleprof_error::success;
1242}
1243
1244static bool hasNonEmptyLBRProfile(const FunctionSamples &S, bool IsNested) {
1245 return S.getTotalSamples() != 0 || (!IsNested && S.getHeadSamples() != 0) ||
1246 !S.getBodySamples().empty();
1247}
1248
1249std::error_code
1250SampleProfileWriterBinary::writeCompositeProfile(const FunctionSamples &S,
1251 bool IsNested) {
1252 auto &OS = *OutputStream;
1253 bool WriteLBRProf = hasNonEmptyLBRProfile(S, IsNested);
1254 // Other profile types should be added here.
1255 uint32_t TypesNum = WriteLBRProf;
1256
1257 // Write the number of profile types for function.
1258 encodeULEB128(Value: TypesNum, OS);
1259
1260 if (WriteLBRProf)
1261 return writeProfileType(Type: ProfTypeLBR,
1262 WritePayload: [&] { return writeLBRProfile(S, IsNested); });
1263 return sampleprof_error::success;
1264}
1265
1266std::error_code SampleProfileWriterBinary::writeBody(const FunctionSamples &S,
1267 bool IsNested) {
1268 auto &OS = *OutputStream;
1269 if (std::error_code EC = writeContextIdx(Context: S.getContext()))
1270 return EC;
1271
1272 // Emit all the body samples.
1273 if (WriteCompositeProf) {
1274 if (std::error_code EC = writeCompositeProfile(S, IsNested))
1275 return EC;
1276 } else {
1277 if (std::error_code EC = writeLBRProfile(S, IsNested))
1278 return EC;
1279 }
1280
1281 // Recursively emit all the callsite samples.
1282 uint64_t NumCallsites = 0;
1283 for (const auto &J : S.getCallsiteSamples())
1284 NumCallsites += J.second.size();
1285 encodeULEB128(Value: NumCallsites, OS);
1286 for (const auto &J : S.getCallsiteSamples())
1287 for (const auto &FS : J.second) {
1288 J.first.serialize(OS);
1289 if (std::error_code EC = writeBody(S: FS.second, /*IsNested=*/true))
1290 return EC;
1291 }
1292
1293 if (WriteVTableProf)
1294 return writeCallsiteVTableProf(CallsiteTypeMap: S.getCallsiteTypeCounts(), OS);
1295
1296 return sampleprof_error::success;
1297}
1298
1299/// Write samples of a top-level function to a binary file.
1300///
1301/// \returns true if the samples were written successfully, false otherwise.
1302std::error_code
1303SampleProfileWriterBinary::writeSample(const FunctionSamples &S) {
1304 encodeULEB128(Value: S.getHeadSamples(), OS&: *OutputStream);
1305 return writeBody(S, /*IsNested=*/false);
1306}
1307
1308/// Create a sample profile file writer based on the specified format.
1309///
1310/// \param Filename The file to create.
1311///
1312/// \param Format Encoding format for the profile file.
1313///
1314/// \returns an error code indicating the status of the created writer.
1315ErrorOr<std::unique_ptr<SampleProfileWriter>>
1316SampleProfileWriter::create(StringRef Filename, SampleProfileFormat Format) {
1317 std::error_code EC;
1318 std::unique_ptr<raw_ostream> OS;
1319 if (Format == SPF_Binary || Format == SPF_Ext_Binary)
1320 OS.reset(p: new raw_fd_ostream(Filename, EC, sys::fs::OF_None));
1321 else
1322 OS.reset(p: new raw_fd_ostream(Filename, EC, sys::fs::OF_TextWithCRLF));
1323 if (EC)
1324 return EC;
1325
1326 return create(OS, Format);
1327}
1328
1329/// Create a sample profile stream writer based on the specified format.
1330///
1331/// \param OS The output stream to store the profile data to.
1332///
1333/// \param Format Encoding format for the profile file.
1334///
1335/// \returns an error code indicating the status of the created writer.
1336ErrorOr<std::unique_ptr<SampleProfileWriter>>
1337SampleProfileWriter::create(std::unique_ptr<raw_ostream> &OS,
1338 SampleProfileFormat Format) {
1339 std::error_code EC;
1340 std::unique_ptr<SampleProfileWriter> Writer;
1341
1342 // Currently only Text and Extended Binary format are supported for CSSPGO.
1343 if ((FunctionSamples::ProfileIsCS || FunctionSamples::ProfileIsProbeBased) &&
1344 Format == SPF_Binary)
1345 return sampleprof_error::unsupported_writing_format;
1346
1347 if (Format == SPF_Binary)
1348 Writer.reset(p: new SampleProfileWriterRawBinary(OS));
1349 else if (Format == SPF_Ext_Binary)
1350 Writer.reset(p: new SampleProfileWriterExtBinary(OS));
1351 else if (Format == SPF_Text)
1352 Writer.reset(p: new SampleProfileWriterText(OS));
1353 else if (Format == SPF_GCC)
1354 EC = sampleprof_error::unsupported_writing_format;
1355 else
1356 EC = sampleprof_error::unrecognized_format;
1357
1358 if (EC)
1359 return EC;
1360
1361 Writer->Format = Format;
1362 if (Format != SPF_Ext_Binary) {
1363 Writer->setFormatVersion(DefaultVersion);
1364 } else {
1365 if (!formatVersionIsSupported(Version: RequestedVersion))
1366 return sampleprof_error::unsupported_version;
1367
1368 // Composite output defaults to its first compatible format version.
1369 // Preserve a compatible version explicitly selected by the user.
1370 if (ExtBinaryCompositeProf) {
1371 if (RequestedVersion.getNumOccurrences() == 0) {
1372 Writer->setFormatVersion(CompositeProfileVersion);
1373 } else {
1374 if (RequestedVersion < CompositeProfileVersion)
1375 return sampleprof_error::unsupported_version;
1376 Writer->setFormatVersion(RequestedVersion);
1377 }
1378 // Keep subsequent writes independent of the global command-line option.
1379 Writer->setUseCompositeProfile(true);
1380 } else {
1381 Writer->setFormatVersion(RequestedVersion);
1382 }
1383 }
1384
1385 return std::move(Writer);
1386}
1387
1388void SampleProfileWriter::computeSummary(const SampleProfileMap &ProfileMap) {
1389 SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs);
1390 Summary = Builder.computeSummaryForProfiles(Profiles: ProfileMap);
1391}
1392