1//===- SampleProfWriter.cpp - Write LLVM sample profile data --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the class that writes LLVM sample profiles. It
10// supports two file formats: text and binary. The textual representation
11// is useful for debugging and testing purposes. The binary representation
12// is more compact, resulting in smaller file sizes. However, they can
13// both be used interchangeably.
14//
15// See lib/ProfileData/SampleProfReader.cpp for documentation on each of the
16// supported formats.
17//
18//===----------------------------------------------------------------------===//
19
20#include "llvm/ProfileData/SampleProfWriter.h"
21#include "llvm/ADT/Eytzinger.h"
22#include "llvm/ADT/StringRef.h"
23#include "llvm/ProfileData/ProfileCommon.h"
24#include "llvm/ProfileData/SampleProf.h"
25#include "llvm/Support/Compression.h"
26#include "llvm/Support/EndianStream.h"
27#include "llvm/Support/ErrorOr.h"
28#include "llvm/Support/FileSystem.h"
29#include "llvm/Support/LEB128.h"
30#include "llvm/Support/MD5.h"
31#include "llvm/Support/SaveAndRestore.h"
32#include "llvm/Support/raw_ostream.h"
33#include <array>
34#include <cmath>
35#include <cstdint>
36#include <memory>
37#include <system_error>
38#include <utility>
39#include <vector>
40
41#define DEBUG_TYPE "llvm-profdata"
42
43using namespace llvm;
44using namespace sampleprof;
45
46// To begin with, make this option off by default.
47static cl::opt<bool> ExtBinaryWriteVTableTypeProf(
48 "extbinary-write-vtable-type-prof", cl::init(Val: false), cl::Hidden,
49 cl::desc("Write vtable type profile in ext-binary sample profile writer"));
50
51static cl::opt<uint64_t> RequestedVersion(
52 "sample-profile-format-version", cl::init(Val: DefaultVersion), cl::Hidden,
53 cl::desc("Format version to write for extensible binary profiles"));
54
55static cl::opt<bool>
56 ExtBinaryCompositeProf("extbinary-composite-prof", cl::init(Val: false),
57 cl::Hidden,
58 cl::desc("Use the composite profile format"));
59
60namespace llvm {
61namespace support {
62namespace endian {
63namespace {
64
65// Adapter class to llvm::support::endian::Writer for pwrite().
66struct SeekableWriter {
67 raw_pwrite_stream &OS;
68 endianness Endian;
69 SeekableWriter(raw_pwrite_stream &OS, endianness Endian)
70 : OS(OS), Endian(Endian) {}
71
72 template <typename ValueType> void pwrite(ValueType Val, size_t Offset) {
73 std::string StringBuf;
74 raw_string_ostream SStream(StringBuf);
75 Writer(SStream, Endian).write(Val);
76 OS.pwrite(Ptr: StringBuf.data(), Size: StringBuf.size(), Offset);
77 }
78};
79
80} // namespace
81} // namespace endian
82} // namespace support
83} // namespace llvm
84
85DefaultFunctionPruningStrategy::DefaultFunctionPruningStrategy(
86 SampleProfileMap &ProfileMap, size_t OutputSizeLimit)
87 : FunctionPruningStrategy(ProfileMap, OutputSizeLimit) {
88 sortFuncProfiles(ProfileMap, SortedProfiles&: SortedFunctions);
89}
90
91void DefaultFunctionPruningStrategy::Erase(size_t CurrentOutputSize) {
92 double D = (double)OutputSizeLimit / CurrentOutputSize;
93 size_t NewSize = (size_t)round(x: ProfileMap.size() * D * D);
94 size_t NumToRemove = ProfileMap.size() - NewSize;
95 if (NumToRemove < 1)
96 NumToRemove = 1;
97
98 assert(NumToRemove <= SortedFunctions.size());
99 for (const NameFunctionSamples &E :
100 llvm::drop_begin(RangeOrContainer&: SortedFunctions, N: SortedFunctions.size() - NumToRemove))
101 ProfileMap.erase(Key: E.first);
102 SortedFunctions.resize(new_size: SortedFunctions.size() - NumToRemove);
103}
104
105std::error_code SampleProfileWriter::writeWithSizeLimitInternal(
106 SampleProfileMap &ProfileMap, size_t OutputSizeLimit,
107 FunctionPruningStrategy *Strategy) {
108 if (OutputSizeLimit == 0)
109 return write(ProfileMap);
110
111 size_t OriginalFunctionCount = ProfileMap.size();
112
113 std::unique_ptr<raw_ostream> OriginalOutputStream;
114 OutputStream.swap(u&: OriginalOutputStream);
115
116 size_t IterationCount = 0;
117 size_t TotalSize;
118
119 SmallVector<char> StringBuffer;
120 do {
121 StringBuffer.clear();
122 OutputStream.reset(p: new raw_svector_ostream(StringBuffer));
123 if (std::error_code EC = write(ProfileMap))
124 return EC;
125
126 TotalSize = StringBuffer.size();
127 // On Windows every "\n" is actually written as "\r\n" to disk but not to
128 // memory buffer, this difference should be added when considering the total
129 // output size.
130#ifdef _WIN32
131 if (Format == SPF_Text)
132 TotalSize += LineCount;
133#endif
134 if (TotalSize <= OutputSizeLimit)
135 break;
136
137 Strategy->Erase(CurrentOutputSize: TotalSize);
138 IterationCount++;
139 } while (ProfileMap.size() != 0);
140
141 if (ProfileMap.size() == 0)
142 return sampleprof_error::too_large;
143
144 OutputStream.swap(u&: OriginalOutputStream);
145 OutputStream->write(Ptr: StringBuffer.data(), Size: StringBuffer.size());
146 LLVM_DEBUG(dbgs() << "Profile originally has " << OriginalFunctionCount
147 << " functions, reduced to " << ProfileMap.size() << " in "
148 << IterationCount << " iterations\n");
149 // Silence warning on Release build.
150 (void)OriginalFunctionCount;
151 (void)IterationCount;
152 return sampleprof_error::success;
153}
154
155std::error_code
156SampleProfileWriter::writeFuncProfiles(const SampleProfileMap &ProfileMap) {
157 std::vector<NameFunctionSamples> V;
158 sortFuncProfiles(ProfileMap, SortedProfiles&: V);
159 for (const auto &I : V) {
160 if (std::error_code EC = writeSample(S: *I.second))
161 return EC;
162 }
163 return sampleprof_error::success;
164}
165
166std::error_code SampleProfileWriter::write(const SampleProfileMap &ProfileMap) {
167 if (std::error_code EC = writeHeader(ProfileMap))
168 return EC;
169
170 if (std::error_code EC = writeFuncProfiles(ProfileMap))
171 return EC;
172
173 return sampleprof_error::success;
174}
175
176/// Return the current position and prepare to use it as the start
177/// position of a section given the section type \p Type and its position
178/// \p LayoutIdx in SectionHdrLayout.
179uint64_t
180SampleProfileWriterExtBinaryBase::markSectionStart(SecType Type,
181 uint32_t LayoutIdx) {
182 uint64_t SectionStart = OutputStream->tell();
183 assert(LayoutIdx < SectionHdrLayout.size() && "LayoutIdx out of range");
184 const auto &Entry = SectionHdrLayout[LayoutIdx];
185 assert(Entry.Type == Type && "Unexpected section type");
186 // Use LocalBuf as a temporary output for writing data.
187 if (hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagCompress))
188 LocalBufStream.swap(u&: OutputStream);
189 return SectionStart;
190}
191
192std::error_code SampleProfileWriterExtBinaryBase::compressAndOutput() {
193 if (!llvm::compression::zlib::isAvailable())
194 return sampleprof_error::zlib_unavailable;
195 std::string &UncompressedStrings =
196 static_cast<raw_string_ostream *>(LocalBufStream.get())->str();
197 if (UncompressedStrings.empty())
198 return sampleprof_error::success;
199 auto &OS = *OutputStream;
200 SmallVector<uint8_t, 128> CompressedStrings;
201 compression::zlib::compress(Input: arrayRefFromStringRef(Input: UncompressedStrings),
202 CompressedBuffer&: CompressedStrings,
203 Level: compression::zlib::BestSizeCompression);
204 encodeULEB128(Value: UncompressedStrings.size(), OS);
205 encodeULEB128(Value: CompressedStrings.size(), OS);
206 OS << toStringRef(Input: CompressedStrings);
207 UncompressedStrings.clear();
208 return sampleprof_error::success;
209}
210
211/// Add a new section into section header table given the section type
212/// \p Type, its position \p LayoutIdx in SectionHdrLayout and the
213/// location \p SectionStart where the section should be written to.
214std::error_code SampleProfileWriterExtBinaryBase::addNewSection(
215 SecType Type, uint32_t LayoutIdx, uint64_t SectionStart) {
216 assert(LayoutIdx < SectionHdrLayout.size() && "LayoutIdx out of range");
217 const auto &Entry = SectionHdrLayout[LayoutIdx];
218 assert(Entry.Type == Type && "Unexpected section type");
219 if (hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagCompress)) {
220 LocalBufStream.swap(u&: OutputStream);
221 if (std::error_code EC = compressAndOutput())
222 return EC;
223 }
224 SecHdrTable.push_back(x: {.Type: Type, .Flags: Entry.Flags, .Offset: SectionStart - FileStart,
225 .Size: OutputStream->tell() - SectionStart, .LayoutIndex: LayoutIdx});
226 return sampleprof_error::success;
227}
228
229std::error_code
230SampleProfileWriterExtBinaryBase::write(const SampleProfileMap &ProfileMap) {
231 // When calling write on a different profile map, existing states should be
232 // cleared.
233 NameTable.clear();
234 CSNameTable.clear();
235 SecHdrTable.clear();
236
237 if (std::error_code EC = writeHeader(ProfileMap))
238 return EC;
239
240 std::string LocalBuf;
241 LocalBufStream = std::make_unique<raw_string_ostream>(args&: LocalBuf);
242 if (std::error_code EC = writeSections(ProfileMap))
243 return EC;
244
245 if (std::error_code EC = writeSecHdrTable())
246 return EC;
247
248 return sampleprof_error::success;
249}
250
251std::error_code SampleProfileWriterExtBinaryBase::writeContextIdx(
252 const SampleContext &Context) {
253 if (Context.hasContext())
254 return writeCSNameIdx(Context);
255 else
256 return SampleProfileWriterBinary::writeNameIdx(FName: Context.getFunction());
257}
258
259std::error_code
260SampleProfileWriterExtBinaryBase::writeCSNameIdx(const SampleContext &Context) {
261 const auto &Ret = CSNameTable.find(Key: Context);
262 if (Ret == CSNameTable.end())
263 return sampleprof_error::truncated_name_table;
264 encodeULEB128(Value: Ret->second, OS&: *OutputStream);
265 return sampleprof_error::success;
266}
267
268std::error_code
269SampleProfileWriterExtBinaryBase::writeSample(const FunctionSamples &S) {
270 uint64_t Offset = OutputStream->tell();
271 auto &Context = S.getContext();
272 FuncOffsetTable[Context] = Offset - SecLBRProfileStart;
273 if (!WriteCompositeProf)
274 encodeULEB128(Value: S.getHeadSamples(), OS&: *OutputStream);
275 return writeBody(S, /*IsNested=*/false);
276}
277
278std::error_code
279SampleProfileWriterExtBinaryBase::writeFuncOffsetTable(SecType Type,
280 bool IsNested) {
281 if (UseMD5IndexedTables) {
282 // Eytzinger layout requires MD5 representation and does not support
283 // multi-context Context-Sensitive profiles.
284 if (!UseMD5 || FunctionSamples::ProfileIsCS)
285 return sampleprof_error::unsupported_writing_format;
286 return writeEytzingerFuncOffsetTable(Type, IsNested);
287 }
288 return writeLegacyFuncOffsetTable(Type);
289}
290
291std::error_code
292SampleProfileWriterExtBinaryBase::writeEytzingerFuncOffsetTable(SecType Type,
293 bool IsNested) {
294 assert((NumNested + NumFlat > 0 || FuncOffsetTable.empty()) &&
295 "SecNameTable must be written before SecFuncOffsetTable to establish "
296 "Eytzinger indices!");
297
298 size_t SpanSize = IsNested ? NumNested : NumFlat;
299 size_t BaseIdx = IsNested ? 0 : NumNested;
300
301 std::vector<support::ulittle32_t> FuncOffsets(
302 SpanSize, support::ulittle32_t(UINT32_MAX));
303
304 // Populate the function offset array parallel to the Eytzinger span.
305 for (const auto &[Context, RelativeOffset] : FuncOffsetTable) {
306 if (RelativeOffset >= UINT32_MAX)
307 return sampleprof_error::too_large;
308
309 FunctionId FId = Context.getFunction();
310 auto It = NameTable.find(Key: FId);
311 if (It == NameTable.end())
312 continue;
313
314 size_t GlobalIdx = It->second;
315 if (GlobalIdx < BaseIdx || (GlobalIdx - BaseIdx) >= SpanSize)
316 continue;
317
318 size_t LocalIdx = GlobalIdx - BaseIdx;
319 assert(
320 FuncOffsets[LocalIdx] == UINT32_MAX &&
321 "Function offset slot already populated; duplicate GUID or collision!");
322 FuncOffsets[LocalIdx] = static_cast<uint32_t>(RelativeOffset);
323 }
324
325 assert(!llvm::is_contained(FuncOffsets, support::ulittle32_t(UINT32_MAX)) &&
326 "Unpopulated slot in Eytzinger function offset array!");
327
328 OutputStream->write(Ptr: reinterpret_cast<const char *>(FuncOffsets.data()),
329 Size: SpanSize * sizeof(support::ulittle32_t));
330 // Type is SecFuncOffsetTable or SecCompositeFuncOffsetTable.
331 addSectionFlag(Type, Flag: SecFuncOffsetFlags::SecFlagEytzinger);
332 FuncOffsetTable.clear();
333 return sampleprof_error::success;
334}
335
336std::error_code
337SampleProfileWriterExtBinaryBase::writeLegacyFuncOffsetTable(SecType Type) {
338 auto &OS = *OutputStream;
339
340 // Write out the table size.
341 encodeULEB128(Value: FuncOffsetTable.size(), OS);
342
343 // Write out FuncOffsetTable.
344 auto WriteItem = [&](const SampleContext &Context, uint64_t Offset) {
345 if (std::error_code EC = writeContextIdx(Context))
346 return EC;
347 encodeULEB128(Value: Offset, OS);
348 return (std::error_code)sampleprof_error::success;
349 };
350
351 if (FunctionSamples::ProfileIsCS) {
352 // Sort the contexts before writing them out. This is to help fast load all
353 // context profiles for a function as well as their callee contexts which
354 // can help profile-guided importing for ThinLTO.
355 std::map<SampleContext, uint64_t> OrderedFuncOffsetTable(
356 FuncOffsetTable.begin(), FuncOffsetTable.end());
357 for (const auto &Entry : OrderedFuncOffsetTable) {
358 if (std::error_code EC = WriteItem(Entry.first, Entry.second))
359 return EC;
360 }
361 addSectionFlag(Type, Flag: SecFuncOffsetFlags::SecFlagOrdered);
362 } else {
363 for (const auto &Entry : FuncOffsetTable) {
364 if (std::error_code EC = WriteItem(Entry.first, Entry.second))
365 return EC;
366 }
367 }
368
369 FuncOffsetTable.clear();
370 return sampleprof_error::success;
371}
372
373std::error_code SampleProfileWriterExtBinaryBase::writeFuncMetadata(
374 const FunctionSamples &FunctionProfile) {
375 auto &OS = *OutputStream;
376 if (std::error_code EC = writeContextIdx(Context: FunctionProfile.getContext()))
377 return EC;
378
379 if (FunctionSamples::ProfileIsProbeBased)
380 encodeULEB128(Value: FunctionProfile.getFunctionHash(), OS);
381 if (FunctionSamples::ProfileIsCS || FunctionSamples::ProfileIsPreInlined) {
382 encodeULEB128(Value: FunctionProfile.getContext().getAllAttributes(), OS);
383 }
384
385 if (!FunctionSamples::ProfileIsCS) {
386 // Recursively emit attributes for all callee samples.
387 uint64_t NumCallsites = 0;
388 for (const auto &J : FunctionProfile.getCallsiteSamples())
389 NumCallsites += J.second.size();
390 encodeULEB128(Value: NumCallsites, OS);
391 for (const auto &J : FunctionProfile.getCallsiteSamples()) {
392 for (const auto &FS : J.second) {
393 LineLocation Loc = J.first;
394 encodeULEB128(Value: Loc.LineOffset, OS);
395 encodeULEB128(Value: Loc.Discriminator, OS);
396 if (std::error_code EC = writeFuncMetadata(FunctionProfile: FS.second))
397 return EC;
398 }
399 }
400 }
401
402 return sampleprof_error::success;
403}
404
405std::error_code SampleProfileWriterExtBinaryBase::writeFuncMetadata(
406 const SampleProfileMap &Profiles) {
407 if (!FunctionSamples::ProfileIsProbeBased && !FunctionSamples::ProfileIsCS &&
408 !FunctionSamples::ProfileIsPreInlined)
409 return sampleprof_error::success;
410 for (const auto &Entry : Profiles) {
411 if (std::error_code EC = writeFuncMetadata(FunctionProfile: Entry.second))
412 return EC;
413 }
414 return sampleprof_error::success;
415}
416
417template <class KeyT, class ValT>
418static SmallVector<std::pair<KeyT, ValT> *, 0>
419stabilizeTable(MapVector<KeyT, ValT> &Table) {
420 SmallVector<std::pair<KeyT, ValT> *, 0> Entries(
421 llvm::make_pointer_range(Table));
422
423 llvm::sort(Entries,
424 [](const auto *L, const auto *R) { return L->first < R->first; });
425
426 for (const auto &[I, Entry] : llvm::enumerate(Entries))
427 Entry->second = I;
428
429 return Entries;
430}
431
432std::error_code SampleProfileWriterExtBinaryBase::writeNameTable() {
433 if (!UseMD5)
434 return SampleProfileWriterBinary::writeNameTable();
435
436 auto &OS = *OutputStream;
437
438 // Write out the MD5 name table. We wrote unencoded MD5 so reader can
439 // retrieve the name using the name index without having to read the
440 // whole name table.
441 encodeULEB128(Value: NameTable.size(), OS);
442 support::endian::Writer Writer(OS, llvm::endianness::little);
443 for (const auto *Entry : stabilizeTable(Table&: NameTable))
444 Writer.write(Val: Entry->first.getHashCode());
445 return sampleprof_error::success;
446}
447
448std::error_code SampleProfileWriterExtBinaryBase::writeNameTableSection(
449 const SampleProfileMap &ProfileMap) {
450 for (const auto &I : ProfileMap) {
451 addContext(Context: I.second.getContext());
452 addNames(S: I.second);
453 }
454
455 // If NameTable contains ".__uniq." suffix, set SecFlagUniqSuffix flag
456 // so compiler won't strip the suffix during profile matching after
457 // seeing the flag in the profile.
458 // Original names are unavailable if using MD5, so this option has no use.
459 if (!UseMD5) {
460 for (const auto &I : NameTable) {
461 if (I.first.stringRef().contains(Other: FunctionSamples::UniqSuffix)) {
462 addSectionFlag(Type: SecNameTable, Flag: SecNameTableFlags::SecFlagUniqSuffix);
463 break;
464 }
465 }
466 }
467
468 if (UseMD5 && UseMD5IndexedTables) {
469 // Eytzinger name tables do not support CSSPGO profiles
470 // (FunctionSamples::ProfileIsCS).
471 if (FunctionSamples::ProfileIsCS)
472 return sampleprof_error::unsupported_writing_format;
473 if (auto EC = writeEytzingerNameTableSection(ProfileMap))
474 return EC;
475 return sampleprof_error::success;
476 }
477
478 if (auto EC = writeNameTable())
479 return EC;
480 return sampleprof_error::success;
481}
482
483namespace {
484
485// Helper class to construct and write the SecNameTable section in Eytzinger
486// layout for ExtBinary MD5 profiles.
487//
488// The on-disk layout of the Eytzinger name table section consists of symbol
489// counts followed by three contiguous Eytzinger hash arrays:
490// - ULEB128 count of Nested top-level profile symbol keys
491// - ULEB128 count of Flat top-level profile symbol keys
492// - ULEB128 count of Inlinee and auxiliary profile symbol keys
493// - Array of 64-bit little-endian MD5 hash keys for Nested profiles in
494// Eytzinger order
495// - Array of 64-bit little-endian MD5 hash keys for Flat profiles in Eytzinger
496// order
497// - Array of 64-bit little-endian MD5 hash keys for Inlinees in Eytzinger order
498class EytzingerNameTable {
499 using TableT = llvm::EytzingerTable<support::ulittle64_t>;
500 std::array<TableT, static_cast<size_t>(EytzingerSpan::NumSpans)> Spans;
501
502public:
503 EytzingerNameTable(std::vector<support::ulittle64_t> NestedKeys,
504 std::vector<support::ulittle64_t> FlatKeys,
505 std::vector<support::ulittle64_t> InlineeKeys)
506 : Spans{TableT::create(Keys: std::move(NestedKeys)),
507 TableT::create(Keys: std::move(FlatKeys)),
508 TableT::create(Keys: std::move(InlineeKeys))} {}
509
510 // Find the global index of GUID across the three Eytzinger table spans.
511 uint64_t findGlobalIdx(uint64_t GUID) const {
512 uint64_t BaseIdx = 0;
513 for (const auto &Table : Spans) {
514 if (std::optional<size_t> LocalIdx = Table.findIndex(Target: GUID))
515 return BaseIdx + *LocalIdx;
516 BaseIdx += Table.size();
517 }
518 llvm_unreachable("Symbol in NameTable missing from Eytzinger spans");
519 }
520
521 void write(raw_ostream &OS) const {
522 for (const auto &Table : Spans)
523 encodeULEB128(Value: uint64_t(Table.size()), OS);
524 for (const auto &Table : Spans)
525 OS.write(Ptr: reinterpret_cast<const char *>(Table.data()),
526 Size: Table.size() * sizeof(support::ulittle64_t));
527 }
528
529 size_t size(EytzingerSpan S) const {
530 return Spans[static_cast<size_t>(S)].size();
531 }
532};
533
534} // end anonymous namespace
535
536std::error_code
537SampleProfileWriterExtBinaryBase::writeEytzingerNameTableSection(
538 const SampleProfileMap &ProfileMap) {
539 DenseSet<uint64_t> TopLevelGUIDs;
540 std::vector<support::ulittle64_t> NestedKeys, FlatKeys, InlineeKeys;
541
542 // Collect top-level Nested and Flat keys directly from ProfileMap.
543 for (const auto &I : ProfileMap) {
544 const SampleContext &Ctx = I.second.getContext();
545 uint64_t GUID = Ctx.getFunction().getHashCode();
546 if (TopLevelGUIDs.insert(V: GUID).second) {
547 // In single-table default layouts, unify all top-level symbols in the
548 // Nested partition so they match the single unflagged function offset
549 // table.
550 if (SecLayout != CtxSplitLayout || I.second.hasCallsiteSamples())
551 NestedKeys.emplace_back(args&: GUID);
552 else
553 FlatKeys.emplace_back(args&: GUID);
554 }
555 }
556
557 // Collect remaining non-top-level symbols (inlinees, targets, vtables) from
558 // NameTable.
559 for (const auto &Entry : NameTable) {
560 uint64_t GUID = Entry.first.getHashCode();
561 if (!TopLevelGUIDs.contains(V: GUID))
562 InlineeKeys.emplace_back(args&: GUID);
563 }
564
565 EytzingerNameTable Tables(std::move(NestedKeys), std::move(FlatKeys),
566 std::move(InlineeKeys));
567
568 // Assign each symbol its corresponding index in the Eytzinger layout.
569 for (auto &[FId, Idx] : NameTable)
570 Idx = Tables.findGlobalIdx(GUID: FId.getHashCode());
571
572 Tables.write(OS&: *OutputStream);
573 NumNested = Tables.size(S: EytzingerSpan::Nested);
574 NumFlat = Tables.size(S: EytzingerSpan::Flat);
575
576 return sampleprof_error::success;
577}
578
579std::error_code SampleProfileWriterExtBinaryBase::writeCSNameTableSection() {
580 auto &OS = *OutputStream;
581 encodeULEB128(Value: CSNameTable.size(), OS);
582 support::endian::Writer Writer(OS, llvm::endianness::little);
583 for (const auto *Entry : stabilizeTable(Table&: CSNameTable)) {
584 auto Frames = Entry->first.getContextFrames();
585 encodeULEB128(Value: Frames.size(), OS);
586 for (auto &Callsite : Frames) {
587 if (std::error_code EC = writeNameIdx(FName: Callsite.Func))
588 return EC;
589 encodeULEB128(Value: Callsite.Location.LineOffset, OS);
590 encodeULEB128(Value: Callsite.Location.Discriminator, OS);
591 }
592 }
593
594 return sampleprof_error::success;
595}
596
597std::error_code
598SampleProfileWriterExtBinaryBase::writeProfileSymbolListSection() {
599 if (UseMD5ProfSymList)
600 return writeMD5ProfileSymbolListSection();
601 return writeStringBasedProfileSymbolListSection();
602}
603
604std::error_code
605SampleProfileWriterExtBinaryBase::writeStringBasedProfileSymbolListSection() {
606 assert((!ProfSymList || !ProfSymList->isMD5()) &&
607 "Writing string-based ProfileSymbolListSection from MD5 table "
608 "not yet implemented");
609 if (ProfSymList && ProfSymList->size() > 0)
610 if (std::error_code EC = ProfSymList->write(OS&: *OutputStream))
611 return EC;
612
613 return sampleprof_error::success;
614}
615
616std::error_code
617SampleProfileWriterExtBinaryBase::writeMD5ProfileSymbolListSection() {
618 if (!ProfSymList || ProfSymList->size() == 0)
619 return sampleprof_error::success;
620
621 auto &OS = *OutputStream;
622 std::vector<uint64_t> Keys = ProfSymList->collectGUIDs();
623
624 auto Table =
625 llvm::EytzingerTable<support::ulittle64_t>::create(Keys: std::move(Keys));
626
627 OS.write(Ptr: reinterpret_cast<const char *>(Table.data()),
628 Size: Table.size() * sizeof(support::ulittle64_t));
629 return sampleprof_error::success;
630}
631
632unsigned SampleProfileWriterExtBinaryBase::findUnwrittenEntry(SecType Type) {
633 auto WrittenIndices =
634 llvm::map_range(C&: SecHdrTable, F: &SecHdrTableEntry::LayoutIndex);
635 for (auto [I, Entry] : llvm::enumerate(First&: SectionHdrLayout))
636 if (Entry.Type == Type && !llvm::is_contained(Range&: WrittenIndices, Element: I))
637 return I;
638 llvm_unreachable("Matching section not found in SectionHdrLayout");
639}
640
641std::error_code SampleProfileWriterExtBinaryBase::writeOneSection(
642 SecType Type, const SampleProfileMap &ProfileMap) {
643 unsigned LayoutIdx = findUnwrittenEntry(Type);
644 SecHdrTableEntry &Entry = SectionHdrLayout[LayoutIdx];
645
646 // The setting of SecFlagCompress should happen before markSectionStart.
647 if (Type == SecFuncMetadata && FunctionSamples::ProfileIsProbeBased)
648 addSectionFlag(Type: SecFuncMetadata, Flag: SecFuncMetadataFlags::SecFlagIsProbeBased);
649 if (Type == SecFuncMetadata &&
650 (FunctionSamples::ProfileIsCS || FunctionSamples::ProfileIsPreInlined))
651 addSectionFlag(Type: SecFuncMetadata, Flag: SecFuncMetadataFlags::SecFlagHasAttribute);
652 if (Type == SecProfSummary && FunctionSamples::ProfileIsCS)
653 addSectionFlag(Type: SecProfSummary, Flag: SecProfSummaryFlags::SecFlagFullContext);
654 if (Type == SecProfSummary && FunctionSamples::ProfileIsPreInlined)
655 addSectionFlag(Type: SecProfSummary, Flag: SecProfSummaryFlags::SecFlagIsPreInlined);
656 if (Type == SecProfSummary && FunctionSamples::ProfileIsFS)
657 addSectionFlag(Type: SecProfSummary, Flag: SecProfSummaryFlags::SecFlagFSDiscriminator);
658 if (Type == SecProfSummary && ExtBinaryWriteVTableTypeProf)
659 addSectionFlag(Type: SecProfSummary,
660 Flag: SecProfSummaryFlags::SecFlagHasVTableTypeProf);
661 if (Type == SecProfileSymbolList && UseMD5ProfSymList)
662 addSectionFlag(Type: SecProfileSymbolList, Flag: SecProfileSymbolListFlags::SecFlagMD5);
663 if (Type == SecNameTable && UseMD5IndexedTables && UseMD5)
664 addSectionFlag(Type: SecNameTable, Flag: SecNameTableFlags::SecFlagEytzinger);
665
666 uint64_t SectionStart = markSectionStart(Type, LayoutIdx);
667 switch (Type) {
668 case SecProfSummary:
669 computeSummary(ProfileMap);
670 if (auto EC = writeSummary())
671 return EC;
672 break;
673 case SecNameTable:
674 if (auto EC = writeNameTableSection(ProfileMap))
675 return EC;
676 break;
677 case SecCSNameTable:
678 if (auto EC = writeCSNameTableSection())
679 return EC;
680 break;
681 case SecLBRProfile:
682 case SecCompositeProfile:
683 SecLBRProfileStart = OutputStream->tell();
684 if (std::error_code EC = writeFuncProfiles(ProfileMap))
685 return EC;
686 break;
687 case SecFuncOffsetTable:
688 case SecCompositeFuncOffsetTable: {
689 bool IsFlat = hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagFlat);
690 // An unflagged function offset table inherently indexes the primary
691 // Nested symbol span.
692 bool IsNested = !IsFlat;
693 if (auto EC = writeFuncOffsetTable(Type, IsNested))
694 return EC;
695 break;
696 }
697 case SecFuncMetadata:
698 if (std::error_code EC = writeFuncMetadata(Profiles: ProfileMap))
699 return EC;
700 break;
701 case SecProfileSymbolList:
702 if (auto EC = writeProfileSymbolListSection())
703 return EC;
704 break;
705 default:
706 if (auto EC = writeCustomSection(Type))
707 return EC;
708 break;
709 }
710 if (std::error_code EC = addNewSection(Type, LayoutIdx, SectionStart))
711 return EC;
712 return sampleprof_error::success;
713}
714
715SampleProfileWriterExtBinary::SampleProfileWriterExtBinary(
716 std::unique_ptr<raw_ostream> &OS)
717 : SampleProfileWriterExtBinaryBase(OS) {
718 WriteVTableProf = ExtBinaryWriteVTableTypeProf;
719}
720
721std::error_code SampleProfileWriterExtBinary::writeDefaultLayout(
722 const SampleProfileMap &ProfileMap) {
723 // ProfSection / FuncOffsetSection are SecLBR* or SecComposite* after
724 // configureCompositeProfile.
725 const SecType Sections[] = {
726 SecProfSummary, SecNameTable, SecCSNameTable, ProfSection,
727 SecProfileSymbolList, FuncOffsetSection, SecFuncMetadata,
728 };
729 for (SecType Type : Sections)
730 if (std::error_code EC = writeOneSection(Type, ProfileMap))
731 return EC;
732 return sampleprof_error::success;
733}
734
735static void splitProfileMapToTwo(const SampleProfileMap &ProfileMap,
736 SampleProfileMap &NestedProfileMap,
737 SampleProfileMap &FlatProfileMap) {
738 for (const auto &I : ProfileMap) {
739 if (I.second.hasCallsiteSamples())
740 NestedProfileMap.insert(x: {I.first, I.second});
741 else
742 FlatProfileMap.insert(x: {I.first, I.second});
743 }
744}
745
746std::error_code SampleProfileWriterExtBinary::writeCtxSplitLayout(
747 const SampleProfileMap &ProfileMap) {
748 SampleProfileMap NestedProfileMap, FlatProfileMap;
749 splitProfileMapToTwo(ProfileMap, NestedProfileMap, FlatProfileMap);
750
751 // Flat SecFlag is pre-set in ExtBinaryHdrLayoutTable; findUnwrittenEntry
752 // picks the matching unwritten ProfSection / FuncOffsetSection slot.
753 const std::pair<SecType, const SampleProfileMap &> Sections[] = {
754 {SecProfSummary, ProfileMap}, {SecNameTable, ProfileMap},
755 {ProfSection, NestedProfileMap}, {FuncOffsetSection, NestedProfileMap},
756 {ProfSection, FlatProfileMap}, {FuncOffsetSection, FlatProfileMap},
757 {SecProfileSymbolList, ProfileMap}, {SecFuncMetadata, ProfileMap},
758 };
759 for (const auto &[Type, Map] : Sections)
760 if (std::error_code EC = writeOneSection(Type, ProfileMap: Map))
761 return EC;
762
763 return sampleprof_error::success;
764}
765
766void SampleProfileWriterExtBinary::configureCompositeProfile() {
767 ProfSection = WriteCompositeProf ? SecCompositeProfile : SecLBRProfile;
768 FuncOffsetSection =
769 WriteCompositeProf ? SecCompositeFuncOffsetTable : SecFuncOffsetTable;
770
771 // Change the section types in place to avoid duplicating the whole layout and
772 // its handling. Rewrite both legacy and composite entries so repeated writes
773 // can switch formats without losing configured flags.
774 for (auto &Entry : SectionHdrLayout) {
775 if (Entry.Type == SecFuncOffsetTable ||
776 Entry.Type == SecCompositeFuncOffsetTable)
777 Entry.Type = FuncOffsetSection;
778 else if (Entry.Type == SecLBRProfile || Entry.Type == SecCompositeProfile)
779 Entry.Type = ProfSection;
780 }
781}
782
783std::error_code SampleProfileWriterExtBinary::writeSections(
784 const SampleProfileMap &ProfileMap) {
785 // Rewrite the final configured layout immediately before its section types
786 // are consumed. Earlier layout configuration may replace SectionHdrLayout.
787 configureCompositeProfile();
788
789 std::error_code EC;
790 if (SecLayout == DefaultLayout)
791 EC = writeDefaultLayout(ProfileMap);
792 else if (SecLayout == CtxSplitLayout)
793 EC = writeCtxSplitLayout(ProfileMap);
794 else
795 llvm_unreachable("Unsupported layout");
796 return EC;
797}
798
799/// Write samples to a text file.
800///
801/// Note: it may be tempting to implement this in terms of
802/// FunctionSamples::print(). Please don't. The dump functionality is intended
803/// for debugging and has no specified form.
804///
805/// The format used here is more structured and deliberate because
806/// it needs to be parsed by the SampleProfileReaderText class.
807std::error_code SampleProfileWriterText::writeSample(const FunctionSamples &S) {
808 auto &OS = *OutputStream;
809 if (FunctionSamples::ProfileIsCS)
810 OS << "[" << S.getContext().toString() << "]:" << S.getTotalSamples();
811 else
812 OS << S.getFunction() << ":" << S.getTotalSamples();
813
814 if (Indent == 0)
815 OS << ":" << S.getHeadSamples();
816 OS << "\n";
817 LineCount++;
818
819 for (const auto &[Loc, Sample] : S.getBodySamples()) {
820 OS.indent(NumSpaces: Indent + 1);
821 Loc.print(OS);
822 OS << ": " << Sample.getSamples();
823
824 for (const auto &J : Sample.getSortedCallTargets())
825 OS << " " << J.first << ":" << J.second;
826 OS << "\n";
827 LineCount++;
828
829 if (const TypeCountMap *Map = S.findCallsiteTypeSamplesAt(Loc);
830 Map && !Map->empty()) {
831 OS.indent(NumSpaces: Indent + 1);
832 Loc.print(OS);
833 OS << ": ";
834 OS << kVTableProfPrefix;
835 for (const auto &[TypeName, Count] : *Map) {
836 OS << TypeName << ":" << Count << " ";
837 }
838 OS << "\n";
839 LineCount++;
840 }
841 }
842
843 Indent += 1;
844 for (const auto &[Loc, FunctionSamplesMap] : S.getCallsiteSamples()) {
845 for (const FunctionSamples &CalleeSamples :
846 make_second_range(c: FunctionSamplesMap)) {
847 OS.indent(NumSpaces: Indent);
848 Loc.print(OS);
849 OS << ": ";
850 if (std::error_code EC = writeSample(S: CalleeSamples))
851 return EC;
852 }
853
854 if (const TypeCountMap *Map = S.findCallsiteTypeSamplesAt(Loc);
855 Map && !Map->empty()) {
856 OS.indent(NumSpaces: Indent);
857 Loc.print(OS);
858 OS << ": ";
859 OS << kVTableProfPrefix;
860 for (const auto &[TypeId, Count] : *Map) {
861 OS << TypeId << ":" << Count << " ";
862 }
863 OS << "\n";
864 LineCount++;
865 }
866 }
867
868 Indent -= 1;
869
870 if (FunctionSamples::ProfileIsProbeBased) {
871 OS.indent(NumSpaces: Indent + 1);
872 OS << "!CFGChecksum: " << S.getFunctionHash() << "\n";
873 LineCount++;
874 }
875
876 if (S.getContext().getAllAttributes()) {
877 OS.indent(NumSpaces: Indent + 1);
878 OS << "!Attributes: " << S.getContext().getAllAttributes() << "\n";
879 LineCount++;
880 }
881
882 if (Indent == 0 && MarkFlatProfiles && S.getCallsiteSamples().size() == 0)
883 OS << " !Flat\n";
884
885 return sampleprof_error::success;
886}
887
888std::error_code
889SampleProfileWriterBinary::writeContextIdx(const SampleContext &Context) {
890 assert(!Context.hasContext() && "cs profile is not supported");
891 return writeNameIdx(FName: Context.getFunction());
892}
893
894std::error_code SampleProfileWriterBinary::writeNameIdx(FunctionId FName) {
895 auto &NTable = getNameTable();
896 const auto &Ret = NTable.find(Key: FName);
897 if (Ret == NTable.end())
898 return sampleprof_error::truncated_name_table;
899 encodeULEB128(Value: Ret->second, OS&: *OutputStream);
900 return sampleprof_error::success;
901}
902
903void SampleProfileWriterBinary::addName(FunctionId FName) {
904 auto &NTable = getNameTable();
905 NTable.insert(KV: std::make_pair(x&: FName, y: 0));
906}
907
908void SampleProfileWriterBinary::addContext(const SampleContext &Context) {
909 addName(FName: Context.getFunction());
910}
911
912void SampleProfileWriterBinary::addNames(const FunctionSamples &S) {
913 // Add all the names in indirect call targets.
914 for (const auto &I : S.getBodySamples()) {
915 const SampleRecord &Sample = I.second;
916 for (const auto &J : Sample.getCallTargets())
917 addName(FName: J.first);
918 }
919
920 // Recursively add all the names for inlined callsites.
921 for (const auto &J : S.getCallsiteSamples())
922 for (const auto &FS : J.second) {
923 const FunctionSamples &CalleeSamples = FS.second;
924 addName(FName: CalleeSamples.getFunction());
925 addNames(S: CalleeSamples);
926 }
927
928 if (!WriteVTableProf)
929 return;
930 // Add all the vtable names to NameTable.
931 for (const auto &VTableAccessCountMap :
932 llvm::make_second_range(c: S.getCallsiteTypeCounts())) {
933 // Add type name to NameTable.
934 for (const auto Type : llvm::make_first_range(c: VTableAccessCountMap)) {
935 addName(FName: Type);
936 }
937 }
938}
939
940void SampleProfileWriterExtBinaryBase::addContext(
941 const SampleContext &Context) {
942 if (Context.hasContext()) {
943 for (auto &Callsite : Context.getContextFrames())
944 SampleProfileWriterBinary::addName(FName: Callsite.Func);
945 CSNameTable.insert(KV: std::make_pair(x: Context, y: 0));
946 } else {
947 SampleProfileWriterBinary::addName(FName: Context.getFunction());
948 }
949}
950
951std::error_code SampleProfileWriterBinary::writeNameTable() {
952 auto &OS = *OutputStream;
953
954 // Write out the name table.
955 encodeULEB128(Value: NameTable.size(), OS);
956 for (const auto *Entry : stabilizeTable(Table&: NameTable)) {
957 OS << Entry->first;
958 encodeULEB128(Value: 0, OS);
959 }
960 return sampleprof_error::success;
961}
962
963std::error_code
964SampleProfileWriterBinary::writeMagicIdent(SampleProfileFormat Format) {
965 auto &OS = *OutputStream;
966 // Write file magic identifier.
967 encodeULEB128(Value: SPMagic(Format), OS);
968 encodeULEB128(Value: FormatVersion, OS);
969 return sampleprof_error::success;
970}
971
972std::error_code
973SampleProfileWriterBinary::writeHeader(const SampleProfileMap &ProfileMap) {
974 // When calling write on a different profile map, existing names should be
975 // cleared.
976 NameTable.clear();
977
978 writeMagicIdent(Format);
979
980 computeSummary(ProfileMap);
981 if (auto EC = writeSummary())
982 return EC;
983
984 // Generate the name table for all the functions referenced in the profile.
985 for (const auto &I : ProfileMap) {
986 addContext(Context: I.second.getContext());
987 addNames(S: I.second);
988 }
989
990 writeNameTable();
991 return sampleprof_error::success;
992}
993
994void SampleProfileWriterExtBinaryBase::setToCompressAllSections() {
995 for (auto &Entry : SectionHdrLayout)
996 addSecFlag(Entry, Flag: SecCommonFlags::SecFlagCompress);
997}
998
999void SampleProfileWriterExtBinaryBase::setToCompressSection(SecType Type) {
1000 addSectionFlag(Type, Flag: SecCommonFlags::SecFlagCompress);
1001}
1002
1003void SampleProfileWriterExtBinaryBase::allocSecHdrTable() {
1004 support::endian::Writer Writer(*OutputStream, llvm::endianness::little);
1005
1006 Writer.write(Val: static_cast<uint64_t>(SectionHdrLayout.size()));
1007 SecHdrTableOffset = OutputStream->tell();
1008 for (uint32_t i = 0; i < SectionHdrLayout.size(); i++) {
1009 Writer.write(Val: static_cast<uint64_t>(-1));
1010 Writer.write(Val: static_cast<uint64_t>(-1));
1011 Writer.write(Val: static_cast<uint64_t>(-1));
1012 Writer.write(Val: static_cast<uint64_t>(-1));
1013 }
1014}
1015
1016std::error_code SampleProfileWriterExtBinaryBase::writeSecHdrTable() {
1017 assert(SecHdrTable.size() == SectionHdrLayout.size() &&
1018 "SecHdrTable entries doesn't match SectionHdrLayout");
1019 SmallVector<uint32_t, 16> IndexMap(SecHdrTable.size(), -1);
1020 for (uint32_t TableIdx = 0; TableIdx < SecHdrTable.size(); TableIdx++) {
1021 IndexMap[SecHdrTable[TableIdx].LayoutIndex] = TableIdx;
1022 }
1023
1024 // Write the section header table in the order specified in
1025 // SectionHdrLayout. SectionHdrLayout specifies the sections
1026 // order in which profile reader expect to read, so the section
1027 // header table should be written in the order in SectionHdrLayout.
1028 // Note that the section order in SecHdrTable may be different
1029 // from the order in SectionHdrLayout, for example, SecFuncOffsetTable
1030 // needs to be computed after SecLBRProfile (the order in SecHdrTable),
1031 // but it needs to be read before SecLBRProfile (the order in
1032 // SectionHdrLayout). So we use IndexMap above to switch the order.
1033 support::endian::SeekableWriter Writer(
1034 static_cast<raw_pwrite_stream &>(*OutputStream),
1035 llvm::endianness::little);
1036 for (uint32_t LayoutIdx = 0; LayoutIdx < SectionHdrLayout.size();
1037 LayoutIdx++) {
1038 assert(IndexMap[LayoutIdx] < SecHdrTable.size() &&
1039 "Incorrect LayoutIdx in SecHdrTable");
1040 auto Entry = SecHdrTable[IndexMap[LayoutIdx]];
1041 Writer.pwrite(Val: static_cast<uint64_t>(Entry.Type),
1042 Offset: SecHdrTableOffset + 4 * LayoutIdx * sizeof(uint64_t));
1043 Writer.pwrite(Val: static_cast<uint64_t>(Entry.Flags),
1044 Offset: SecHdrTableOffset + (4 * LayoutIdx + 1) * sizeof(uint64_t));
1045 Writer.pwrite(Val: static_cast<uint64_t>(Entry.Offset),
1046 Offset: SecHdrTableOffset + (4 * LayoutIdx + 2) * sizeof(uint64_t));
1047 Writer.pwrite(Val: static_cast<uint64_t>(Entry.Size),
1048 Offset: SecHdrTableOffset + (4 * LayoutIdx + 3) * sizeof(uint64_t));
1049 }
1050
1051 return sampleprof_error::success;
1052}
1053
1054std::error_code SampleProfileWriterExtBinaryBase::writeHeader(
1055 const SampleProfileMap &ProfileMap) {
1056 // Reject a version that cannot describe the selected profile encoding before
1057 // emitting any part of the header.
1058 if (WriteCompositeProf && FormatVersion < CompositeProfileVersion)
1059 return sampleprof_error::unsupported_version;
1060
1061 auto &OS = *OutputStream;
1062 FileStart = OS.tell();
1063 writeMagicIdent(Format);
1064
1065 allocSecHdrTable();
1066 return sampleprof_error::success;
1067}
1068
1069std::error_code SampleProfileWriterBinary::writeCallsiteVTableProf(
1070 const CallsiteTypeMap &CallsiteTypeMap, raw_ostream &OS) {
1071 assert(WriteVTableProf &&
1072 "writeCallsiteVTableProf should not be called if WriteVTableProf is "
1073 "false");
1074
1075 encodeULEB128(Value: CallsiteTypeMap.size(), OS);
1076 for (const auto &[Loc, TypeMap] : CallsiteTypeMap) {
1077 Loc.serialize(OS);
1078 if (std::error_code EC = serializeTypeMap(Map: TypeMap, NameTable: getNameTable(), OS))
1079 return EC;
1080 }
1081
1082 return sampleprof_error::success;
1083}
1084
1085std::error_code SampleProfileWriterBinary::writeSummary() {
1086 auto &OS = *OutputStream;
1087 encodeULEB128(Value: Summary->getTotalCount(), OS);
1088 encodeULEB128(Value: Summary->getMaxCount(), OS);
1089 encodeULEB128(Value: Summary->getMaxFunctionCount(), OS);
1090 encodeULEB128(Value: Summary->getNumCounts(), OS);
1091 encodeULEB128(Value: Summary->getNumFunctions(), OS);
1092 ArrayRef<ProfileSummaryEntry> Entries = Summary->getDetailedSummary();
1093 encodeULEB128(Value: Entries.size(), OS);
1094 for (auto Entry : Entries) {
1095 encodeULEB128(Value: Entry.Cutoff, OS);
1096 encodeULEB128(Value: Entry.MinCount, OS);
1097 encodeULEB128(Value: Entry.NumCounts, OS);
1098 }
1099 return sampleprof_error::success;
1100}
1101
1102std::error_code
1103SampleProfileWriterBinary::writeLBRProfile(const FunctionSamples &S,
1104 bool IsNested) {
1105 auto &OS = *OutputStream;
1106 if (WriteCompositeProf && !IsNested)
1107 encodeULEB128(Value: S.getHeadSamples(), OS);
1108 encodeULEB128(Value: S.getTotalSamples(), OS);
1109 encodeULEB128(Value: S.getBodySamples().size(), OS);
1110 for (const auto &I : S.getBodySamples()) {
1111 LineLocation Loc = I.first;
1112 const SampleRecord &Sample = I.second;
1113 Loc.serialize(OS);
1114 if (std::error_code EC = Sample.serialize(OS, NameTable: getNameTable()))
1115 return EC;
1116 }
1117 return sampleprof_error::success;
1118}
1119
1120namespace {
1121
1122/// A reusable stream that discards payload bytes while counting their size.
1123class PayloadSizeCountingStream final : public raw_ostream {
1124public:
1125 /// Avoid retaining payload data in raw_ostream's internal buffer.
1126 PayloadSizeCountingStream() { SetUnbuffered(); }
1127
1128 /// Prepare the stream to count another payload.
1129 void resetPayload() {
1130 PayloadSize = 0;
1131 Overflowed = false;
1132 }
1133
1134 /// Return whether the payload size exceeded the representable range.
1135 bool overflowed() const { return Overflowed; }
1136
1137 /// Return the complete payload size when overflowed() is false.
1138 uint64_t payloadSize() const { return PayloadSize; }
1139
1140private:
1141 /// Count incoming bytes without retaining their contents.
1142 void write_impl(const char *, size_t Size) override {
1143 if (Overflowed)
1144 return;
1145
1146 // Fail closed if the payload cannot be represented by its uint64_t size.
1147 if (Size > UINT64_MAX - PayloadSize) {
1148 Overflowed = true;
1149 return;
1150 }
1151 PayloadSize += Size;
1152 }
1153
1154 /// Report the number of bytes accepted from the current payload.
1155 uint64_t current_pos() const override { return PayloadSize; }
1156
1157 /// Number of bytes observed during the counting pass.
1158 uint64_t PayloadSize = 0;
1159 /// Whether the counted size no longer fits in uint64_t.
1160 bool Overflowed = false;
1161};
1162
1163} // namespace
1164
1165std::error_code SampleProfileWriterBinary::writeProfileType(
1166 ProfTypes Type, function_ref<std::error_code()> WritePayload) {
1167 // PayloadSizeStream temporarily owns the real output while the callback
1168 // writes through OutputStream. A nested call would therefore mistake the
1169 // real output for PayloadSizeCountingStream.
1170 if (WritingProfileType)
1171 return sampleprof_error::malformed;
1172 SaveAndRestore RestoreWritingProfileType(WritingProfileType, true);
1173
1174 // A profile block stores its payload size before the payload, but that size
1175 // is not known until it has been serialized. Count one complete serialization
1176 // without retaining its bytes, then emit the header and serialize it again.
1177 // TODO: Avoid serializing each payload twice while retaining bounded memory
1178 // use and compatibility with compressed section output.
1179 if (!PayloadSizeStream)
1180 PayloadSizeStream = std::make_unique<PayloadSizeCountingStream>();
1181 auto *SizeStream =
1182 static_cast<PayloadSizeCountingStream *>(PayloadSizeStream.get());
1183 SizeStream->resetPayload();
1184 OutputStream.swap(u&: PayloadSizeStream);
1185 std::error_code EC = WritePayload();
1186 OutputStream.swap(u&: PayloadSizeStream);
1187 if (EC)
1188 return EC;
1189 if (SizeStream->overflowed())
1190 return sampleprof_error::too_large;
1191
1192 // Emit the compact header followed by the second, materialized pass.
1193 auto &OS = *OutputStream;
1194 encodeULEB128(Value: Type, OS);
1195 encodeULEB128(Value: SizeStream->payloadSize(), OS);
1196 uint64_t PayloadStart = OS.tell();
1197 if (std::error_code SecondPassEC = WritePayload())
1198 return SecondPassEC;
1199
1200 // Reject a stateful callback that did not reproduce the counted payload.
1201 if (OS.tell() - PayloadStart != SizeStream->payloadSize())
1202 return sampleprof_error::malformed;
1203 return sampleprof_error::success;
1204}
1205
1206static bool hasNonEmptyLBRProfile(const FunctionSamples &S, bool IsNested) {
1207 return S.getTotalSamples() != 0 || (!IsNested && S.getHeadSamples() != 0) ||
1208 !S.getBodySamples().empty();
1209}
1210
1211std::error_code
1212SampleProfileWriterBinary::writeCompositeProfile(const FunctionSamples &S,
1213 bool IsNested) {
1214 auto &OS = *OutputStream;
1215 bool WriteLBRProf = hasNonEmptyLBRProfile(S, IsNested);
1216 // Other profile types should be added here.
1217 uint32_t TypesNum = WriteLBRProf;
1218
1219 // Write the number of profile types for function.
1220 encodeULEB128(Value: TypesNum, OS);
1221
1222 if (WriteLBRProf)
1223 return writeProfileType(Type: ProfTypeLBR,
1224 WritePayload: [&] { return writeLBRProfile(S, IsNested); });
1225 return sampleprof_error::success;
1226}
1227
1228std::error_code SampleProfileWriterBinary::writeBody(const FunctionSamples &S,
1229 bool IsNested) {
1230 auto &OS = *OutputStream;
1231 if (std::error_code EC = writeContextIdx(Context: S.getContext()))
1232 return EC;
1233
1234 // Emit all the body samples.
1235 if (WriteCompositeProf) {
1236 if (std::error_code EC = writeCompositeProfile(S, IsNested))
1237 return EC;
1238 } else {
1239 if (std::error_code EC = writeLBRProfile(S, IsNested))
1240 return EC;
1241 }
1242
1243 // Recursively emit all the callsite samples.
1244 uint64_t NumCallsites = 0;
1245 for (const auto &J : S.getCallsiteSamples())
1246 NumCallsites += J.second.size();
1247 encodeULEB128(Value: NumCallsites, OS);
1248 for (const auto &J : S.getCallsiteSamples())
1249 for (const auto &FS : J.second) {
1250 J.first.serialize(OS);
1251 if (std::error_code EC = writeBody(S: FS.second, /*IsNested=*/true))
1252 return EC;
1253 }
1254
1255 if (WriteVTableProf)
1256 return writeCallsiteVTableProf(CallsiteTypeMap: S.getCallsiteTypeCounts(), OS);
1257
1258 return sampleprof_error::success;
1259}
1260
1261/// Write samples of a top-level function to a binary file.
1262///
1263/// \returns true if the samples were written successfully, false otherwise.
1264std::error_code
1265SampleProfileWriterBinary::writeSample(const FunctionSamples &S) {
1266 encodeULEB128(Value: S.getHeadSamples(), OS&: *OutputStream);
1267 return writeBody(S, /*IsNested=*/false);
1268}
1269
1270/// Create a sample profile file writer based on the specified format.
1271///
1272/// \param Filename The file to create.
1273///
1274/// \param Format Encoding format for the profile file.
1275///
1276/// \returns an error code indicating the status of the created writer.
1277ErrorOr<std::unique_ptr<SampleProfileWriter>>
1278SampleProfileWriter::create(StringRef Filename, SampleProfileFormat Format) {
1279 std::error_code EC;
1280 std::unique_ptr<raw_ostream> OS;
1281 if (Format == SPF_Binary || Format == SPF_Ext_Binary)
1282 OS.reset(p: new raw_fd_ostream(Filename, EC, sys::fs::OF_None));
1283 else
1284 OS.reset(p: new raw_fd_ostream(Filename, EC, sys::fs::OF_TextWithCRLF));
1285 if (EC)
1286 return EC;
1287
1288 return create(OS, Format);
1289}
1290
1291/// Create a sample profile stream writer based on the specified format.
1292///
1293/// \param OS The output stream to store the profile data to.
1294///
1295/// \param Format Encoding format for the profile file.
1296///
1297/// \returns an error code indicating the status of the created writer.
1298ErrorOr<std::unique_ptr<SampleProfileWriter>>
1299SampleProfileWriter::create(std::unique_ptr<raw_ostream> &OS,
1300 SampleProfileFormat Format) {
1301 std::error_code EC;
1302 std::unique_ptr<SampleProfileWriter> Writer;
1303
1304 // Currently only Text and Extended Binary format are supported for CSSPGO.
1305 if ((FunctionSamples::ProfileIsCS || FunctionSamples::ProfileIsProbeBased) &&
1306 Format == SPF_Binary)
1307 return sampleprof_error::unsupported_writing_format;
1308
1309 if (Format == SPF_Binary)
1310 Writer.reset(p: new SampleProfileWriterRawBinary(OS));
1311 else if (Format == SPF_Ext_Binary)
1312 Writer.reset(p: new SampleProfileWriterExtBinary(OS));
1313 else if (Format == SPF_Text)
1314 Writer.reset(p: new SampleProfileWriterText(OS));
1315 else if (Format == SPF_GCC)
1316 EC = sampleprof_error::unsupported_writing_format;
1317 else
1318 EC = sampleprof_error::unrecognized_format;
1319
1320 if (EC)
1321 return EC;
1322
1323 Writer->Format = Format;
1324 if (Format != SPF_Ext_Binary) {
1325 Writer->setFormatVersion(DefaultVersion);
1326 } else {
1327 if (!formatVersionIsSupported(Version: RequestedVersion))
1328 return sampleprof_error::unsupported_version;
1329
1330 // Composite output defaults to its first compatible format version.
1331 // Preserve a compatible version explicitly selected by the user.
1332 if (ExtBinaryCompositeProf) {
1333 if (RequestedVersion.getNumOccurrences() == 0) {
1334 Writer->setFormatVersion(CompositeProfileVersion);
1335 } else {
1336 if (RequestedVersion < CompositeProfileVersion)
1337 return sampleprof_error::unsupported_version;
1338 Writer->setFormatVersion(RequestedVersion);
1339 }
1340 // Keep subsequent writes independent of the global command-line option.
1341 Writer->setUseCompositeProfile(true);
1342 } else {
1343 Writer->setFormatVersion(RequestedVersion);
1344 }
1345 }
1346
1347 return std::move(Writer);
1348}
1349
1350void SampleProfileWriter::computeSummary(const SampleProfileMap &ProfileMap) {
1351 SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs);
1352 Summary = Builder.computeSummaryForProfiles(Profiles: ProfileMap);
1353}
1354