1//===- SampleProfWriter.cpp - Write LLVM sample profile data --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the class that writes LLVM sample profiles. It
10// supports two file formats: text and binary. The textual representation
11// is useful for debugging and testing purposes. The binary representation
12// is more compact, resulting in smaller file sizes. However, they can
13// both be used interchangeably.
14//
15// See lib/ProfileData/SampleProfReader.cpp for documentation on each of the
16// supported formats.
17//
18//===----------------------------------------------------------------------===//
19
20#include "llvm/ProfileData/SampleProfWriter.h"
21#include "llvm/ADT/Eytzinger.h"
22#include "llvm/ADT/StringRef.h"
23#include "llvm/ProfileData/ProfileCommon.h"
24#include "llvm/ProfileData/SampleProf.h"
25#include "llvm/Support/Compression.h"
26#include "llvm/Support/EndianStream.h"
27#include "llvm/Support/ErrorOr.h"
28#include "llvm/Support/FileSystem.h"
29#include "llvm/Support/LEB128.h"
30#include "llvm/Support/MD5.h"
31#include "llvm/Support/SaveAndRestore.h"
32#include "llvm/Support/raw_ostream.h"
33#include <array>
34#include <cmath>
35#include <cstdint>
36#include <memory>
37#include <system_error>
38#include <utility>
39#include <vector>
40
41#define DEBUG_TYPE "llvm-profdata"
42
43using namespace llvm;
44using namespace sampleprof;
45
46// To begin with, make this option off by default.
47static cl::opt<bool> ExtBinaryWriteVTableTypeProf(
48 "extbinary-write-vtable-type-prof", cl::init(Val: false), cl::Hidden,
49 cl::desc("Write vtable type profile in ext-binary sample profile writer"));
50
51static cl::opt<uint64_t> RequestedVersion(
52 "sample-profile-format-version", cl::init(Val: DefaultVersion), cl::Hidden,
53 cl::desc("Format version to write for extensible binary profiles"));
54
55static cl::opt<bool>
56 ExtBinaryCompositeProf("extbinary-composite-prof", cl::init(Val: false),
57 cl::Hidden,
58 cl::desc("Use the composite profile format"));
59
60namespace llvm {
61namespace support {
62namespace endian {
63namespace {
64
65// Adapter class to llvm::support::endian::Writer for pwrite().
66struct SeekableWriter {
67 raw_pwrite_stream &OS;
68 endianness Endian;
69 SeekableWriter(raw_pwrite_stream &OS, endianness Endian)
70 : OS(OS), Endian(Endian) {}
71
72 template <typename ValueType> void pwrite(ValueType Val, size_t Offset) {
73 std::string StringBuf;
74 raw_string_ostream SStream(StringBuf);
75 Writer(SStream, Endian).write(Val);
76 OS.pwrite(Ptr: StringBuf.data(), Size: StringBuf.size(), Offset);
77 }
78};
79
80} // namespace
81} // namespace endian
82} // namespace support
83} // namespace llvm
84
85DefaultFunctionPruningStrategy::DefaultFunctionPruningStrategy(
86 SampleProfileMap &ProfileMap, size_t OutputSizeLimit)
87 : FunctionPruningStrategy(ProfileMap, OutputSizeLimit) {
88 sortFuncProfiles(ProfileMap, SortedProfiles&: SortedFunctions);
89}
90
91void DefaultFunctionPruningStrategy::Erase(size_t CurrentOutputSize) {
92 double D = (double)OutputSizeLimit / CurrentOutputSize;
93 size_t NewSize = (size_t)round(x: ProfileMap.size() * D * D);
94 size_t NumToRemove = ProfileMap.size() - NewSize;
95 if (NumToRemove < 1)
96 NumToRemove = 1;
97
98 assert(NumToRemove <= SortedFunctions.size());
99 for (const NameFunctionSamples &E :
100 llvm::drop_begin(RangeOrContainer&: SortedFunctions, N: SortedFunctions.size() - NumToRemove))
101 ProfileMap.erase(Key: E.first);
102 SortedFunctions.resize(new_size: SortedFunctions.size() - NumToRemove);
103}
104
105std::error_code SampleProfileWriter::writeWithSizeLimitInternal(
106 SampleProfileMap &ProfileMap, size_t OutputSizeLimit,
107 FunctionPruningStrategy *Strategy) {
108 if (OutputSizeLimit == 0)
109 return write(ProfileMap);
110
111 size_t OriginalFunctionCount = ProfileMap.size();
112
113 std::unique_ptr<raw_ostream> OriginalOutputStream;
114 OutputStream.swap(u&: OriginalOutputStream);
115
116 size_t IterationCount = 0;
117 size_t TotalSize;
118
119 SmallVector<char> StringBuffer;
120 do {
121 StringBuffer.clear();
122 OutputStream.reset(p: new raw_svector_ostream(StringBuffer));
123 if (std::error_code EC = write(ProfileMap))
124 return EC;
125
126 TotalSize = StringBuffer.size();
127 // On Windows every "\n" is actually written as "\r\n" to disk but not to
128 // memory buffer, this difference should be added when considering the total
129 // output size.
130#ifdef _WIN32
131 if (Format == SPF_Text)
132 TotalSize += LineCount;
133#endif
134 if (TotalSize <= OutputSizeLimit)
135 break;
136
137 Strategy->Erase(CurrentOutputSize: TotalSize);
138 IterationCount++;
139 } while (ProfileMap.size() != 0);
140
141 if (ProfileMap.size() == 0)
142 return sampleprof_error::too_large;
143
144 OutputStream.swap(u&: OriginalOutputStream);
145 OutputStream->write(Ptr: StringBuffer.data(), Size: StringBuffer.size());
146 LLVM_DEBUG(dbgs() << "Profile originally has " << OriginalFunctionCount
147 << " functions, reduced to " << ProfileMap.size() << " in "
148 << IterationCount << " iterations\n");
149 // Silence warning on Release build.
150 (void)OriginalFunctionCount;
151 (void)IterationCount;
152 return sampleprof_error::success;
153}
154
155std::error_code
156SampleProfileWriter::writeFuncProfiles(const SampleProfileMap &ProfileMap) {
157 std::vector<NameFunctionSamples> V;
158 sortFuncProfiles(ProfileMap, SortedProfiles&: V);
159 for (const auto &I : V) {
160 if (std::error_code EC = writeSample(S: *I.second))
161 return EC;
162 }
163 return sampleprof_error::success;
164}
165
166std::error_code SampleProfileWriter::write(const SampleProfileMap &ProfileMap) {
167 if (std::error_code EC = writeHeader(ProfileMap))
168 return EC;
169
170 if (std::error_code EC = writeFuncProfiles(ProfileMap))
171 return EC;
172
173 return sampleprof_error::success;
174}
175
176/// Return the current position and prepare to use it as the start
177/// position of a section given the section type \p Type and its position
178/// \p LayoutIdx in SectionHdrLayout.
179uint64_t
180SampleProfileWriterExtBinaryBase::markSectionStart(SecType Type,
181 uint32_t LayoutIdx) {
182 uint64_t SectionStart = OutputStream->tell();
183 assert(LayoutIdx < SectionHdrLayout.size() && "LayoutIdx out of range");
184 const auto &Entry = SectionHdrLayout[LayoutIdx];
185 assert(Entry.Type == Type && "Unexpected section type");
186 // Use LocalBuf as a temporary output for writing data.
187 if (hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagCompress))
188 LocalBufStream.swap(u&: OutputStream);
189 return SectionStart;
190}
191
192std::error_code SampleProfileWriterExtBinaryBase::compressAndOutput() {
193 if (!llvm::compression::zlib::isAvailable())
194 return sampleprof_error::zlib_unavailable;
195 std::string &UncompressedStrings =
196 static_cast<raw_string_ostream *>(LocalBufStream.get())->str();
197 if (UncompressedStrings.empty())
198 return sampleprof_error::success;
199 auto &OS = *OutputStream;
200 SmallVector<uint8_t, 128> CompressedStrings;
201 compression::zlib::compress(Input: arrayRefFromStringRef(Input: UncompressedStrings),
202 CompressedBuffer&: CompressedStrings,
203 Level: compression::zlib::BestSizeCompression);
204 encodeULEB128(Value: UncompressedStrings.size(), OS);
205 encodeULEB128(Value: CompressedStrings.size(), OS);
206 OS << toStringRef(Input: CompressedStrings);
207 UncompressedStrings.clear();
208 return sampleprof_error::success;
209}
210
211/// Add a new section into section header table given the section type
212/// \p Type, its position \p LayoutIdx in SectionHdrLayout and the
213/// location \p SectionStart where the section should be written to.
214std::error_code SampleProfileWriterExtBinaryBase::addNewSection(
215 SecType Type, uint32_t LayoutIdx, uint64_t SectionStart) {
216 assert(LayoutIdx < SectionHdrLayout.size() && "LayoutIdx out of range");
217 const auto &Entry = SectionHdrLayout[LayoutIdx];
218 assert(Entry.Type == Type && "Unexpected section type");
219 if (hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagCompress)) {
220 LocalBufStream.swap(u&: OutputStream);
221 if (std::error_code EC = compressAndOutput())
222 return EC;
223 }
224 SecHdrTable.push_back(x: {.Type: Type, .Flags: Entry.Flags, .Offset: SectionStart - FileStart,
225 .Size: OutputStream->tell() - SectionStart, .LayoutIndex: LayoutIdx});
226 return sampleprof_error::success;
227}
228
229std::error_code
230SampleProfileWriterExtBinaryBase::write(const SampleProfileMap &ProfileMap) {
231 // When calling write on a different profile map, existing states should be
232 // cleared.
233 NameTable.clear();
234 CSNameTable.clear();
235 SecHdrTable.clear();
236
237 if (std::error_code EC = writeHeader(ProfileMap))
238 return EC;
239
240 std::string LocalBuf;
241 LocalBufStream = std::make_unique<raw_string_ostream>(args&: LocalBuf);
242 if (std::error_code EC = writeSections(ProfileMap))
243 return EC;
244
245 if (std::error_code EC = writeSecHdrTable())
246 return EC;
247
248 return sampleprof_error::success;
249}
250
251std::error_code SampleProfileWriterExtBinaryBase::writeContextIdx(
252 const SampleContext &Context) {
253 if (Context.hasContext())
254 return writeCSNameIdx(Context);
255 else
256 return SampleProfileWriterBinary::writeNameIdx(FName: Context.getFunction());
257}
258
259std::error_code
260SampleProfileWriterExtBinaryBase::writeCSNameIdx(const SampleContext &Context) {
261 const auto &Ret = CSNameTable.find(Key: Context);
262 if (Ret == CSNameTable.end())
263 return sampleprof_error::truncated_name_table;
264 encodeULEB128(Value: Ret->second, OS&: *OutputStream);
265 return sampleprof_error::success;
266}
267
268std::error_code
269SampleProfileWriterExtBinaryBase::writeSample(const FunctionSamples &S) {
270 uint64_t Offset = OutputStream->tell();
271 auto &Context = S.getContext();
272 FuncOffsetTable[Context] = Offset - SecLBRProfileStart;
273 if (!WriteCompositeProf)
274 encodeULEB128(Value: S.getHeadSamples(), OS&: *OutputStream);
275 return writeBody(S, /*IsNested=*/false);
276}
277
278std::error_code
279SampleProfileWriterExtBinaryBase::writeFuncOffsetTable(SecType Type,
280 bool IsNested) {
281 if (UseMD5IndexedTables) {
282 // Eytzinger layout requires MD5 representation and does not support
283 // multi-context Context-Sensitive profiles.
284 if (!UseMD5 || FunctionSamples::ProfileIsCS)
285 return sampleprof_error::unsupported_writing_format;
286 return writeEytzingerFuncOffsetTable(Type, IsNested);
287 }
288 return writeLegacyFuncOffsetTable(Type);
289}
290
291std::error_code
292SampleProfileWriterExtBinaryBase::writeEytzingerFuncOffsetTable(SecType Type,
293 bool IsNested) {
294 assert((NumNested + NumFlat > 0 || FuncOffsetTable.empty()) &&
295 "SecNameTable must be written before SecFuncOffsetTable to establish "
296 "Eytzinger indices!");
297
298 size_t SpanSize = IsNested ? NumNested : NumFlat;
299 size_t BaseIdx = IsNested ? 0 : NumNested;
300
301 std::vector<support::ulittle32_t> FuncOffsets(
302 SpanSize, support::ulittle32_t(UINT32_MAX));
303
304 // Populate the function offset array parallel to the Eytzinger span.
305 for (const auto &[Context, RelativeOffset] : FuncOffsetTable) {
306 if (RelativeOffset >= UINT32_MAX)
307 return sampleprof_error::too_large;
308
309 FunctionId FId = Context.getFunction();
310 auto It = NameTable.find(Key: FId);
311 if (It == NameTable.end())
312 continue;
313
314 size_t GlobalIdx = It->second;
315 if (GlobalIdx < BaseIdx || (GlobalIdx - BaseIdx) >= SpanSize)
316 continue;
317
318 size_t LocalIdx = GlobalIdx - BaseIdx;
319 assert(
320 FuncOffsets[LocalIdx] == UINT32_MAX &&
321 "Function offset slot already populated; duplicate GUID or collision!");
322 FuncOffsets[LocalIdx] = static_cast<uint32_t>(RelativeOffset);
323 }
324
325 assert(!llvm::is_contained(FuncOffsets, support::ulittle32_t(UINT32_MAX)) &&
326 "Unpopulated slot in Eytzinger function offset array!");
327
328 OutputStream->write(Ptr: reinterpret_cast<const char *>(FuncOffsets.data()),
329 Size: SpanSize * sizeof(support::ulittle32_t));
330 // Type is SecFuncOffsetTable or SecCompositeFuncOffsetTable.
331 addSectionFlag(Type, Flag: SecFuncOffsetFlags::SecFlagEytzinger);
332 FuncOffsetTable.clear();
333 return sampleprof_error::success;
334}
335
336std::error_code
337SampleProfileWriterExtBinaryBase::writeLegacyFuncOffsetTable(SecType Type) {
338 auto &OS = *OutputStream;
339
340 // Write out the table size.
341 encodeULEB128(Value: FuncOffsetTable.size(), OS);
342
343 // Write out FuncOffsetTable.
344 auto WriteItem = [&](const SampleContext &Context, uint64_t Offset) {
345 if (std::error_code EC = writeContextIdx(Context))
346 return EC;
347 encodeULEB128(Value: Offset, OS);
348 return (std::error_code)sampleprof_error::success;
349 };
350
351 if (FunctionSamples::ProfileIsCS) {
352 // Sort the contexts before writing them out. This is to help fast load all
353 // context profiles for a function as well as their callee contexts which
354 // can help profile-guided importing for ThinLTO.
355 std::map<SampleContext, uint64_t> OrderedFuncOffsetTable(
356 FuncOffsetTable.begin(), FuncOffsetTable.end());
357 for (const auto &Entry : OrderedFuncOffsetTable) {
358 if (std::error_code EC = WriteItem(Entry.first, Entry.second))
359 return EC;
360 }
361 addSectionFlag(Type, Flag: SecFuncOffsetFlags::SecFlagOrdered);
362 } else {
363 for (const auto &Entry : FuncOffsetTable) {
364 if (std::error_code EC = WriteItem(Entry.first, Entry.second))
365 return EC;
366 }
367 }
368
369 FuncOffsetTable.clear();
370 return sampleprof_error::success;
371}
372
373std::error_code SampleProfileWriterExtBinaryBase::writeFuncMetadata(
374 const FunctionSamples &FunctionProfile) {
375 auto &OS = *OutputStream;
376 if (std::error_code EC = writeContextIdx(Context: FunctionProfile.getContext()))
377 return EC;
378
379 if (FunctionSamples::ProfileIsProbeBased)
380 encodeULEB128(Value: FunctionProfile.getFunctionHash(), OS);
381 if (FunctionSamples::ProfileIsCS || FunctionSamples::ProfileIsPreInlined) {
382 encodeULEB128(Value: FunctionProfile.getContext().getAllAttributes(), OS);
383 }
384
385 if (!FunctionSamples::ProfileIsCS) {
386 // Recursively emit attributes for all callee samples.
387 uint64_t NumCallsites = 0;
388 for (const auto &J : FunctionProfile.getCallsiteSamples())
389 NumCallsites += J.second.size();
390 encodeULEB128(Value: NumCallsites, OS);
391 for (const auto &J : FunctionProfile.getCallsiteSamples()) {
392 for (const auto &FS : J.second) {
393 LineLocation Loc = J.first;
394 encodeULEB128(Value: Loc.LineOffset, OS);
395 encodeULEB128(Value: Loc.Discriminator, OS);
396 if (std::error_code EC = writeFuncMetadata(FunctionProfile: FS.second))
397 return EC;
398 }
399 }
400 }
401
402 return sampleprof_error::success;
403}
404
405std::error_code SampleProfileWriterExtBinaryBase::writeFuncMetadata(
406 const SampleProfileMap &Profiles) {
407 if (!FunctionSamples::ProfileIsProbeBased && !FunctionSamples::ProfileIsCS &&
408 !FunctionSamples::ProfileIsPreInlined)
409 return sampleprof_error::success;
410 for (const auto &Entry : Profiles) {
411 if (std::error_code EC = writeFuncMetadata(FunctionProfile: Entry.second))
412 return EC;
413 }
414 return sampleprof_error::success;
415}
416
417template <class KeyT, class ValT>
418static SmallVector<std::pair<KeyT, ValT> *, 0>
419stabilizeTable(MapVector<KeyT, ValT> &Table) {
420 SmallVector<std::pair<KeyT, ValT> *, 0> Entries(
421 llvm::make_pointer_range(Table));
422
423 llvm::sort(Entries,
424 [](const auto *L, const auto *R) { return L->first < R->first; });
425
426 for (const auto &[I, Entry] : llvm::enumerate(Entries))
427 Entry->second = I;
428
429 return Entries;
430}
431
432std::error_code SampleProfileWriterExtBinaryBase::writeNameTable() {
433 if (!UseMD5)
434 return SampleProfileWriterBinary::writeNameTable();
435
436 auto &OS = *OutputStream;
437
438 // Write out the MD5 name table. We wrote unencoded MD5 so reader can
439 // retrieve the name using the name index without having to read the
440 // whole name table.
441 encodeULEB128(Value: NameTable.size(), OS);
442 support::endian::Writer Writer(OS, llvm::endianness::little);
443 for (const auto *Entry : stabilizeTable(Table&: NameTable))
444 Writer.write(Val: Entry->first.getHashCode());
445 return sampleprof_error::success;
446}
447
448std::error_code SampleProfileWriterExtBinaryBase::writeNameTableSection(
449 const SampleProfileMap &ProfileMap) {
450 for (const auto &I : ProfileMap) {
451 addContext(Context: I.second.getContext());
452 addNames(S: I.second);
453 }
454
455 // If NameTable contains ".__uniq." suffix, set SecFlagUniqSuffix flag
456 // so compiler won't strip the suffix during profile matching after
457 // seeing the flag in the profile.
458 // Original names are unavailable if using MD5, so this option has no use.
459 if (!UseMD5) {
460 for (const auto &I : NameTable) {
461 if (I.first.stringRef().contains(Other: FunctionSamples::UniqSuffix)) {
462 addSectionFlag(Type: SecNameTable, Flag: SecNameTableFlags::SecFlagUniqSuffix);
463 break;
464 }
465 }
466 }
467
468 if (UseMD5 && UseMD5IndexedTables) {
469 // Eytzinger name tables do not support CSSPGO profiles
470 // (FunctionSamples::ProfileIsCS).
471 if (FunctionSamples::ProfileIsCS)
472 return sampleprof_error::unsupported_writing_format;
473 if (auto EC = writeEytzingerNameTableSection(ProfileMap))
474 return EC;
475 return sampleprof_error::success;
476 }
477
478 if (auto EC = writeNameTable())
479 return EC;
480 return sampleprof_error::success;
481}
482
483namespace {
484
485// Helper class to construct and write the SecNameTable section in Eytzinger
486// layout for ExtBinary MD5 profiles.
487//
488// The on-disk layout of the Eytzinger name table section consists of symbol
489// counts followed by three contiguous Eytzinger hash arrays:
490// - ULEB128 count of Nested top-level profile symbol keys
491// - ULEB128 count of Flat top-level profile symbol keys
492// - ULEB128 count of Inlinee and auxiliary profile symbol keys
493// - Array of 64-bit little-endian MD5 hash keys for Nested profiles in
494// Eytzinger order
495// - Array of 64-bit little-endian MD5 hash keys for Flat profiles in Eytzinger
496// order
497// - Array of 64-bit little-endian MD5 hash keys for Inlinees in Eytzinger order
498class EytzingerNameTable {
499 using TableT = llvm::EytzingerTable<support::ulittle64_t>;
500 std::array<TableT, static_cast<size_t>(EytzingerSpan::NumSpans)> Spans;
501
502public:
503 EytzingerNameTable(std::vector<support::ulittle64_t> NestedKeys,
504 std::vector<support::ulittle64_t> FlatKeys,
505 std::vector<support::ulittle64_t> InlineeKeys)
506 : Spans{TableT::create(Keys: std::move(NestedKeys)),
507 TableT::create(Keys: std::move(FlatKeys)),
508 TableT::create(Keys: std::move(InlineeKeys))} {}
509
510 // Find the global index of GUID across the three Eytzinger table spans.
511 uint64_t findGlobalIdx(uint64_t GUID) const {
512 uint64_t BaseIdx = 0;
513 for (const auto &Table : Spans) {
514 if (std::optional<size_t> LocalIdx = Table.findIndex(Target: GUID))
515 return BaseIdx + *LocalIdx;
516 BaseIdx += Table.size();
517 }
518 llvm_unreachable("Symbol in NameTable missing from Eytzinger spans");
519 }
520
521 void write(raw_ostream &OS) const {
522 for (const auto &Table : Spans)
523 encodeULEB128(Value: uint64_t(Table.size()), OS);
524 for (const auto &Table : Spans)
525 OS.write(Ptr: reinterpret_cast<const char *>(Table.data()),
526 Size: Table.size() * sizeof(support::ulittle64_t));
527 }
528
529 size_t size(EytzingerSpan S) const {
530 return Spans[static_cast<size_t>(S)].size();
531 }
532};
533
534} // end anonymous namespace
535
536std::error_code
537SampleProfileWriterExtBinaryBase::writeEytzingerNameTableSection(
538 const SampleProfileMap &ProfileMap) {
539 DenseSet<uint64_t> TopLevelGUIDs;
540 std::vector<support::ulittle64_t> NestedKeys, FlatKeys, InlineeKeys;
541
542 // Collect top-level Nested and Flat keys directly from ProfileMap.
543 for (const auto &I : ProfileMap) {
544 const SampleContext &Ctx = I.second.getContext();
545 uint64_t GUID = Ctx.getFunction().getHashCode();
546 if (TopLevelGUIDs.insert(V: GUID).second) {
547 // In single-table default layouts, unify all top-level symbols in the
548 // Nested partition so they match the single unflagged function offset
549 // table.
550 if (SecLayout != CtxSplitLayout || I.second.hasCallsiteSamples())
551 NestedKeys.emplace_back(args&: GUID);
552 else
553 FlatKeys.emplace_back(args&: GUID);
554 }
555 }
556
557 // Collect remaining non-top-level symbols (inlinees, targets, vtables) from
558 // NameTable.
559 for (const auto &Entry : NameTable) {
560 uint64_t GUID = Entry.first.getHashCode();
561 if (!TopLevelGUIDs.contains(V: GUID))
562 InlineeKeys.emplace_back(args&: GUID);
563 }
564
565 EytzingerNameTable Tables(std::move(NestedKeys), std::move(FlatKeys),
566 std::move(InlineeKeys));
567
568 // Assign each symbol its corresponding index in the Eytzinger layout.
569 for (auto &[FId, Idx] : NameTable)
570 Idx = Tables.findGlobalIdx(GUID: FId.getHashCode());
571
572 Tables.write(OS&: *OutputStream);
573 NumNested = Tables.size(S: EytzingerSpan::Nested);
574 NumFlat = Tables.size(S: EytzingerSpan::Flat);
575
576 return sampleprof_error::success;
577}
578
579std::error_code SampleProfileWriterExtBinaryBase::writeCSNameTableSection() {
580 auto &OS = *OutputStream;
581 encodeULEB128(Value: CSNameTable.size(), OS);
582 support::endian::Writer Writer(OS, llvm::endianness::little);
583 for (const auto *Entry : stabilizeTable(Table&: CSNameTable)) {
584 auto Frames = Entry->first.getContextFrames();
585 encodeULEB128(Value: Frames.size(), OS);
586 for (auto &Callsite : Frames) {
587 if (std::error_code EC = writeNameIdx(FName: Callsite.Func))
588 return EC;
589 encodeULEB128(Value: Callsite.Location.LineOffset, OS);
590 encodeULEB128(Value: Callsite.Location.Discriminator, OS);
591 }
592 }
593
594 return sampleprof_error::success;
595}
596
597std::error_code
598SampleProfileWriterExtBinaryBase::writeProfileSymbolListSection() {
599 if (UseMD5ProfSymList)
600 return writeMD5ProfileSymbolListSection();
601 return writeStringBasedProfileSymbolListSection();
602}
603
604std::error_code
605SampleProfileWriterExtBinaryBase::writeStringBasedProfileSymbolListSection() {
606 assert((!ProfSymList || !ProfSymList->isMD5()) &&
607 "Writing string-based ProfileSymbolListSection from MD5 table "
608 "not yet implemented");
609 if (ProfSymList && ProfSymList->size() > 0)
610 if (std::error_code EC = ProfSymList->write(OS&: *OutputStream))
611 return EC;
612
613 return sampleprof_error::success;
614}
615
616std::error_code
617SampleProfileWriterExtBinaryBase::writeMD5ProfileSymbolListSection() {
618 if (!ProfSymList || ProfSymList->size() == 0)
619 return sampleprof_error::success;
620 assert(!ProfSymList->isMD5() &&
621 "Writing MD5 ProfileSymbolListSection from existing MD5 "
622 "table not yet implemented");
623
624 auto &OS = *OutputStream;
625 std::vector<uint64_t> Keys = ProfSymList->collectGUIDs();
626
627 auto Table =
628 llvm::EytzingerTable<support::ulittle64_t>::create(Keys: std::move(Keys));
629
630 OS.write(Ptr: reinterpret_cast<const char *>(Table.data()),
631 Size: Table.size() * sizeof(support::ulittle64_t));
632 return sampleprof_error::success;
633}
634
635unsigned SampleProfileWriterExtBinaryBase::findUnwrittenEntry(SecType Type) {
636 auto WrittenIndices =
637 llvm::map_range(C&: SecHdrTable, F: &SecHdrTableEntry::LayoutIndex);
638 for (auto [I, Entry] : llvm::enumerate(First&: SectionHdrLayout))
639 if (Entry.Type == Type && !llvm::is_contained(Range&: WrittenIndices, Element: I))
640 return I;
641 llvm_unreachable("Matching section not found in SectionHdrLayout");
642}
643
644std::error_code SampleProfileWriterExtBinaryBase::writeOneSection(
645 SecType Type, const SampleProfileMap &ProfileMap) {
646 unsigned LayoutIdx = findUnwrittenEntry(Type);
647 SecHdrTableEntry &Entry = SectionHdrLayout[LayoutIdx];
648
649 // The setting of SecFlagCompress should happen before markSectionStart.
650 if (Type == SecFuncMetadata && FunctionSamples::ProfileIsProbeBased)
651 addSectionFlag(Type: SecFuncMetadata, Flag: SecFuncMetadataFlags::SecFlagIsProbeBased);
652 if (Type == SecFuncMetadata &&
653 (FunctionSamples::ProfileIsCS || FunctionSamples::ProfileIsPreInlined))
654 addSectionFlag(Type: SecFuncMetadata, Flag: SecFuncMetadataFlags::SecFlagHasAttribute);
655 if (Type == SecProfSummary && FunctionSamples::ProfileIsCS)
656 addSectionFlag(Type: SecProfSummary, Flag: SecProfSummaryFlags::SecFlagFullContext);
657 if (Type == SecProfSummary && FunctionSamples::ProfileIsPreInlined)
658 addSectionFlag(Type: SecProfSummary, Flag: SecProfSummaryFlags::SecFlagIsPreInlined);
659 if (Type == SecProfSummary && FunctionSamples::ProfileIsFS)
660 addSectionFlag(Type: SecProfSummary, Flag: SecProfSummaryFlags::SecFlagFSDiscriminator);
661 if (Type == SecProfSummary && ExtBinaryWriteVTableTypeProf)
662 addSectionFlag(Type: SecProfSummary,
663 Flag: SecProfSummaryFlags::SecFlagHasVTableTypeProf);
664 if (Type == SecProfileSymbolList && UseMD5ProfSymList)
665 addSectionFlag(Type: SecProfileSymbolList, Flag: SecProfileSymbolListFlags::SecFlagMD5);
666 if (Type == SecNameTable && UseMD5IndexedTables && UseMD5)
667 addSectionFlag(Type: SecNameTable, Flag: SecNameTableFlags::SecFlagEytzinger);
668
669 uint64_t SectionStart = markSectionStart(Type, LayoutIdx);
670 switch (Type) {
671 case SecProfSummary:
672 computeSummary(ProfileMap);
673 if (auto EC = writeSummary())
674 return EC;
675 break;
676 case SecNameTable:
677 if (auto EC = writeNameTableSection(ProfileMap))
678 return EC;
679 break;
680 case SecCSNameTable:
681 if (auto EC = writeCSNameTableSection())
682 return EC;
683 break;
684 case SecLBRProfile:
685 case SecCompositeProfile:
686 SecLBRProfileStart = OutputStream->tell();
687 if (std::error_code EC = writeFuncProfiles(ProfileMap))
688 return EC;
689 break;
690 case SecFuncOffsetTable:
691 case SecCompositeFuncOffsetTable: {
692 bool IsFlat = hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagFlat);
693 // An unflagged function offset table inherently indexes the primary
694 // Nested symbol span.
695 bool IsNested = !IsFlat;
696 if (auto EC = writeFuncOffsetTable(Type, IsNested))
697 return EC;
698 break;
699 }
700 case SecFuncMetadata:
701 if (std::error_code EC = writeFuncMetadata(Profiles: ProfileMap))
702 return EC;
703 break;
704 case SecProfileSymbolList:
705 if (auto EC = writeProfileSymbolListSection())
706 return EC;
707 break;
708 default:
709 if (auto EC = writeCustomSection(Type))
710 return EC;
711 break;
712 }
713 if (std::error_code EC = addNewSection(Type, LayoutIdx, SectionStart))
714 return EC;
715 return sampleprof_error::success;
716}
717
718SampleProfileWriterExtBinary::SampleProfileWriterExtBinary(
719 std::unique_ptr<raw_ostream> &OS)
720 : SampleProfileWriterExtBinaryBase(OS) {
721 WriteVTableProf = ExtBinaryWriteVTableTypeProf;
722}
723
724std::error_code SampleProfileWriterExtBinary::writeDefaultLayout(
725 const SampleProfileMap &ProfileMap) {
726 // ProfSection / FuncOffsetSection are SecLBR* or SecComposite* after
727 // configureCompositeProfile.
728 const SecType Sections[] = {
729 SecProfSummary, SecNameTable, SecCSNameTable, ProfSection,
730 SecProfileSymbolList, FuncOffsetSection, SecFuncMetadata,
731 };
732 for (SecType Type : Sections)
733 if (std::error_code EC = writeOneSection(Type, ProfileMap))
734 return EC;
735 return sampleprof_error::success;
736}
737
738static void splitProfileMapToTwo(const SampleProfileMap &ProfileMap,
739 SampleProfileMap &NestedProfileMap,
740 SampleProfileMap &FlatProfileMap) {
741 for (const auto &I : ProfileMap) {
742 if (I.second.hasCallsiteSamples())
743 NestedProfileMap.insert(x: {I.first, I.second});
744 else
745 FlatProfileMap.insert(x: {I.first, I.second});
746 }
747}
748
749std::error_code SampleProfileWriterExtBinary::writeCtxSplitLayout(
750 const SampleProfileMap &ProfileMap) {
751 SampleProfileMap NestedProfileMap, FlatProfileMap;
752 splitProfileMapToTwo(ProfileMap, NestedProfileMap, FlatProfileMap);
753
754 // Flat SecFlag is pre-set in ExtBinaryHdrLayoutTable; findUnwrittenEntry
755 // picks the matching unwritten ProfSection / FuncOffsetSection slot.
756 const std::pair<SecType, const SampleProfileMap &> Sections[] = {
757 {SecProfSummary, ProfileMap}, {SecNameTable, ProfileMap},
758 {ProfSection, NestedProfileMap}, {FuncOffsetSection, NestedProfileMap},
759 {ProfSection, FlatProfileMap}, {FuncOffsetSection, FlatProfileMap},
760 {SecProfileSymbolList, ProfileMap}, {SecFuncMetadata, ProfileMap},
761 };
762 for (const auto &[Type, Map] : Sections)
763 if (std::error_code EC = writeOneSection(Type, ProfileMap: Map))
764 return EC;
765
766 return sampleprof_error::success;
767}
768
769void SampleProfileWriterExtBinary::configureCompositeProfile() {
770 ProfSection = WriteCompositeProf ? SecCompositeProfile : SecLBRProfile;
771 FuncOffsetSection =
772 WriteCompositeProf ? SecCompositeFuncOffsetTable : SecFuncOffsetTable;
773
774 // Change the section types in place to avoid duplicating the whole layout and
775 // its handling. Rewrite both legacy and composite entries so repeated writes
776 // can switch formats without losing configured flags.
777 for (auto &Entry : SectionHdrLayout) {
778 if (Entry.Type == SecFuncOffsetTable ||
779 Entry.Type == SecCompositeFuncOffsetTable)
780 Entry.Type = FuncOffsetSection;
781 else if (Entry.Type == SecLBRProfile || Entry.Type == SecCompositeProfile)
782 Entry.Type = ProfSection;
783 }
784}
785
786std::error_code SampleProfileWriterExtBinary::writeSections(
787 const SampleProfileMap &ProfileMap) {
788 // Rewrite the final configured layout immediately before its section types
789 // are consumed. Earlier layout configuration may replace SectionHdrLayout.
790 configureCompositeProfile();
791
792 std::error_code EC;
793 if (SecLayout == DefaultLayout)
794 EC = writeDefaultLayout(ProfileMap);
795 else if (SecLayout == CtxSplitLayout)
796 EC = writeCtxSplitLayout(ProfileMap);
797 else
798 llvm_unreachable("Unsupported layout");
799 return EC;
800}
801
802/// Write samples to a text file.
803///
804/// Note: it may be tempting to implement this in terms of
805/// FunctionSamples::print(). Please don't. The dump functionality is intended
806/// for debugging and has no specified form.
807///
808/// The format used here is more structured and deliberate because
809/// it needs to be parsed by the SampleProfileReaderText class.
810std::error_code SampleProfileWriterText::writeSample(const FunctionSamples &S) {
811 auto &OS = *OutputStream;
812 if (FunctionSamples::ProfileIsCS)
813 OS << "[" << S.getContext().toString() << "]:" << S.getTotalSamples();
814 else
815 OS << S.getFunction() << ":" << S.getTotalSamples();
816
817 if (Indent == 0)
818 OS << ":" << S.getHeadSamples();
819 OS << "\n";
820 LineCount++;
821
822 for (const auto &[Loc, Sample] : S.getBodySamples()) {
823 OS.indent(NumSpaces: Indent + 1);
824 Loc.print(OS);
825 OS << ": " << Sample.getSamples();
826
827 for (const auto &J : Sample.getSortedCallTargets())
828 OS << " " << J.first << ":" << J.second;
829 OS << "\n";
830 LineCount++;
831
832 if (const TypeCountMap *Map = S.findCallsiteTypeSamplesAt(Loc);
833 Map && !Map->empty()) {
834 OS.indent(NumSpaces: Indent + 1);
835 Loc.print(OS);
836 OS << ": ";
837 OS << kVTableProfPrefix;
838 for (const auto &[TypeName, Count] : *Map) {
839 OS << TypeName << ":" << Count << " ";
840 }
841 OS << "\n";
842 LineCount++;
843 }
844 }
845
846 Indent += 1;
847 for (const auto &[Loc, FunctionSamplesMap] : S.getCallsiteSamples()) {
848 for (const FunctionSamples &CalleeSamples :
849 make_second_range(c: FunctionSamplesMap)) {
850 OS.indent(NumSpaces: Indent);
851 Loc.print(OS);
852 OS << ": ";
853 if (std::error_code EC = writeSample(S: CalleeSamples))
854 return EC;
855 }
856
857 if (const TypeCountMap *Map = S.findCallsiteTypeSamplesAt(Loc);
858 Map && !Map->empty()) {
859 OS.indent(NumSpaces: Indent);
860 Loc.print(OS);
861 OS << ": ";
862 OS << kVTableProfPrefix;
863 for (const auto &[TypeId, Count] : *Map) {
864 OS << TypeId << ":" << Count << " ";
865 }
866 OS << "\n";
867 LineCount++;
868 }
869 }
870
871 Indent -= 1;
872
873 if (FunctionSamples::ProfileIsProbeBased) {
874 OS.indent(NumSpaces: Indent + 1);
875 OS << "!CFGChecksum: " << S.getFunctionHash() << "\n";
876 LineCount++;
877 }
878
879 if (S.getContext().getAllAttributes()) {
880 OS.indent(NumSpaces: Indent + 1);
881 OS << "!Attributes: " << S.getContext().getAllAttributes() << "\n";
882 LineCount++;
883 }
884
885 if (Indent == 0 && MarkFlatProfiles && S.getCallsiteSamples().size() == 0)
886 OS << " !Flat\n";
887
888 return sampleprof_error::success;
889}
890
891std::error_code
892SampleProfileWriterBinary::writeContextIdx(const SampleContext &Context) {
893 assert(!Context.hasContext() && "cs profile is not supported");
894 return writeNameIdx(FName: Context.getFunction());
895}
896
897std::error_code SampleProfileWriterBinary::writeNameIdx(FunctionId FName) {
898 auto &NTable = getNameTable();
899 const auto &Ret = NTable.find(Key: FName);
900 if (Ret == NTable.end())
901 return sampleprof_error::truncated_name_table;
902 encodeULEB128(Value: Ret->second, OS&: *OutputStream);
903 return sampleprof_error::success;
904}
905
906void SampleProfileWriterBinary::addName(FunctionId FName) {
907 auto &NTable = getNameTable();
908 NTable.insert(KV: std::make_pair(x&: FName, y: 0));
909}
910
911void SampleProfileWriterBinary::addContext(const SampleContext &Context) {
912 addName(FName: Context.getFunction());
913}
914
915void SampleProfileWriterBinary::addNames(const FunctionSamples &S) {
916 // Add all the names in indirect call targets.
917 for (const auto &I : S.getBodySamples()) {
918 const SampleRecord &Sample = I.second;
919 for (const auto &J : Sample.getCallTargets())
920 addName(FName: J.first);
921 }
922
923 // Recursively add all the names for inlined callsites.
924 for (const auto &J : S.getCallsiteSamples())
925 for (const auto &FS : J.second) {
926 const FunctionSamples &CalleeSamples = FS.second;
927 addName(FName: CalleeSamples.getFunction());
928 addNames(S: CalleeSamples);
929 }
930
931 if (!WriteVTableProf)
932 return;
933 // Add all the vtable names to NameTable.
934 for (const auto &VTableAccessCountMap :
935 llvm::make_second_range(c: S.getCallsiteTypeCounts())) {
936 // Add type name to NameTable.
937 for (const auto Type : llvm::make_first_range(c: VTableAccessCountMap)) {
938 addName(FName: Type);
939 }
940 }
941}
942
943void SampleProfileWriterExtBinaryBase::addContext(
944 const SampleContext &Context) {
945 if (Context.hasContext()) {
946 for (auto &Callsite : Context.getContextFrames())
947 SampleProfileWriterBinary::addName(FName: Callsite.Func);
948 CSNameTable.insert(KV: std::make_pair(x: Context, y: 0));
949 } else {
950 SampleProfileWriterBinary::addName(FName: Context.getFunction());
951 }
952}
953
954std::error_code SampleProfileWriterBinary::writeNameTable() {
955 auto &OS = *OutputStream;
956
957 // Write out the name table.
958 encodeULEB128(Value: NameTable.size(), OS);
959 for (const auto *Entry : stabilizeTable(Table&: NameTable)) {
960 OS << Entry->first;
961 encodeULEB128(Value: 0, OS);
962 }
963 return sampleprof_error::success;
964}
965
966std::error_code
967SampleProfileWriterBinary::writeMagicIdent(SampleProfileFormat Format) {
968 auto &OS = *OutputStream;
969 // Write file magic identifier.
970 encodeULEB128(Value: SPMagic(Format), OS);
971 encodeULEB128(Value: FormatVersion, OS);
972 return sampleprof_error::success;
973}
974
975std::error_code
976SampleProfileWriterBinary::writeHeader(const SampleProfileMap &ProfileMap) {
977 // When calling write on a different profile map, existing names should be
978 // cleared.
979 NameTable.clear();
980
981 writeMagicIdent(Format);
982
983 computeSummary(ProfileMap);
984 if (auto EC = writeSummary())
985 return EC;
986
987 // Generate the name table for all the functions referenced in the profile.
988 for (const auto &I : ProfileMap) {
989 addContext(Context: I.second.getContext());
990 addNames(S: I.second);
991 }
992
993 writeNameTable();
994 return sampleprof_error::success;
995}
996
997void SampleProfileWriterExtBinaryBase::setToCompressAllSections() {
998 for (auto &Entry : SectionHdrLayout)
999 addSecFlag(Entry, Flag: SecCommonFlags::SecFlagCompress);
1000}
1001
1002void SampleProfileWriterExtBinaryBase::setToCompressSection(SecType Type) {
1003 addSectionFlag(Type, Flag: SecCommonFlags::SecFlagCompress);
1004}
1005
1006void SampleProfileWriterExtBinaryBase::allocSecHdrTable() {
1007 support::endian::Writer Writer(*OutputStream, llvm::endianness::little);
1008
1009 Writer.write(Val: static_cast<uint64_t>(SectionHdrLayout.size()));
1010 SecHdrTableOffset = OutputStream->tell();
1011 for (uint32_t i = 0; i < SectionHdrLayout.size(); i++) {
1012 Writer.write(Val: static_cast<uint64_t>(-1));
1013 Writer.write(Val: static_cast<uint64_t>(-1));
1014 Writer.write(Val: static_cast<uint64_t>(-1));
1015 Writer.write(Val: static_cast<uint64_t>(-1));
1016 }
1017}
1018
1019std::error_code SampleProfileWriterExtBinaryBase::writeSecHdrTable() {
1020 assert(SecHdrTable.size() == SectionHdrLayout.size() &&
1021 "SecHdrTable entries doesn't match SectionHdrLayout");
1022 SmallVector<uint32_t, 16> IndexMap(SecHdrTable.size(), -1);
1023 for (uint32_t TableIdx = 0; TableIdx < SecHdrTable.size(); TableIdx++) {
1024 IndexMap[SecHdrTable[TableIdx].LayoutIndex] = TableIdx;
1025 }
1026
1027 // Write the section header table in the order specified in
1028 // SectionHdrLayout. SectionHdrLayout specifies the sections
1029 // order in which profile reader expect to read, so the section
1030 // header table should be written in the order in SectionHdrLayout.
1031 // Note that the section order in SecHdrTable may be different
1032 // from the order in SectionHdrLayout, for example, SecFuncOffsetTable
1033 // needs to be computed after SecLBRProfile (the order in SecHdrTable),
1034 // but it needs to be read before SecLBRProfile (the order in
1035 // SectionHdrLayout). So we use IndexMap above to switch the order.
1036 support::endian::SeekableWriter Writer(
1037 static_cast<raw_pwrite_stream &>(*OutputStream),
1038 llvm::endianness::little);
1039 for (uint32_t LayoutIdx = 0; LayoutIdx < SectionHdrLayout.size();
1040 LayoutIdx++) {
1041 assert(IndexMap[LayoutIdx] < SecHdrTable.size() &&
1042 "Incorrect LayoutIdx in SecHdrTable");
1043 auto Entry = SecHdrTable[IndexMap[LayoutIdx]];
1044 Writer.pwrite(Val: static_cast<uint64_t>(Entry.Type),
1045 Offset: SecHdrTableOffset + 4 * LayoutIdx * sizeof(uint64_t));
1046 Writer.pwrite(Val: static_cast<uint64_t>(Entry.Flags),
1047 Offset: SecHdrTableOffset + (4 * LayoutIdx + 1) * sizeof(uint64_t));
1048 Writer.pwrite(Val: static_cast<uint64_t>(Entry.Offset),
1049 Offset: SecHdrTableOffset + (4 * LayoutIdx + 2) * sizeof(uint64_t));
1050 Writer.pwrite(Val: static_cast<uint64_t>(Entry.Size),
1051 Offset: SecHdrTableOffset + (4 * LayoutIdx + 3) * sizeof(uint64_t));
1052 }
1053
1054 return sampleprof_error::success;
1055}
1056
1057std::error_code SampleProfileWriterExtBinaryBase::writeHeader(
1058 const SampleProfileMap &ProfileMap) {
1059 // Reject a version that cannot describe the selected profile encoding before
1060 // emitting any part of the header.
1061 if (WriteCompositeProf && FormatVersion < CompositeProfileVersion)
1062 return sampleprof_error::unsupported_version;
1063
1064 auto &OS = *OutputStream;
1065 FileStart = OS.tell();
1066 writeMagicIdent(Format);
1067
1068 allocSecHdrTable();
1069 return sampleprof_error::success;
1070}
1071
1072std::error_code SampleProfileWriterBinary::writeCallsiteVTableProf(
1073 const CallsiteTypeMap &CallsiteTypeMap, raw_ostream &OS) {
1074 assert(WriteVTableProf &&
1075 "writeCallsiteVTableProf should not be called if WriteVTableProf is "
1076 "false");
1077
1078 encodeULEB128(Value: CallsiteTypeMap.size(), OS);
1079 for (const auto &[Loc, TypeMap] : CallsiteTypeMap) {
1080 Loc.serialize(OS);
1081 if (std::error_code EC = serializeTypeMap(Map: TypeMap, NameTable: getNameTable(), OS))
1082 return EC;
1083 }
1084
1085 return sampleprof_error::success;
1086}
1087
1088std::error_code SampleProfileWriterBinary::writeSummary() {
1089 auto &OS = *OutputStream;
1090 encodeULEB128(Value: Summary->getTotalCount(), OS);
1091 encodeULEB128(Value: Summary->getMaxCount(), OS);
1092 encodeULEB128(Value: Summary->getMaxFunctionCount(), OS);
1093 encodeULEB128(Value: Summary->getNumCounts(), OS);
1094 encodeULEB128(Value: Summary->getNumFunctions(), OS);
1095 ArrayRef<ProfileSummaryEntry> Entries = Summary->getDetailedSummary();
1096 encodeULEB128(Value: Entries.size(), OS);
1097 for (auto Entry : Entries) {
1098 encodeULEB128(Value: Entry.Cutoff, OS);
1099 encodeULEB128(Value: Entry.MinCount, OS);
1100 encodeULEB128(Value: Entry.NumCounts, OS);
1101 }
1102 return sampleprof_error::success;
1103}
1104
1105std::error_code
1106SampleProfileWriterBinary::writeLBRProfile(const FunctionSamples &S,
1107 bool IsNested) {
1108 auto &OS = *OutputStream;
1109 if (WriteCompositeProf && !IsNested)
1110 encodeULEB128(Value: S.getHeadSamples(), OS);
1111 encodeULEB128(Value: S.getTotalSamples(), OS);
1112 encodeULEB128(Value: S.getBodySamples().size(), OS);
1113 for (const auto &I : S.getBodySamples()) {
1114 LineLocation Loc = I.first;
1115 const SampleRecord &Sample = I.second;
1116 Loc.serialize(OS);
1117 if (std::error_code EC = Sample.serialize(OS, NameTable: getNameTable()))
1118 return EC;
1119 }
1120 return sampleprof_error::success;
1121}
1122
1123namespace {
1124
1125/// A reusable stream that discards payload bytes while counting their size.
1126class PayloadSizeCountingStream final : public raw_ostream {
1127public:
1128 /// Avoid retaining payload data in raw_ostream's internal buffer.
1129 PayloadSizeCountingStream() { SetUnbuffered(); }
1130
1131 /// Prepare the stream to count another payload.
1132 void resetPayload() {
1133 PayloadSize = 0;
1134 Overflowed = false;
1135 }
1136
1137 /// Return whether the payload size exceeded the representable range.
1138 bool overflowed() const { return Overflowed; }
1139
1140 /// Return the complete payload size when overflowed() is false.
1141 uint64_t payloadSize() const { return PayloadSize; }
1142
1143private:
1144 /// Count incoming bytes without retaining their contents.
1145 void write_impl(const char *, size_t Size) override {
1146 if (Overflowed)
1147 return;
1148
1149 // Fail closed if the payload cannot be represented by its uint64_t size.
1150 if (Size > UINT64_MAX - PayloadSize) {
1151 Overflowed = true;
1152 return;
1153 }
1154 PayloadSize += Size;
1155 }
1156
1157 /// Report the number of bytes accepted from the current payload.
1158 uint64_t current_pos() const override { return PayloadSize; }
1159
1160 /// Number of bytes observed during the counting pass.
1161 uint64_t PayloadSize = 0;
1162 /// Whether the counted size no longer fits in uint64_t.
1163 bool Overflowed = false;
1164};
1165
1166} // namespace
1167
1168std::error_code SampleProfileWriterBinary::writeProfileType(
1169 ProfTypes Type, function_ref<std::error_code()> WritePayload) {
1170 // PayloadSizeStream temporarily owns the real output while the callback
1171 // writes through OutputStream. A nested call would therefore mistake the
1172 // real output for PayloadSizeCountingStream.
1173 if (WritingProfileType)
1174 return sampleprof_error::malformed;
1175 SaveAndRestore RestoreWritingProfileType(WritingProfileType, true);
1176
1177 // A profile block stores its payload size before the payload, but that size
1178 // is not known until it has been serialized. Count one complete serialization
1179 // without retaining its bytes, then emit the header and serialize it again.
1180 // TODO: Avoid serializing each payload twice while retaining bounded memory
1181 // use and compatibility with compressed section output.
1182 if (!PayloadSizeStream)
1183 PayloadSizeStream = std::make_unique<PayloadSizeCountingStream>();
1184 auto *SizeStream =
1185 static_cast<PayloadSizeCountingStream *>(PayloadSizeStream.get());
1186 SizeStream->resetPayload();
1187 OutputStream.swap(u&: PayloadSizeStream);
1188 std::error_code EC = WritePayload();
1189 OutputStream.swap(u&: PayloadSizeStream);
1190 if (EC)
1191 return EC;
1192 if (SizeStream->overflowed())
1193 return sampleprof_error::too_large;
1194
1195 // Emit the compact header followed by the second, materialized pass.
1196 auto &OS = *OutputStream;
1197 encodeULEB128(Value: Type, OS);
1198 encodeULEB128(Value: SizeStream->payloadSize(), OS);
1199 uint64_t PayloadStart = OS.tell();
1200 if (std::error_code SecondPassEC = WritePayload())
1201 return SecondPassEC;
1202
1203 // Reject a stateful callback that did not reproduce the counted payload.
1204 if (OS.tell() - PayloadStart != SizeStream->payloadSize())
1205 return sampleprof_error::malformed;
1206 return sampleprof_error::success;
1207}
1208
1209static bool hasNonEmptyLBRProfile(const FunctionSamples &S, bool IsNested) {
1210 return S.getTotalSamples() != 0 || (!IsNested && S.getHeadSamples() != 0) ||
1211 !S.getBodySamples().empty();
1212}
1213
1214std::error_code
1215SampleProfileWriterBinary::writeCompositeProfile(const FunctionSamples &S,
1216 bool IsNested) {
1217 auto &OS = *OutputStream;
1218 bool WriteLBRProf = hasNonEmptyLBRProfile(S, IsNested);
1219 // Other profile types should be added here.
1220 uint32_t TypesNum = WriteLBRProf;
1221
1222 // Write the number of profile types for function.
1223 encodeULEB128(Value: TypesNum, OS);
1224
1225 if (WriteLBRProf)
1226 return writeProfileType(Type: ProfTypeLBR,
1227 WritePayload: [&] { return writeLBRProfile(S, IsNested); });
1228 return sampleprof_error::success;
1229}
1230
1231std::error_code SampleProfileWriterBinary::writeBody(const FunctionSamples &S,
1232 bool IsNested) {
1233 auto &OS = *OutputStream;
1234 if (std::error_code EC = writeContextIdx(Context: S.getContext()))
1235 return EC;
1236
1237 // Emit all the body samples.
1238 if (WriteCompositeProf) {
1239 if (std::error_code EC = writeCompositeProfile(S, IsNested))
1240 return EC;
1241 } else {
1242 if (std::error_code EC = writeLBRProfile(S, IsNested))
1243 return EC;
1244 }
1245
1246 // Recursively emit all the callsite samples.
1247 uint64_t NumCallsites = 0;
1248 for (const auto &J : S.getCallsiteSamples())
1249 NumCallsites += J.second.size();
1250 encodeULEB128(Value: NumCallsites, OS);
1251 for (const auto &J : S.getCallsiteSamples())
1252 for (const auto &FS : J.second) {
1253 J.first.serialize(OS);
1254 if (std::error_code EC = writeBody(S: FS.second, /*IsNested=*/true))
1255 return EC;
1256 }
1257
1258 if (WriteVTableProf)
1259 return writeCallsiteVTableProf(CallsiteTypeMap: S.getCallsiteTypeCounts(), OS);
1260
1261 return sampleprof_error::success;
1262}
1263
1264/// Write samples of a top-level function to a binary file.
1265///
1266/// \returns true if the samples were written successfully, false otherwise.
1267std::error_code
1268SampleProfileWriterBinary::writeSample(const FunctionSamples &S) {
1269 encodeULEB128(Value: S.getHeadSamples(), OS&: *OutputStream);
1270 return writeBody(S, /*IsNested=*/false);
1271}
1272
1273/// Create a sample profile file writer based on the specified format.
1274///
1275/// \param Filename The file to create.
1276///
1277/// \param Format Encoding format for the profile file.
1278///
1279/// \returns an error code indicating the status of the created writer.
1280ErrorOr<std::unique_ptr<SampleProfileWriter>>
1281SampleProfileWriter::create(StringRef Filename, SampleProfileFormat Format) {
1282 std::error_code EC;
1283 std::unique_ptr<raw_ostream> OS;
1284 if (Format == SPF_Binary || Format == SPF_Ext_Binary)
1285 OS.reset(p: new raw_fd_ostream(Filename, EC, sys::fs::OF_None));
1286 else
1287 OS.reset(p: new raw_fd_ostream(Filename, EC, sys::fs::OF_TextWithCRLF));
1288 if (EC)
1289 return EC;
1290
1291 return create(OS, Format);
1292}
1293
1294/// Create a sample profile stream writer based on the specified format.
1295///
1296/// \param OS The output stream to store the profile data to.
1297///
1298/// \param Format Encoding format for the profile file.
1299///
1300/// \returns an error code indicating the status of the created writer.
1301ErrorOr<std::unique_ptr<SampleProfileWriter>>
1302SampleProfileWriter::create(std::unique_ptr<raw_ostream> &OS,
1303 SampleProfileFormat Format) {
1304 std::error_code EC;
1305 std::unique_ptr<SampleProfileWriter> Writer;
1306
1307 // Currently only Text and Extended Binary format are supported for CSSPGO.
1308 if ((FunctionSamples::ProfileIsCS || FunctionSamples::ProfileIsProbeBased) &&
1309 Format == SPF_Binary)
1310 return sampleprof_error::unsupported_writing_format;
1311
1312 if (Format == SPF_Binary)
1313 Writer.reset(p: new SampleProfileWriterRawBinary(OS));
1314 else if (Format == SPF_Ext_Binary)
1315 Writer.reset(p: new SampleProfileWriterExtBinary(OS));
1316 else if (Format == SPF_Text)
1317 Writer.reset(p: new SampleProfileWriterText(OS));
1318 else if (Format == SPF_GCC)
1319 EC = sampleprof_error::unsupported_writing_format;
1320 else
1321 EC = sampleprof_error::unrecognized_format;
1322
1323 if (EC)
1324 return EC;
1325
1326 Writer->Format = Format;
1327 if (Format != SPF_Ext_Binary) {
1328 Writer->setFormatVersion(DefaultVersion);
1329 } else {
1330 if (!formatVersionIsSupported(Version: RequestedVersion))
1331 return sampleprof_error::unsupported_version;
1332
1333 // Composite output defaults to its first compatible format version.
1334 // Preserve a compatible version explicitly selected by the user.
1335 if (ExtBinaryCompositeProf) {
1336 if (RequestedVersion.getNumOccurrences() == 0) {
1337 Writer->setFormatVersion(CompositeProfileVersion);
1338 } else {
1339 if (RequestedVersion < CompositeProfileVersion)
1340 return sampleprof_error::unsupported_version;
1341 Writer->setFormatVersion(RequestedVersion);
1342 }
1343 // Keep subsequent writes independent of the global command-line option.
1344 Writer->setUseCompositeProfile(true);
1345 } else {
1346 Writer->setFormatVersion(RequestedVersion);
1347 }
1348 }
1349
1350 return std::move(Writer);
1351}
1352
1353void SampleProfileWriter::computeSummary(const SampleProfileMap &ProfileMap) {
1354 SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs);
1355 Summary = Builder.computeSummaryForProfiles(Profiles: ProfileMap);
1356}
1357