1//===- SampleProfWriter.cpp - Write LLVM sample profile data --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the class that writes LLVM sample profiles. It
10// supports two file formats: text and binary. The textual representation
11// is useful for debugging and testing purposes. The binary representation
12// is more compact, resulting in smaller file sizes. However, they can
13// both be used interchangeably.
14//
15// See lib/ProfileData/SampleProfReader.cpp for documentation on each of the
16// supported formats.
17//
18//===----------------------------------------------------------------------===//
19
20#include "llvm/ProfileData/SampleProfWriter.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/ProfileData/ProfileCommon.h"
23#include "llvm/ProfileData/SampleProf.h"
24#include "llvm/Support/Compression.h"
25#include "llvm/Support/EndianStream.h"
26#include "llvm/Support/ErrorOr.h"
27#include "llvm/Support/FileSystem.h"
28#include "llvm/Support/LEB128.h"
29#include "llvm/Support/MD5.h"
30#include "llvm/Support/raw_ostream.h"
31#include <cmath>
32#include <cstdint>
33#include <memory>
34#include <system_error>
35#include <utility>
36#include <vector>
37
38#define DEBUG_TYPE "llvm-profdata"
39
40using namespace llvm;
41using namespace sampleprof;
42
43// To begin with, make this option off by default.
44static cl::opt<bool> ExtBinaryWriteVTableTypeProf(
45 "extbinary-write-vtable-type-prof", cl::init(Val: false), cl::Hidden,
46 cl::desc("Write vtable type profile in ext-binary sample profile writer"));
47
48static cl::opt<uint64_t> RequestedVersion(
49 "sample-profile-format-version", cl::init(Val: DefaultVersion), cl::Hidden,
50 cl::desc("Format version to write for extensible binary profiles"));
51
52static cl::opt<bool>
53 WriteMD5ProfSymList("md5-prof-sym-list", cl::init(Val: false), cl::Hidden,
54 cl::desc("Write ProfileSymbolList (Cold Symbols) as "
55 "64-bit MD5 hashes in Eytzinger layout"));
56
57namespace llvm {
58namespace support {
59namespace endian {
60namespace {
61
62// Adapter class to llvm::support::endian::Writer for pwrite().
63struct SeekableWriter {
64 raw_pwrite_stream &OS;
65 endianness Endian;
66 SeekableWriter(raw_pwrite_stream &OS, endianness Endian)
67 : OS(OS), Endian(Endian) {}
68
69 template <typename ValueType> void pwrite(ValueType Val, size_t Offset) {
70 std::string StringBuf;
71 raw_string_ostream SStream(StringBuf);
72 Writer(SStream, Endian).write(Val);
73 OS.pwrite(Ptr: StringBuf.data(), Size: StringBuf.size(), Offset);
74 }
75};
76
77} // namespace
78} // namespace endian
79} // namespace support
80} // namespace llvm
81
82DefaultFunctionPruningStrategy::DefaultFunctionPruningStrategy(
83 SampleProfileMap &ProfileMap, size_t OutputSizeLimit)
84 : FunctionPruningStrategy(ProfileMap, OutputSizeLimit) {
85 sortFuncProfiles(ProfileMap, SortedProfiles&: SortedFunctions);
86}
87
88void DefaultFunctionPruningStrategy::Erase(size_t CurrentOutputSize) {
89 double D = (double)OutputSizeLimit / CurrentOutputSize;
90 size_t NewSize = (size_t)round(x: ProfileMap.size() * D * D);
91 size_t NumToRemove = ProfileMap.size() - NewSize;
92 if (NumToRemove < 1)
93 NumToRemove = 1;
94
95 assert(NumToRemove <= SortedFunctions.size());
96 for (const NameFunctionSamples &E :
97 llvm::drop_begin(RangeOrContainer&: SortedFunctions, N: SortedFunctions.size() - NumToRemove))
98 ProfileMap.erase(Key: E.first);
99 SortedFunctions.resize(new_size: SortedFunctions.size() - NumToRemove);
100}
101
102std::error_code SampleProfileWriter::writeWithSizeLimitInternal(
103 SampleProfileMap &ProfileMap, size_t OutputSizeLimit,
104 FunctionPruningStrategy *Strategy) {
105 if (OutputSizeLimit == 0)
106 return write(ProfileMap);
107
108 size_t OriginalFunctionCount = ProfileMap.size();
109
110 std::unique_ptr<raw_ostream> OriginalOutputStream;
111 OutputStream.swap(u&: OriginalOutputStream);
112
113 size_t IterationCount = 0;
114 size_t TotalSize;
115
116 SmallVector<char> StringBuffer;
117 do {
118 StringBuffer.clear();
119 OutputStream.reset(p: new raw_svector_ostream(StringBuffer));
120 if (std::error_code EC = write(ProfileMap))
121 return EC;
122
123 TotalSize = StringBuffer.size();
124 // On Windows every "\n" is actually written as "\r\n" to disk but not to
125 // memory buffer, this difference should be added when considering the total
126 // output size.
127#ifdef _WIN32
128 if (Format == SPF_Text)
129 TotalSize += LineCount;
130#endif
131 if (TotalSize <= OutputSizeLimit)
132 break;
133
134 Strategy->Erase(CurrentOutputSize: TotalSize);
135 IterationCount++;
136 } while (ProfileMap.size() != 0);
137
138 if (ProfileMap.size() == 0)
139 return sampleprof_error::too_large;
140
141 OutputStream.swap(u&: OriginalOutputStream);
142 OutputStream->write(Ptr: StringBuffer.data(), Size: StringBuffer.size());
143 LLVM_DEBUG(dbgs() << "Profile originally has " << OriginalFunctionCount
144 << " functions, reduced to " << ProfileMap.size() << " in "
145 << IterationCount << " iterations\n");
146 // Silence warning on Release build.
147 (void)OriginalFunctionCount;
148 (void)IterationCount;
149 return sampleprof_error::success;
150}
151
152std::error_code
153SampleProfileWriter::writeFuncProfiles(const SampleProfileMap &ProfileMap) {
154 std::vector<NameFunctionSamples> V;
155 sortFuncProfiles(ProfileMap, SortedProfiles&: V);
156 for (const auto &I : V) {
157 if (std::error_code EC = writeSample(S: *I.second))
158 return EC;
159 }
160 return sampleprof_error::success;
161}
162
163std::error_code SampleProfileWriter::write(const SampleProfileMap &ProfileMap) {
164 if (std::error_code EC = writeHeader(ProfileMap))
165 return EC;
166
167 if (std::error_code EC = writeFuncProfiles(ProfileMap))
168 return EC;
169
170 return sampleprof_error::success;
171}
172
173/// Return the current position and prepare to use it as the start
174/// position of a section given the section type \p Type and its position
175/// \p LayoutIdx in SectionHdrLayout.
176uint64_t
177SampleProfileWriterExtBinaryBase::markSectionStart(SecType Type,
178 uint32_t LayoutIdx) {
179 uint64_t SectionStart = OutputStream->tell();
180 assert(LayoutIdx < SectionHdrLayout.size() && "LayoutIdx out of range");
181 const auto &Entry = SectionHdrLayout[LayoutIdx];
182 assert(Entry.Type == Type && "Unexpected section type");
183 // Use LocalBuf as a temporary output for writting data.
184 if (hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagCompress))
185 LocalBufStream.swap(u&: OutputStream);
186 return SectionStart;
187}
188
189std::error_code SampleProfileWriterExtBinaryBase::compressAndOutput() {
190 if (!llvm::compression::zlib::isAvailable())
191 return sampleprof_error::zlib_unavailable;
192 std::string &UncompressedStrings =
193 static_cast<raw_string_ostream *>(LocalBufStream.get())->str();
194 if (UncompressedStrings.size() == 0)
195 return sampleprof_error::success;
196 auto &OS = *OutputStream;
197 SmallVector<uint8_t, 128> CompressedStrings;
198 compression::zlib::compress(Input: arrayRefFromStringRef(Input: UncompressedStrings),
199 CompressedBuffer&: CompressedStrings,
200 Level: compression::zlib::BestSizeCompression);
201 encodeULEB128(Value: UncompressedStrings.size(), OS);
202 encodeULEB128(Value: CompressedStrings.size(), OS);
203 OS << toStringRef(Input: CompressedStrings);
204 UncompressedStrings.clear();
205 return sampleprof_error::success;
206}
207
208/// Add a new section into section header table given the section type
209/// \p Type, its position \p LayoutIdx in SectionHdrLayout and the
210/// location \p SectionStart where the section should be written to.
211std::error_code SampleProfileWriterExtBinaryBase::addNewSection(
212 SecType Type, uint32_t LayoutIdx, uint64_t SectionStart) {
213 assert(LayoutIdx < SectionHdrLayout.size() && "LayoutIdx out of range");
214 const auto &Entry = SectionHdrLayout[LayoutIdx];
215 assert(Entry.Type == Type && "Unexpected section type");
216 if (hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagCompress)) {
217 LocalBufStream.swap(u&: OutputStream);
218 if (std::error_code EC = compressAndOutput())
219 return EC;
220 }
221 SecHdrTable.push_back(x: {.Type: Type, .Flags: Entry.Flags, .Offset: SectionStart - FileStart,
222 .Size: OutputStream->tell() - SectionStart, .LayoutIndex: LayoutIdx});
223 return sampleprof_error::success;
224}
225
226std::error_code
227SampleProfileWriterExtBinaryBase::write(const SampleProfileMap &ProfileMap) {
228 // When calling write on a different profile map, existing states should be
229 // cleared.
230 NameTable.clear();
231 CSNameTable.clear();
232 SecHdrTable.clear();
233
234 if (std::error_code EC = writeHeader(ProfileMap))
235 return EC;
236
237 std::string LocalBuf;
238 LocalBufStream = std::make_unique<raw_string_ostream>(args&: LocalBuf);
239 if (std::error_code EC = writeSections(ProfileMap))
240 return EC;
241
242 if (std::error_code EC = writeSecHdrTable())
243 return EC;
244
245 return sampleprof_error::success;
246}
247
248std::error_code SampleProfileWriterExtBinaryBase::writeContextIdx(
249 const SampleContext &Context) {
250 if (Context.hasContext())
251 return writeCSNameIdx(Context);
252 else
253 return SampleProfileWriterBinary::writeNameIdx(FName: Context.getFunction());
254}
255
256std::error_code
257SampleProfileWriterExtBinaryBase::writeCSNameIdx(const SampleContext &Context) {
258 const auto &Ret = CSNameTable.find(Key: Context);
259 if (Ret == CSNameTable.end())
260 return sampleprof_error::truncated_name_table;
261 encodeULEB128(Value: Ret->second, OS&: *OutputStream);
262 return sampleprof_error::success;
263}
264
265std::error_code
266SampleProfileWriterExtBinaryBase::writeSample(const FunctionSamples &S) {
267 uint64_t Offset = OutputStream->tell();
268 auto &Context = S.getContext();
269 FuncOffsetTable[Context] = Offset - SecLBRProfileStart;
270 encodeULEB128(Value: S.getHeadSamples(), OS&: *OutputStream);
271 return writeBody(S);
272}
273
274std::error_code SampleProfileWriterExtBinaryBase::writeFuncOffsetTable() {
275 auto &OS = *OutputStream;
276
277 // Write out the table size.
278 encodeULEB128(Value: FuncOffsetTable.size(), OS);
279
280 // Write out FuncOffsetTable.
281 auto WriteItem = [&](const SampleContext &Context, uint64_t Offset) {
282 if (std::error_code EC = writeContextIdx(Context))
283 return EC;
284 encodeULEB128(Value: Offset, OS);
285 return (std::error_code)sampleprof_error::success;
286 };
287
288 if (FunctionSamples::ProfileIsCS) {
289 // Sort the contexts before writing them out. This is to help fast load all
290 // context profiles for a function as well as their callee contexts which
291 // can help profile-guided importing for ThinLTO.
292 std::map<SampleContext, uint64_t> OrderedFuncOffsetTable(
293 FuncOffsetTable.begin(), FuncOffsetTable.end());
294 for (const auto &Entry : OrderedFuncOffsetTable) {
295 if (std::error_code EC = WriteItem(Entry.first, Entry.second))
296 return EC;
297 }
298 addSectionFlag(Type: SecFuncOffsetTable, Flag: SecFuncOffsetFlags::SecFlagOrdered);
299 } else {
300 for (const auto &Entry : FuncOffsetTable) {
301 if (std::error_code EC = WriteItem(Entry.first, Entry.second))
302 return EC;
303 }
304 }
305
306 FuncOffsetTable.clear();
307 return sampleprof_error::success;
308}
309
310std::error_code SampleProfileWriterExtBinaryBase::writeFuncMetadata(
311 const FunctionSamples &FunctionProfile) {
312 auto &OS = *OutputStream;
313 if (std::error_code EC = writeContextIdx(Context: FunctionProfile.getContext()))
314 return EC;
315
316 if (FunctionSamples::ProfileIsProbeBased)
317 encodeULEB128(Value: FunctionProfile.getFunctionHash(), OS);
318 if (FunctionSamples::ProfileIsCS || FunctionSamples::ProfileIsPreInlined) {
319 encodeULEB128(Value: FunctionProfile.getContext().getAllAttributes(), OS);
320 }
321
322 if (!FunctionSamples::ProfileIsCS) {
323 // Recursively emit attributes for all callee samples.
324 uint64_t NumCallsites = 0;
325 for (const auto &J : FunctionProfile.getCallsiteSamples())
326 NumCallsites += J.second.size();
327 encodeULEB128(Value: NumCallsites, OS);
328 for (const auto &J : FunctionProfile.getCallsiteSamples()) {
329 for (const auto &FS : J.second) {
330 LineLocation Loc = J.first;
331 encodeULEB128(Value: Loc.LineOffset, OS);
332 encodeULEB128(Value: Loc.Discriminator, OS);
333 if (std::error_code EC = writeFuncMetadata(FunctionProfile: FS.second))
334 return EC;
335 }
336 }
337 }
338
339 return sampleprof_error::success;
340}
341
342std::error_code SampleProfileWriterExtBinaryBase::writeFuncMetadata(
343 const SampleProfileMap &Profiles) {
344 if (!FunctionSamples::ProfileIsProbeBased && !FunctionSamples::ProfileIsCS &&
345 !FunctionSamples::ProfileIsPreInlined)
346 return sampleprof_error::success;
347 for (const auto &Entry : Profiles) {
348 if (std::error_code EC = writeFuncMetadata(FunctionProfile: Entry.second))
349 return EC;
350 }
351 return sampleprof_error::success;
352}
353
354template <class KeyT, class ValT>
355static SmallVector<std::pair<KeyT, ValT> *, 0>
356stabilizeTable(MapVector<KeyT, ValT> &Table) {
357 SmallVector<std::pair<KeyT, ValT> *, 0> Entries(
358 llvm::make_pointer_range(Table));
359
360 llvm::sort(Entries,
361 [](const auto *L, const auto *R) { return L->first < R->first; });
362
363 for (const auto &[I, Entry] : llvm::enumerate(Entries))
364 Entry->second = I;
365
366 return Entries;
367}
368
369std::error_code SampleProfileWriterExtBinaryBase::writeNameTable() {
370 if (!UseMD5)
371 return SampleProfileWriterBinary::writeNameTable();
372
373 auto &OS = *OutputStream;
374
375 // Write out the MD5 name table. We wrote unencoded MD5 so reader can
376 // retrieve the name using the name index without having to read the
377 // whole name table.
378 encodeULEB128(Value: NameTable.size(), OS);
379 support::endian::Writer Writer(OS, llvm::endianness::little);
380 for (const auto *Entry : stabilizeTable(Table&: NameTable))
381 Writer.write(Val: Entry->first.getHashCode());
382 return sampleprof_error::success;
383}
384
385std::error_code SampleProfileWriterExtBinaryBase::writeNameTableSection(
386 const SampleProfileMap &ProfileMap) {
387 for (const auto &I : ProfileMap) {
388 addContext(Context: I.second.getContext());
389 addNames(S: I.second);
390 }
391
392 // If NameTable contains ".__uniq." suffix, set SecFlagUniqSuffix flag
393 // so compiler won't strip the suffix during profile matching after
394 // seeing the flag in the profile.
395 // Original names are unavailable if using MD5, so this option has no use.
396 if (!UseMD5) {
397 for (const auto &I : NameTable) {
398 if (I.first.stringRef().contains(Other: FunctionSamples::UniqSuffix)) {
399 addSectionFlag(Type: SecNameTable, Flag: SecNameTableFlags::SecFlagUniqSuffix);
400 break;
401 }
402 }
403 }
404
405 if (auto EC = writeNameTable())
406 return EC;
407 return sampleprof_error::success;
408}
409
410std::error_code SampleProfileWriterExtBinaryBase::writeCSNameTableSection() {
411 auto &OS = *OutputStream;
412 encodeULEB128(Value: CSNameTable.size(), OS);
413 support::endian::Writer Writer(OS, llvm::endianness::little);
414 for (const auto *Entry : stabilizeTable(Table&: CSNameTable)) {
415 auto Frames = Entry->first.getContextFrames();
416 encodeULEB128(Value: Frames.size(), OS);
417 for (auto &Callsite : Frames) {
418 if (std::error_code EC = writeNameIdx(FName: Callsite.Func))
419 return EC;
420 encodeULEB128(Value: Callsite.Location.LineOffset, OS);
421 encodeULEB128(Value: Callsite.Location.Discriminator, OS);
422 }
423 }
424
425 return sampleprof_error::success;
426}
427
428std::error_code
429SampleProfileWriterExtBinaryBase::writeProfileSymbolListSection() {
430 if (WriteMD5ProfSymList)
431 return writeMD5ProfileSymbolListSection();
432 return writeStringBasedProfileSymbolListSection();
433}
434
435std::error_code
436SampleProfileWriterExtBinaryBase::writeStringBasedProfileSymbolListSection() {
437 assert((!ProfSymList || !ProfSymList->isMD5()) &&
438 "Writing string-based ProfileSymbolListSection from MD5 table "
439 "not yet implemented");
440 if (ProfSymList && ProfSymList->size() > 0)
441 if (std::error_code EC = ProfSymList->write(OS&: *OutputStream))
442 return EC;
443
444 return sampleprof_error::success;
445}
446
447std::error_code
448SampleProfileWriterExtBinaryBase::writeMD5ProfileSymbolListSection() {
449 if (!ProfSymList || ProfSymList->size() == 0)
450 return sampleprof_error::success;
451 assert(!ProfSymList->isMD5() &&
452 "Writing MD5 ProfileSymbolListSection from existing MD5 "
453 "table not yet implemented");
454
455 auto &OS = *OutputStream;
456 std::vector<uint64_t> Keys = ProfSymList->collectGUIDs();
457
458 auto Table =
459 llvm::EytzingerTable<support::ulittle64_t>::create(Keys: std::move(Keys));
460
461 OS.write(Ptr: reinterpret_cast<const char *>(Table.data()),
462 Size: Table.size() * sizeof(support::ulittle64_t));
463 return sampleprof_error::success;
464}
465
466std::error_code SampleProfileWriterExtBinaryBase::writeOneSection(
467 SecType Type, uint32_t LayoutIdx, const SampleProfileMap &ProfileMap) {
468 // The setting of SecFlagCompress should happen before markSectionStart.
469 if (Type == SecProfileSymbolList && ProfSymList && ProfSymList->toCompress())
470 setToCompressSection(SecProfileSymbolList);
471 if (Type == SecFuncMetadata && FunctionSamples::ProfileIsProbeBased)
472 addSectionFlag(Type: SecFuncMetadata, Flag: SecFuncMetadataFlags::SecFlagIsProbeBased);
473 if (Type == SecFuncMetadata &&
474 (FunctionSamples::ProfileIsCS || FunctionSamples::ProfileIsPreInlined))
475 addSectionFlag(Type: SecFuncMetadata, Flag: SecFuncMetadataFlags::SecFlagHasAttribute);
476 if (Type == SecProfSummary && FunctionSamples::ProfileIsCS)
477 addSectionFlag(Type: SecProfSummary, Flag: SecProfSummaryFlags::SecFlagFullContext);
478 if (Type == SecProfSummary && FunctionSamples::ProfileIsPreInlined)
479 addSectionFlag(Type: SecProfSummary, Flag: SecProfSummaryFlags::SecFlagIsPreInlined);
480 if (Type == SecProfSummary && FunctionSamples::ProfileIsFS)
481 addSectionFlag(Type: SecProfSummary, Flag: SecProfSummaryFlags::SecFlagFSDiscriminator);
482 if (Type == SecProfSummary && ExtBinaryWriteVTableTypeProf)
483 addSectionFlag(Type: SecProfSummary,
484 Flag: SecProfSummaryFlags::SecFlagHasVTableTypeProf);
485 if (Type == SecProfileSymbolList && WriteMD5ProfSymList)
486 addSectionFlag(Type: SecProfileSymbolList, Flag: SecProfileSymbolListFlags::SecFlagMD5);
487
488 uint64_t SectionStart = markSectionStart(Type, LayoutIdx);
489 switch (Type) {
490 case SecProfSummary:
491 computeSummary(ProfileMap);
492 if (auto EC = writeSummary())
493 return EC;
494 break;
495 case SecNameTable:
496 if (auto EC = writeNameTableSection(ProfileMap))
497 return EC;
498 break;
499 case SecCSNameTable:
500 if (auto EC = writeCSNameTableSection())
501 return EC;
502 break;
503 case SecLBRProfile:
504 SecLBRProfileStart = OutputStream->tell();
505 if (std::error_code EC = writeFuncProfiles(ProfileMap))
506 return EC;
507 break;
508 case SecFuncOffsetTable:
509 if (auto EC = writeFuncOffsetTable())
510 return EC;
511 break;
512 case SecFuncMetadata:
513 if (std::error_code EC = writeFuncMetadata(Profiles: ProfileMap))
514 return EC;
515 break;
516 case SecProfileSymbolList:
517 if (auto EC = writeProfileSymbolListSection())
518 return EC;
519 break;
520 default:
521 if (auto EC = writeCustomSection(Type))
522 return EC;
523 break;
524 }
525 if (std::error_code EC = addNewSection(Type, LayoutIdx, SectionStart))
526 return EC;
527 return sampleprof_error::success;
528}
529
530SampleProfileWriterExtBinary::SampleProfileWriterExtBinary(
531 std::unique_ptr<raw_ostream> &OS)
532 : SampleProfileWriterExtBinaryBase(OS) {
533 WriteVTableProf = ExtBinaryWriteVTableTypeProf;
534}
535
536std::error_code SampleProfileWriterExtBinary::writeDefaultLayout(
537 const SampleProfileMap &ProfileMap) {
538 // The const indices passed to writeOneSection below are specifying the
539 // positions of the sections in SectionHdrLayout. Look at
540 // initSectionHdrLayout to find out where each section is located in
541 // SectionHdrLayout.
542 if (auto EC = writeOneSection(Type: SecProfSummary, LayoutIdx: 0, ProfileMap))
543 return EC;
544 if (auto EC = writeOneSection(Type: SecNameTable, LayoutIdx: 1, ProfileMap))
545 return EC;
546 if (auto EC = writeOneSection(Type: SecCSNameTable, LayoutIdx: 2, ProfileMap))
547 return EC;
548 if (auto EC = writeOneSection(Type: SecLBRProfile, LayoutIdx: 4, ProfileMap))
549 return EC;
550 if (auto EC = writeOneSection(Type: SecProfileSymbolList, LayoutIdx: 5, ProfileMap))
551 return EC;
552 if (auto EC = writeOneSection(Type: SecFuncOffsetTable, LayoutIdx: 3, ProfileMap))
553 return EC;
554 if (auto EC = writeOneSection(Type: SecFuncMetadata, LayoutIdx: 6, ProfileMap))
555 return EC;
556 return sampleprof_error::success;
557}
558
559static void splitProfileMapToTwo(const SampleProfileMap &ProfileMap,
560 SampleProfileMap &ContextProfileMap,
561 SampleProfileMap &NoContextProfileMap) {
562 for (const auto &I : ProfileMap) {
563 if (I.second.getCallsiteSamples().size())
564 ContextProfileMap.insert(x: {I.first, I.second});
565 else
566 NoContextProfileMap.insert(x: {I.first, I.second});
567 }
568}
569
570std::error_code SampleProfileWriterExtBinary::writeCtxSplitLayout(
571 const SampleProfileMap &ProfileMap) {
572 SampleProfileMap ContextProfileMap, NoContextProfileMap;
573 splitProfileMapToTwo(ProfileMap, ContextProfileMap, NoContextProfileMap);
574
575 if (auto EC = writeOneSection(Type: SecProfSummary, LayoutIdx: 0, ProfileMap))
576 return EC;
577 if (auto EC = writeOneSection(Type: SecNameTable, LayoutIdx: 1, ProfileMap))
578 return EC;
579 if (auto EC = writeOneSection(Type: SecLBRProfile, LayoutIdx: 3, ProfileMap: ContextProfileMap))
580 return EC;
581 if (auto EC = writeOneSection(Type: SecFuncOffsetTable, LayoutIdx: 2, ProfileMap: ContextProfileMap))
582 return EC;
583 // Mark the section to have no context. Note section flag needs to be set
584 // before writing the section.
585 addSectionFlag(SectionIdx: 5, Flag: SecCommonFlags::SecFlagFlat);
586 if (auto EC = writeOneSection(Type: SecLBRProfile, LayoutIdx: 5, ProfileMap: NoContextProfileMap))
587 return EC;
588 // Mark the section to have no context. Note section flag needs to be set
589 // before writing the section.
590 addSectionFlag(SectionIdx: 4, Flag: SecCommonFlags::SecFlagFlat);
591 if (auto EC = writeOneSection(Type: SecFuncOffsetTable, LayoutIdx: 4, ProfileMap: NoContextProfileMap))
592 return EC;
593 if (auto EC = writeOneSection(Type: SecProfileSymbolList, LayoutIdx: 6, ProfileMap))
594 return EC;
595 if (auto EC = writeOneSection(Type: SecFuncMetadata, LayoutIdx: 7, ProfileMap))
596 return EC;
597
598 return sampleprof_error::success;
599}
600
601std::error_code SampleProfileWriterExtBinary::writeSections(
602 const SampleProfileMap &ProfileMap) {
603 std::error_code EC;
604 if (SecLayout == DefaultLayout)
605 EC = writeDefaultLayout(ProfileMap);
606 else if (SecLayout == CtxSplitLayout)
607 EC = writeCtxSplitLayout(ProfileMap);
608 else
609 llvm_unreachable("Unsupported layout");
610 return EC;
611}
612
613/// Write samples to a text file.
614///
615/// Note: it may be tempting to implement this in terms of
616/// FunctionSamples::print(). Please don't. The dump functionality is intended
617/// for debugging and has no specified form.
618///
619/// The format used here is more structured and deliberate because
620/// it needs to be parsed by the SampleProfileReaderText class.
621std::error_code SampleProfileWriterText::writeSample(const FunctionSamples &S) {
622 auto &OS = *OutputStream;
623 if (FunctionSamples::ProfileIsCS)
624 OS << "[" << S.getContext().toString() << "]:" << S.getTotalSamples();
625 else
626 OS << S.getFunction() << ":" << S.getTotalSamples();
627
628 if (Indent == 0)
629 OS << ":" << S.getHeadSamples();
630 OS << "\n";
631 LineCount++;
632
633 SampleSorter<LineLocation, SampleRecord> SortedSamples(S.getBodySamples());
634 for (const auto &I : SortedSamples.get()) {
635 LineLocation Loc = I->first;
636 const SampleRecord &Sample = I->second;
637 OS.indent(NumSpaces: Indent + 1);
638 Loc.print(OS);
639 OS << ": " << Sample.getSamples();
640
641 for (const auto &J : Sample.getSortedCallTargets())
642 OS << " " << J.first << ":" << J.second;
643 OS << "\n";
644 LineCount++;
645
646 if (const TypeCountMap *Map = S.findCallsiteTypeSamplesAt(Loc);
647 Map && !Map->empty()) {
648 OS.indent(NumSpaces: Indent + 1);
649 Loc.print(OS);
650 OS << ": ";
651 OS << kVTableProfPrefix;
652 for (const auto [TypeName, Count] : *Map) {
653 OS << TypeName << ":" << Count << " ";
654 }
655 OS << "\n";
656 LineCount++;
657 }
658 }
659
660 SampleSorter<LineLocation, FunctionSamplesMap> SortedCallsiteSamples(
661 S.getCallsiteSamples());
662 Indent += 1;
663 for (const auto *Element : SortedCallsiteSamples.get()) {
664 // Element is a pointer to a pair of LineLocation and FunctionSamplesMap.
665 const auto &[Loc, FunctionSamplesMap] = *Element;
666 for (const FunctionSamples &CalleeSamples :
667 make_second_range(c: FunctionSamplesMap)) {
668 OS.indent(NumSpaces: Indent);
669 Loc.print(OS);
670 OS << ": ";
671 if (std::error_code EC = writeSample(S: CalleeSamples))
672 return EC;
673 }
674
675 if (const TypeCountMap *Map = S.findCallsiteTypeSamplesAt(Loc);
676 Map && !Map->empty()) {
677 OS.indent(NumSpaces: Indent);
678 Loc.print(OS);
679 OS << ": ";
680 OS << kVTableProfPrefix;
681 for (const auto [TypeId, Count] : *Map) {
682 OS << TypeId << ":" << Count << " ";
683 }
684 OS << "\n";
685 LineCount++;
686 }
687 }
688
689 Indent -= 1;
690
691 if (FunctionSamples::ProfileIsProbeBased) {
692 OS.indent(NumSpaces: Indent + 1);
693 OS << "!CFGChecksum: " << S.getFunctionHash() << "\n";
694 LineCount++;
695 }
696
697 if (S.getContext().getAllAttributes()) {
698 OS.indent(NumSpaces: Indent + 1);
699 OS << "!Attributes: " << S.getContext().getAllAttributes() << "\n";
700 LineCount++;
701 }
702
703 if (Indent == 0 && MarkFlatProfiles && S.getCallsiteSamples().size() == 0)
704 OS << " !Flat\n";
705
706 return sampleprof_error::success;
707}
708
709std::error_code
710SampleProfileWriterBinary::writeContextIdx(const SampleContext &Context) {
711 assert(!Context.hasContext() && "cs profile is not supported");
712 return writeNameIdx(FName: Context.getFunction());
713}
714
715std::error_code SampleProfileWriterBinary::writeNameIdx(FunctionId FName) {
716 auto &NTable = getNameTable();
717 const auto &Ret = NTable.find(Key: FName);
718 if (Ret == NTable.end())
719 return sampleprof_error::truncated_name_table;
720 encodeULEB128(Value: Ret->second, OS&: *OutputStream);
721 return sampleprof_error::success;
722}
723
724void SampleProfileWriterBinary::addName(FunctionId FName) {
725 auto &NTable = getNameTable();
726 NTable.insert(KV: std::make_pair(x&: FName, y: 0));
727}
728
729void SampleProfileWriterBinary::addContext(const SampleContext &Context) {
730 addName(FName: Context.getFunction());
731}
732
733void SampleProfileWriterBinary::addNames(const FunctionSamples &S) {
734 // Add all the names in indirect call targets.
735 for (const auto &I : S.getBodySamples()) {
736 const SampleRecord &Sample = I.second;
737 for (const auto &J : Sample.getCallTargets())
738 addName(FName: J.first);
739 }
740
741 // Recursively add all the names for inlined callsites.
742 for (const auto &J : S.getCallsiteSamples())
743 for (const auto &FS : J.second) {
744 const FunctionSamples &CalleeSamples = FS.second;
745 addName(FName: CalleeSamples.getFunction());
746 addNames(S: CalleeSamples);
747 }
748
749 if (!WriteVTableProf)
750 return;
751 // Add all the vtable names to NameTable.
752 for (const auto &VTableAccessCountMap :
753 llvm::make_second_range(c: S.getCallsiteTypeCounts())) {
754 // Add type name to NameTable.
755 for (const auto Type : llvm::make_first_range(c: VTableAccessCountMap)) {
756 addName(FName: Type);
757 }
758 }
759}
760
761void SampleProfileWriterExtBinaryBase::addContext(
762 const SampleContext &Context) {
763 if (Context.hasContext()) {
764 for (auto &Callsite : Context.getContextFrames())
765 SampleProfileWriterBinary::addName(FName: Callsite.Func);
766 CSNameTable.insert(KV: std::make_pair(x: Context, y: 0));
767 } else {
768 SampleProfileWriterBinary::addName(FName: Context.getFunction());
769 }
770}
771
772std::error_code SampleProfileWriterBinary::writeNameTable() {
773 auto &OS = *OutputStream;
774
775 // Write out the name table.
776 encodeULEB128(Value: NameTable.size(), OS);
777 for (const auto *Entry : stabilizeTable(Table&: NameTable)) {
778 OS << Entry->first;
779 encodeULEB128(Value: 0, OS);
780 }
781 return sampleprof_error::success;
782}
783
784std::error_code
785SampleProfileWriterBinary::writeMagicIdent(SampleProfileFormat Format) {
786 auto &OS = *OutputStream;
787 // Write file magic identifier.
788 encodeULEB128(Value: SPMagic(Format), OS);
789 encodeULEB128(Value: FormatVersion, OS);
790 return sampleprof_error::success;
791}
792
793std::error_code
794SampleProfileWriterBinary::writeHeader(const SampleProfileMap &ProfileMap) {
795 // When calling write on a different profile map, existing names should be
796 // cleared.
797 NameTable.clear();
798
799 writeMagicIdent(Format);
800
801 computeSummary(ProfileMap);
802 if (auto EC = writeSummary())
803 return EC;
804
805 // Generate the name table for all the functions referenced in the profile.
806 for (const auto &I : ProfileMap) {
807 addContext(Context: I.second.getContext());
808 addNames(S: I.second);
809 }
810
811 writeNameTable();
812 return sampleprof_error::success;
813}
814
815void SampleProfileWriterExtBinaryBase::setToCompressAllSections() {
816 for (auto &Entry : SectionHdrLayout)
817 addSecFlag(Entry, Flag: SecCommonFlags::SecFlagCompress);
818}
819
820void SampleProfileWriterExtBinaryBase::setToCompressSection(SecType Type) {
821 addSectionFlag(Type, Flag: SecCommonFlags::SecFlagCompress);
822}
823
824void SampleProfileWriterExtBinaryBase::allocSecHdrTable() {
825 support::endian::Writer Writer(*OutputStream, llvm::endianness::little);
826
827 Writer.write(Val: static_cast<uint64_t>(SectionHdrLayout.size()));
828 SecHdrTableOffset = OutputStream->tell();
829 for (uint32_t i = 0; i < SectionHdrLayout.size(); i++) {
830 Writer.write(Val: static_cast<uint64_t>(-1));
831 Writer.write(Val: static_cast<uint64_t>(-1));
832 Writer.write(Val: static_cast<uint64_t>(-1));
833 Writer.write(Val: static_cast<uint64_t>(-1));
834 }
835}
836
837std::error_code SampleProfileWriterExtBinaryBase::writeSecHdrTable() {
838 assert(SecHdrTable.size() == SectionHdrLayout.size() &&
839 "SecHdrTable entries doesn't match SectionHdrLayout");
840 SmallVector<uint32_t, 16> IndexMap(SecHdrTable.size(), -1);
841 for (uint32_t TableIdx = 0; TableIdx < SecHdrTable.size(); TableIdx++) {
842 IndexMap[SecHdrTable[TableIdx].LayoutIndex] = TableIdx;
843 }
844
845 // Write the section header table in the order specified in
846 // SectionHdrLayout. SectionHdrLayout specifies the sections
847 // order in which profile reader expect to read, so the section
848 // header table should be written in the order in SectionHdrLayout.
849 // Note that the section order in SecHdrTable may be different
850 // from the order in SectionHdrLayout, for example, SecFuncOffsetTable
851 // needs to be computed after SecLBRProfile (the order in SecHdrTable),
852 // but it needs to be read before SecLBRProfile (the order in
853 // SectionHdrLayout). So we use IndexMap above to switch the order.
854 support::endian::SeekableWriter Writer(
855 static_cast<raw_pwrite_stream &>(*OutputStream),
856 llvm::endianness::little);
857 for (uint32_t LayoutIdx = 0; LayoutIdx < SectionHdrLayout.size();
858 LayoutIdx++) {
859 assert(IndexMap[LayoutIdx] < SecHdrTable.size() &&
860 "Incorrect LayoutIdx in SecHdrTable");
861 auto Entry = SecHdrTable[IndexMap[LayoutIdx]];
862 Writer.pwrite(Val: static_cast<uint64_t>(Entry.Type),
863 Offset: SecHdrTableOffset + 4 * LayoutIdx * sizeof(uint64_t));
864 Writer.pwrite(Val: static_cast<uint64_t>(Entry.Flags),
865 Offset: SecHdrTableOffset + (4 * LayoutIdx + 1) * sizeof(uint64_t));
866 Writer.pwrite(Val: static_cast<uint64_t>(Entry.Offset),
867 Offset: SecHdrTableOffset + (4 * LayoutIdx + 2) * sizeof(uint64_t));
868 Writer.pwrite(Val: static_cast<uint64_t>(Entry.Size),
869 Offset: SecHdrTableOffset + (4 * LayoutIdx + 3) * sizeof(uint64_t));
870 }
871
872 return sampleprof_error::success;
873}
874
875std::error_code SampleProfileWriterExtBinaryBase::writeHeader(
876 const SampleProfileMap &ProfileMap) {
877 auto &OS = *OutputStream;
878 FileStart = OS.tell();
879 writeMagicIdent(Format);
880
881 allocSecHdrTable();
882 return sampleprof_error::success;
883}
884
885std::error_code SampleProfileWriterBinary::writeCallsiteVTableProf(
886 const CallsiteTypeMap &CallsiteTypeMap, raw_ostream &OS) {
887 assert(WriteVTableProf &&
888 "writeCallsiteVTableProf should not be called if WriteVTableProf is "
889 "false");
890
891 encodeULEB128(Value: CallsiteTypeMap.size(), OS);
892 for (const auto &[Loc, TypeMap] : CallsiteTypeMap) {
893 Loc.serialize(OS);
894 if (std::error_code EC = serializeTypeMap(Map: TypeMap, NameTable: getNameTable(), OS))
895 return EC;
896 }
897
898 return sampleprof_error::success;
899}
900
901std::error_code SampleProfileWriterBinary::writeSummary() {
902 auto &OS = *OutputStream;
903 encodeULEB128(Value: Summary->getTotalCount(), OS);
904 encodeULEB128(Value: Summary->getMaxCount(), OS);
905 encodeULEB128(Value: Summary->getMaxFunctionCount(), OS);
906 encodeULEB128(Value: Summary->getNumCounts(), OS);
907 encodeULEB128(Value: Summary->getNumFunctions(), OS);
908 ArrayRef<ProfileSummaryEntry> Entries = Summary->getDetailedSummary();
909 encodeULEB128(Value: Entries.size(), OS);
910 for (auto Entry : Entries) {
911 encodeULEB128(Value: Entry.Cutoff, OS);
912 encodeULEB128(Value: Entry.MinCount, OS);
913 encodeULEB128(Value: Entry.NumCounts, OS);
914 }
915 return sampleprof_error::success;
916}
917std::error_code SampleProfileWriterBinary::writeBody(const FunctionSamples &S) {
918 auto &OS = *OutputStream;
919 if (std::error_code EC = writeContextIdx(Context: S.getContext()))
920 return EC;
921
922 encodeULEB128(Value: S.getTotalSamples(), OS);
923
924 // Emit all the body samples.
925 encodeULEB128(Value: S.getBodySamples().size(), OS);
926 for (const auto &I : S.getBodySamples()) {
927 LineLocation Loc = I.first;
928 const SampleRecord &Sample = I.second;
929 Loc.serialize(OS);
930 Sample.serialize(OS, NameTable: getNameTable());
931 }
932
933 // Recursively emit all the callsite samples.
934 uint64_t NumCallsites = 0;
935 for (const auto &J : S.getCallsiteSamples())
936 NumCallsites += J.second.size();
937 encodeULEB128(Value: NumCallsites, OS);
938 for (const auto &J : S.getCallsiteSamples())
939 for (const auto &FS : J.second) {
940 J.first.serialize(OS);
941 if (std::error_code EC = writeBody(S: FS.second))
942 return EC;
943 }
944
945 if (WriteVTableProf)
946 return writeCallsiteVTableProf(CallsiteTypeMap: S.getCallsiteTypeCounts(), OS);
947
948 return sampleprof_error::success;
949}
950
951/// Write samples of a top-level function to a binary file.
952///
953/// \returns true if the samples were written successfully, false otherwise.
954std::error_code
955SampleProfileWriterBinary::writeSample(const FunctionSamples &S) {
956 encodeULEB128(Value: S.getHeadSamples(), OS&: *OutputStream);
957 return writeBody(S);
958}
959
960/// Create a sample profile file writer based on the specified format.
961///
962/// \param Filename The file to create.
963///
964/// \param Format Encoding format for the profile file.
965///
966/// \returns an error code indicating the status of the created writer.
967ErrorOr<std::unique_ptr<SampleProfileWriter>>
968SampleProfileWriter::create(StringRef Filename, SampleProfileFormat Format) {
969 std::error_code EC;
970 std::unique_ptr<raw_ostream> OS;
971 if (Format == SPF_Binary || Format == SPF_Ext_Binary)
972 OS.reset(p: new raw_fd_ostream(Filename, EC, sys::fs::OF_None));
973 else
974 OS.reset(p: new raw_fd_ostream(Filename, EC, sys::fs::OF_TextWithCRLF));
975 if (EC)
976 return EC;
977
978 return create(OS, Format);
979}
980
981/// Create a sample profile stream writer based on the specified format.
982///
983/// \param OS The output stream to store the profile data to.
984///
985/// \param Format Encoding format for the profile file.
986///
987/// \returns an error code indicating the status of the created writer.
988ErrorOr<std::unique_ptr<SampleProfileWriter>>
989SampleProfileWriter::create(std::unique_ptr<raw_ostream> &OS,
990 SampleProfileFormat Format) {
991 std::error_code EC;
992 std::unique_ptr<SampleProfileWriter> Writer;
993
994 // Currently only Text and Extended Binary format are supported for CSSPGO.
995 if ((FunctionSamples::ProfileIsCS || FunctionSamples::ProfileIsProbeBased) &&
996 Format == SPF_Binary)
997 return sampleprof_error::unsupported_writing_format;
998
999 if (Format == SPF_Binary)
1000 Writer.reset(p: new SampleProfileWriterRawBinary(OS));
1001 else if (Format == SPF_Ext_Binary)
1002 Writer.reset(p: new SampleProfileWriterExtBinary(OS));
1003 else if (Format == SPF_Text)
1004 Writer.reset(p: new SampleProfileWriterText(OS));
1005 else if (Format == SPF_GCC)
1006 EC = sampleprof_error::unsupported_writing_format;
1007 else
1008 EC = sampleprof_error::unrecognized_format;
1009
1010 if (EC)
1011 return EC;
1012
1013 Writer->Format = Format;
1014 if (Format != SPF_Ext_Binary)
1015 Writer->setFormatVersion(DefaultVersion);
1016 else if (formatVersionIsSupported(Version: RequestedVersion))
1017 Writer->setFormatVersion(RequestedVersion);
1018 else
1019 return sampleprof_error::unsupported_version;
1020 return std::move(Writer);
1021}
1022
1023void SampleProfileWriter::computeSummary(const SampleProfileMap &ProfileMap) {
1024 SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs);
1025 Summary = Builder.computeSummaryForProfiles(Profiles: ProfileMap);
1026}
1027