1//===- SampleProfWriter.cpp - Write LLVM sample profile data --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the class that writes LLVM sample profiles. It
10// supports two file formats: text and binary. The textual representation
11// is useful for debugging and testing purposes. The binary representation
12// is more compact, resulting in smaller file sizes. However, they can
13// both be used interchangeably.
14//
15// See lib/ProfileData/SampleProfReader.cpp for documentation on each of the
16// supported formats.
17//
18//===----------------------------------------------------------------------===//
19
20#include "llvm/ProfileData/SampleProfWriter.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/ProfileData/ProfileCommon.h"
23#include "llvm/ProfileData/SampleProf.h"
24#include "llvm/Support/Compression.h"
25#include "llvm/Support/EndianStream.h"
26#include "llvm/Support/ErrorOr.h"
27#include "llvm/Support/FileSystem.h"
28#include "llvm/Support/LEB128.h"
29#include "llvm/Support/MD5.h"
30#include "llvm/Support/raw_ostream.h"
31#include <cmath>
32#include <cstdint>
33#include <memory>
34#include <set>
35#include <system_error>
36#include <utility>
37#include <vector>
38
39#define DEBUG_TYPE "llvm-profdata"
40
41using namespace llvm;
42using namespace sampleprof;
43
44// To begin with, make this option off by default.
45static cl::opt<bool> ExtBinaryWriteVTableTypeProf(
46 "extbinary-write-vtable-type-prof", cl::init(Val: false), cl::Hidden,
47 cl::desc("Write vtable type profile in ext-binary sample profile writer"));
48
49namespace llvm {
50namespace support {
51namespace endian {
52namespace {
53
54// Adapter class to llvm::support::endian::Writer for pwrite().
55struct SeekableWriter {
56 raw_pwrite_stream &OS;
57 endianness Endian;
58 SeekableWriter(raw_pwrite_stream &OS, endianness Endian)
59 : OS(OS), Endian(Endian) {}
60
61 template <typename ValueType>
62 void pwrite(ValueType Val, size_t Offset) {
63 std::string StringBuf;
64 raw_string_ostream SStream(StringBuf);
65 Writer(SStream, Endian).write(Val);
66 OS.pwrite(Ptr: StringBuf.data(), Size: StringBuf.size(), Offset);
67 }
68};
69
70} // namespace
71} // namespace endian
72} // namespace support
73} // namespace llvm
74
75DefaultFunctionPruningStrategy::DefaultFunctionPruningStrategy(
76 SampleProfileMap &ProfileMap, size_t OutputSizeLimit)
77 : FunctionPruningStrategy(ProfileMap, OutputSizeLimit) {
78 sortFuncProfiles(ProfileMap, SortedProfiles&: SortedFunctions);
79}
80
81void DefaultFunctionPruningStrategy::Erase(size_t CurrentOutputSize) {
82 double D = (double)OutputSizeLimit / CurrentOutputSize;
83 size_t NewSize = (size_t)round(x: ProfileMap.size() * D * D);
84 size_t NumToRemove = ProfileMap.size() - NewSize;
85 if (NumToRemove < 1)
86 NumToRemove = 1;
87
88 assert(NumToRemove <= SortedFunctions.size());
89 for (const NameFunctionSamples &E :
90 llvm::drop_begin(RangeOrContainer&: SortedFunctions, N: SortedFunctions.size() - NumToRemove))
91 ProfileMap.erase(Key: E.first);
92 SortedFunctions.resize(new_size: SortedFunctions.size() - NumToRemove);
93}
94
95std::error_code SampleProfileWriter::writeWithSizeLimitInternal(
96 SampleProfileMap &ProfileMap, size_t OutputSizeLimit,
97 FunctionPruningStrategy *Strategy) {
98 if (OutputSizeLimit == 0)
99 return write(ProfileMap);
100
101 size_t OriginalFunctionCount = ProfileMap.size();
102
103 std::unique_ptr<raw_ostream> OriginalOutputStream;
104 OutputStream.swap(u&: OriginalOutputStream);
105
106 size_t IterationCount = 0;
107 size_t TotalSize;
108
109 SmallVector<char> StringBuffer;
110 do {
111 StringBuffer.clear();
112 OutputStream.reset(p: new raw_svector_ostream(StringBuffer));
113 if (std::error_code EC = write(ProfileMap))
114 return EC;
115
116 TotalSize = StringBuffer.size();
117 // On Windows every "\n" is actually written as "\r\n" to disk but not to
118 // memory buffer, this difference should be added when considering the total
119 // output size.
120#ifdef _WIN32
121 if (Format == SPF_Text)
122 TotalSize += LineCount;
123#endif
124 if (TotalSize <= OutputSizeLimit)
125 break;
126
127 Strategy->Erase(CurrentOutputSize: TotalSize);
128 IterationCount++;
129 } while (ProfileMap.size() != 0);
130
131 if (ProfileMap.size() == 0)
132 return sampleprof_error::too_large;
133
134 OutputStream.swap(u&: OriginalOutputStream);
135 OutputStream->write(Ptr: StringBuffer.data(), Size: StringBuffer.size());
136 LLVM_DEBUG(dbgs() << "Profile originally has " << OriginalFunctionCount
137 << " functions, reduced to " << ProfileMap.size() << " in "
138 << IterationCount << " iterations\n");
139 // Silence warning on Release build.
140 (void)OriginalFunctionCount;
141 (void)IterationCount;
142 return sampleprof_error::success;
143}
144
145std::error_code
146SampleProfileWriter::writeFuncProfiles(const SampleProfileMap &ProfileMap) {
147 std::vector<NameFunctionSamples> V;
148 sortFuncProfiles(ProfileMap, SortedProfiles&: V);
149 for (const auto &I : V) {
150 if (std::error_code EC = writeSample(S: *I.second))
151 return EC;
152 }
153 return sampleprof_error::success;
154}
155
156std::error_code SampleProfileWriter::write(const SampleProfileMap &ProfileMap) {
157 if (std::error_code EC = writeHeader(ProfileMap))
158 return EC;
159
160 if (std::error_code EC = writeFuncProfiles(ProfileMap))
161 return EC;
162
163 return sampleprof_error::success;
164}
165
166/// Return the current position and prepare to use it as the start
167/// position of a section given the section type \p Type and its position
168/// \p LayoutIdx in SectionHdrLayout.
169uint64_t
170SampleProfileWriterExtBinaryBase::markSectionStart(SecType Type,
171 uint32_t LayoutIdx) {
172 uint64_t SectionStart = OutputStream->tell();
173 assert(LayoutIdx < SectionHdrLayout.size() && "LayoutIdx out of range");
174 const auto &Entry = SectionHdrLayout[LayoutIdx];
175 assert(Entry.Type == Type && "Unexpected section type");
176 // Use LocalBuf as a temporary output for writting data.
177 if (hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagCompress))
178 LocalBufStream.swap(u&: OutputStream);
179 return SectionStart;
180}
181
182std::error_code SampleProfileWriterExtBinaryBase::compressAndOutput() {
183 if (!llvm::compression::zlib::isAvailable())
184 return sampleprof_error::zlib_unavailable;
185 std::string &UncompressedStrings =
186 static_cast<raw_string_ostream *>(LocalBufStream.get())->str();
187 if (UncompressedStrings.size() == 0)
188 return sampleprof_error::success;
189 auto &OS = *OutputStream;
190 SmallVector<uint8_t, 128> CompressedStrings;
191 compression::zlib::compress(Input: arrayRefFromStringRef(Input: UncompressedStrings),
192 CompressedBuffer&: CompressedStrings,
193 Level: compression::zlib::BestSizeCompression);
194 encodeULEB128(Value: UncompressedStrings.size(), OS);
195 encodeULEB128(Value: CompressedStrings.size(), OS);
196 OS << toStringRef(Input: CompressedStrings);
197 UncompressedStrings.clear();
198 return sampleprof_error::success;
199}
200
201/// Add a new section into section header table given the section type
202/// \p Type, its position \p LayoutIdx in SectionHdrLayout and the
203/// location \p SectionStart where the section should be written to.
204std::error_code SampleProfileWriterExtBinaryBase::addNewSection(
205 SecType Type, uint32_t LayoutIdx, uint64_t SectionStart) {
206 assert(LayoutIdx < SectionHdrLayout.size() && "LayoutIdx out of range");
207 const auto &Entry = SectionHdrLayout[LayoutIdx];
208 assert(Entry.Type == Type && "Unexpected section type");
209 if (hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagCompress)) {
210 LocalBufStream.swap(u&: OutputStream);
211 if (std::error_code EC = compressAndOutput())
212 return EC;
213 }
214 SecHdrTable.push_back(x: {.Type: Type, .Flags: Entry.Flags, .Offset: SectionStart - FileStart,
215 .Size: OutputStream->tell() - SectionStart, .LayoutIndex: LayoutIdx});
216 return sampleprof_error::success;
217}
218
219std::error_code
220SampleProfileWriterExtBinaryBase::write(const SampleProfileMap &ProfileMap) {
221 // When calling write on a different profile map, existing states should be
222 // cleared.
223 NameTable.clear();
224 CSNameTable.clear();
225 SecHdrTable.clear();
226
227 if (std::error_code EC = writeHeader(ProfileMap))
228 return EC;
229
230 std::string LocalBuf;
231 LocalBufStream = std::make_unique<raw_string_ostream>(args&: LocalBuf);
232 if (std::error_code EC = writeSections(ProfileMap))
233 return EC;
234
235 if (std::error_code EC = writeSecHdrTable())
236 return EC;
237
238 return sampleprof_error::success;
239}
240
241std::error_code SampleProfileWriterExtBinaryBase::writeContextIdx(
242 const SampleContext &Context) {
243 if (Context.hasContext())
244 return writeCSNameIdx(Context);
245 else
246 return SampleProfileWriterBinary::writeNameIdx(FName: Context.getFunction());
247}
248
249std::error_code
250SampleProfileWriterExtBinaryBase::writeCSNameIdx(const SampleContext &Context) {
251 const auto &Ret = CSNameTable.find(Key: Context);
252 if (Ret == CSNameTable.end())
253 return sampleprof_error::truncated_name_table;
254 encodeULEB128(Value: Ret->second, OS&: *OutputStream);
255 return sampleprof_error::success;
256}
257
258std::error_code
259SampleProfileWriterExtBinaryBase::writeSample(const FunctionSamples &S) {
260 uint64_t Offset = OutputStream->tell();
261 auto &Context = S.getContext();
262 FuncOffsetTable[Context] = Offset - SecLBRProfileStart;
263 encodeULEB128(Value: S.getHeadSamples(), OS&: *OutputStream);
264 return writeBody(S);
265}
266
267std::error_code SampleProfileWriterExtBinaryBase::writeFuncOffsetTable() {
268 auto &OS = *OutputStream;
269
270 // Write out the table size.
271 encodeULEB128(Value: FuncOffsetTable.size(), OS);
272
273 // Write out FuncOffsetTable.
274 auto WriteItem = [&](const SampleContext &Context, uint64_t Offset) {
275 if (std::error_code EC = writeContextIdx(Context))
276 return EC;
277 encodeULEB128(Value: Offset, OS);
278 return (std::error_code)sampleprof_error::success;
279 };
280
281 if (FunctionSamples::ProfileIsCS) {
282 // Sort the contexts before writing them out. This is to help fast load all
283 // context profiles for a function as well as their callee contexts which
284 // can help profile-guided importing for ThinLTO.
285 std::map<SampleContext, uint64_t> OrderedFuncOffsetTable(
286 FuncOffsetTable.begin(), FuncOffsetTable.end());
287 for (const auto &Entry : OrderedFuncOffsetTable) {
288 if (std::error_code EC = WriteItem(Entry.first, Entry.second))
289 return EC;
290 }
291 addSectionFlag(Type: SecFuncOffsetTable, Flag: SecFuncOffsetFlags::SecFlagOrdered);
292 } else {
293 for (const auto &Entry : FuncOffsetTable) {
294 if (std::error_code EC = WriteItem(Entry.first, Entry.second))
295 return EC;
296 }
297 }
298
299 FuncOffsetTable.clear();
300 return sampleprof_error::success;
301}
302
303std::error_code SampleProfileWriterExtBinaryBase::writeFuncMetadata(
304 const FunctionSamples &FunctionProfile) {
305 auto &OS = *OutputStream;
306 if (std::error_code EC = writeContextIdx(Context: FunctionProfile.getContext()))
307 return EC;
308
309 if (FunctionSamples::ProfileIsProbeBased)
310 encodeULEB128(Value: FunctionProfile.getFunctionHash(), OS);
311 if (FunctionSamples::ProfileIsCS || FunctionSamples::ProfileIsPreInlined) {
312 encodeULEB128(Value: FunctionProfile.getContext().getAllAttributes(), OS);
313 }
314
315 if (!FunctionSamples::ProfileIsCS) {
316 // Recursively emit attributes for all callee samples.
317 uint64_t NumCallsites = 0;
318 for (const auto &J : FunctionProfile.getCallsiteSamples())
319 NumCallsites += J.second.size();
320 encodeULEB128(Value: NumCallsites, OS);
321 for (const auto &J : FunctionProfile.getCallsiteSamples()) {
322 for (const auto &FS : J.second) {
323 LineLocation Loc = J.first;
324 encodeULEB128(Value: Loc.LineOffset, OS);
325 encodeULEB128(Value: Loc.Discriminator, OS);
326 if (std::error_code EC = writeFuncMetadata(FunctionProfile: FS.second))
327 return EC;
328 }
329 }
330 }
331
332 return sampleprof_error::success;
333}
334
335std::error_code SampleProfileWriterExtBinaryBase::writeFuncMetadata(
336 const SampleProfileMap &Profiles) {
337 if (!FunctionSamples::ProfileIsProbeBased && !FunctionSamples::ProfileIsCS &&
338 !FunctionSamples::ProfileIsPreInlined)
339 return sampleprof_error::success;
340 for (const auto &Entry : Profiles) {
341 if (std::error_code EC = writeFuncMetadata(FunctionProfile: Entry.second))
342 return EC;
343 }
344 return sampleprof_error::success;
345}
346
347std::error_code SampleProfileWriterExtBinaryBase::writeNameTable() {
348 if (!UseMD5)
349 return SampleProfileWriterBinary::writeNameTable();
350
351 auto &OS = *OutputStream;
352 std::set<FunctionId> V;
353 stablizeNameTable(NameTable, V);
354
355 // Write out the MD5 name table. We wrote unencoded MD5 so reader can
356 // retrieve the name using the name index without having to read the
357 // whole name table.
358 encodeULEB128(Value: NameTable.size(), OS);
359 support::endian::Writer Writer(OS, llvm::endianness::little);
360 for (auto N : V)
361 Writer.write(Val: N.getHashCode());
362 return sampleprof_error::success;
363}
364
365std::error_code SampleProfileWriterExtBinaryBase::writeNameTableSection(
366 const SampleProfileMap &ProfileMap) {
367 for (const auto &I : ProfileMap) {
368 addContext(Context: I.second.getContext());
369 addNames(S: I.second);
370 }
371
372 // If NameTable contains ".__uniq." suffix, set SecFlagUniqSuffix flag
373 // so compiler won't strip the suffix during profile matching after
374 // seeing the flag in the profile.
375 // Original names are unavailable if using MD5, so this option has no use.
376 if (!UseMD5) {
377 for (const auto &I : NameTable) {
378 if (I.first.stringRef().contains(Other: FunctionSamples::UniqSuffix)) {
379 addSectionFlag(Type: SecNameTable, Flag: SecNameTableFlags::SecFlagUniqSuffix);
380 break;
381 }
382 }
383 }
384
385 if (auto EC = writeNameTable())
386 return EC;
387 return sampleprof_error::success;
388}
389
390std::error_code SampleProfileWriterExtBinaryBase::writeCSNameTableSection() {
391 // Sort the names to make CSNameTable deterministic.
392 std::set<SampleContext> OrderedContexts;
393 for (const auto &I : CSNameTable)
394 OrderedContexts.insert(x: I.first);
395 assert(OrderedContexts.size() == CSNameTable.size() &&
396 "Unmatched ordered and unordered contexts");
397 uint64_t I = 0;
398 for (auto &Context : OrderedContexts)
399 CSNameTable[Context] = I++;
400
401 auto &OS = *OutputStream;
402 encodeULEB128(Value: OrderedContexts.size(), OS);
403 support::endian::Writer Writer(OS, llvm::endianness::little);
404 for (auto Context : OrderedContexts) {
405 auto Frames = Context.getContextFrames();
406 encodeULEB128(Value: Frames.size(), OS);
407 for (auto &Callsite : Frames) {
408 if (std::error_code EC = writeNameIdx(FName: Callsite.Func))
409 return EC;
410 encodeULEB128(Value: Callsite.Location.LineOffset, OS);
411 encodeULEB128(Value: Callsite.Location.Discriminator, OS);
412 }
413 }
414
415 return sampleprof_error::success;
416}
417
418std::error_code
419SampleProfileWriterExtBinaryBase::writeProfileSymbolListSection() {
420 if (ProfSymList && ProfSymList->size() > 0)
421 if (std::error_code EC = ProfSymList->write(OS&: *OutputStream))
422 return EC;
423
424 return sampleprof_error::success;
425}
426
427std::error_code SampleProfileWriterExtBinaryBase::writeOneSection(
428 SecType Type, uint32_t LayoutIdx, const SampleProfileMap &ProfileMap) {
429 // The setting of SecFlagCompress should happen before markSectionStart.
430 if (Type == SecProfileSymbolList && ProfSymList && ProfSymList->toCompress())
431 setToCompressSection(SecProfileSymbolList);
432 if (Type == SecFuncMetadata && FunctionSamples::ProfileIsProbeBased)
433 addSectionFlag(Type: SecFuncMetadata, Flag: SecFuncMetadataFlags::SecFlagIsProbeBased);
434 if (Type == SecFuncMetadata &&
435 (FunctionSamples::ProfileIsCS || FunctionSamples::ProfileIsPreInlined))
436 addSectionFlag(Type: SecFuncMetadata, Flag: SecFuncMetadataFlags::SecFlagHasAttribute);
437 if (Type == SecProfSummary && FunctionSamples::ProfileIsCS)
438 addSectionFlag(Type: SecProfSummary, Flag: SecProfSummaryFlags::SecFlagFullContext);
439 if (Type == SecProfSummary && FunctionSamples::ProfileIsPreInlined)
440 addSectionFlag(Type: SecProfSummary, Flag: SecProfSummaryFlags::SecFlagIsPreInlined);
441 if (Type == SecProfSummary && FunctionSamples::ProfileIsFS)
442 addSectionFlag(Type: SecProfSummary, Flag: SecProfSummaryFlags::SecFlagFSDiscriminator);
443 if (Type == SecProfSummary && ExtBinaryWriteVTableTypeProf)
444 addSectionFlag(Type: SecProfSummary,
445 Flag: SecProfSummaryFlags::SecFlagHasVTableTypeProf);
446
447 uint64_t SectionStart = markSectionStart(Type, LayoutIdx);
448 switch (Type) {
449 case SecProfSummary:
450 computeSummary(ProfileMap);
451 if (auto EC = writeSummary())
452 return EC;
453 break;
454 case SecNameTable:
455 if (auto EC = writeNameTableSection(ProfileMap))
456 return EC;
457 break;
458 case SecCSNameTable:
459 if (auto EC = writeCSNameTableSection())
460 return EC;
461 break;
462 case SecLBRProfile:
463 SecLBRProfileStart = OutputStream->tell();
464 if (std::error_code EC = writeFuncProfiles(ProfileMap))
465 return EC;
466 break;
467 case SecFuncOffsetTable:
468 if (auto EC = writeFuncOffsetTable())
469 return EC;
470 break;
471 case SecFuncMetadata:
472 if (std::error_code EC = writeFuncMetadata(Profiles: ProfileMap))
473 return EC;
474 break;
475 case SecProfileSymbolList:
476 if (auto EC = writeProfileSymbolListSection())
477 return EC;
478 break;
479 default:
480 if (auto EC = writeCustomSection(Type))
481 return EC;
482 break;
483 }
484 if (std::error_code EC = addNewSection(Type, LayoutIdx, SectionStart))
485 return EC;
486 return sampleprof_error::success;
487}
488
489SampleProfileWriterExtBinary::SampleProfileWriterExtBinary(
490 std::unique_ptr<raw_ostream> &OS)
491 : SampleProfileWriterExtBinaryBase(OS) {
492 WriteVTableProf = ExtBinaryWriteVTableTypeProf;
493}
494
495std::error_code SampleProfileWriterExtBinary::writeDefaultLayout(
496 const SampleProfileMap &ProfileMap) {
497 // The const indices passed to writeOneSection below are specifying the
498 // positions of the sections in SectionHdrLayout. Look at
499 // initSectionHdrLayout to find out where each section is located in
500 // SectionHdrLayout.
501 if (auto EC = writeOneSection(Type: SecProfSummary, LayoutIdx: 0, ProfileMap))
502 return EC;
503 if (auto EC = writeOneSection(Type: SecNameTable, LayoutIdx: 1, ProfileMap))
504 return EC;
505 if (auto EC = writeOneSection(Type: SecCSNameTable, LayoutIdx: 2, ProfileMap))
506 return EC;
507 if (auto EC = writeOneSection(Type: SecLBRProfile, LayoutIdx: 4, ProfileMap))
508 return EC;
509 if (auto EC = writeOneSection(Type: SecProfileSymbolList, LayoutIdx: 5, ProfileMap))
510 return EC;
511 if (auto EC = writeOneSection(Type: SecFuncOffsetTable, LayoutIdx: 3, ProfileMap))
512 return EC;
513 if (auto EC = writeOneSection(Type: SecFuncMetadata, LayoutIdx: 6, ProfileMap))
514 return EC;
515 return sampleprof_error::success;
516}
517
518static void splitProfileMapToTwo(const SampleProfileMap &ProfileMap,
519 SampleProfileMap &ContextProfileMap,
520 SampleProfileMap &NoContextProfileMap) {
521 for (const auto &I : ProfileMap) {
522 if (I.second.getCallsiteSamples().size())
523 ContextProfileMap.insert(x: {I.first, I.second});
524 else
525 NoContextProfileMap.insert(x: {I.first, I.second});
526 }
527}
528
529std::error_code SampleProfileWriterExtBinary::writeCtxSplitLayout(
530 const SampleProfileMap &ProfileMap) {
531 SampleProfileMap ContextProfileMap, NoContextProfileMap;
532 splitProfileMapToTwo(ProfileMap, ContextProfileMap, NoContextProfileMap);
533
534 if (auto EC = writeOneSection(Type: SecProfSummary, LayoutIdx: 0, ProfileMap))
535 return EC;
536 if (auto EC = writeOneSection(Type: SecNameTable, LayoutIdx: 1, ProfileMap))
537 return EC;
538 if (auto EC = writeOneSection(Type: SecLBRProfile, LayoutIdx: 3, ProfileMap: ContextProfileMap))
539 return EC;
540 if (auto EC = writeOneSection(Type: SecFuncOffsetTable, LayoutIdx: 2, ProfileMap: ContextProfileMap))
541 return EC;
542 // Mark the section to have no context. Note section flag needs to be set
543 // before writing the section.
544 addSectionFlag(SectionIdx: 5, Flag: SecCommonFlags::SecFlagFlat);
545 if (auto EC = writeOneSection(Type: SecLBRProfile, LayoutIdx: 5, ProfileMap: NoContextProfileMap))
546 return EC;
547 // Mark the section to have no context. Note section flag needs to be set
548 // before writing the section.
549 addSectionFlag(SectionIdx: 4, Flag: SecCommonFlags::SecFlagFlat);
550 if (auto EC = writeOneSection(Type: SecFuncOffsetTable, LayoutIdx: 4, ProfileMap: NoContextProfileMap))
551 return EC;
552 if (auto EC = writeOneSection(Type: SecProfileSymbolList, LayoutIdx: 6, ProfileMap))
553 return EC;
554 if (auto EC = writeOneSection(Type: SecFuncMetadata, LayoutIdx: 7, ProfileMap))
555 return EC;
556
557 return sampleprof_error::success;
558}
559
560std::error_code SampleProfileWriterExtBinary::writeSections(
561 const SampleProfileMap &ProfileMap) {
562 std::error_code EC;
563 if (SecLayout == DefaultLayout)
564 EC = writeDefaultLayout(ProfileMap);
565 else if (SecLayout == CtxSplitLayout)
566 EC = writeCtxSplitLayout(ProfileMap);
567 else
568 llvm_unreachable("Unsupported layout");
569 return EC;
570}
571
572/// Write samples to a text file.
573///
574/// Note: it may be tempting to implement this in terms of
575/// FunctionSamples::print(). Please don't. The dump functionality is intended
576/// for debugging and has no specified form.
577///
578/// The format used here is more structured and deliberate because
579/// it needs to be parsed by the SampleProfileReaderText class.
580std::error_code SampleProfileWriterText::writeSample(const FunctionSamples &S) {
581 auto &OS = *OutputStream;
582 if (FunctionSamples::ProfileIsCS)
583 OS << "[" << S.getContext().toString() << "]:" << S.getTotalSamples();
584 else
585 OS << S.getFunction() << ":" << S.getTotalSamples();
586
587 if (Indent == 0)
588 OS << ":" << S.getHeadSamples();
589 OS << "\n";
590 LineCount++;
591
592 SampleSorter<LineLocation, SampleRecord> SortedSamples(S.getBodySamples());
593 for (const auto &I : SortedSamples.get()) {
594 LineLocation Loc = I->first;
595 const SampleRecord &Sample = I->second;
596 OS.indent(NumSpaces: Indent + 1);
597 Loc.print(OS);
598 OS << ": " << Sample.getSamples();
599
600 for (const auto &J : Sample.getSortedCallTargets())
601 OS << " " << J.first << ":" << J.second;
602 OS << "\n";
603 LineCount++;
604
605 if (const TypeCountMap *Map = S.findCallsiteTypeSamplesAt(Loc);
606 Map && !Map->empty()) {
607 OS.indent(NumSpaces: Indent + 1);
608 Loc.print(OS);
609 OS << ": ";
610 OS << kVTableProfPrefix;
611 for (const auto [TypeName, Count] : *Map) {
612 OS << TypeName << ":" << Count << " ";
613 }
614 OS << "\n";
615 LineCount++;
616 }
617 }
618
619 SampleSorter<LineLocation, FunctionSamplesMap> SortedCallsiteSamples(
620 S.getCallsiteSamples());
621 Indent += 1;
622 for (const auto *Element : SortedCallsiteSamples.get()) {
623 // Element is a pointer to a pair of LineLocation and FunctionSamplesMap.
624 const auto &[Loc, FunctionSamplesMap] = *Element;
625 for (const FunctionSamples &CalleeSamples :
626 make_second_range(c: FunctionSamplesMap)) {
627 OS.indent(NumSpaces: Indent);
628 Loc.print(OS);
629 OS << ": ";
630 if (std::error_code EC = writeSample(S: CalleeSamples))
631 return EC;
632 }
633
634 if (const TypeCountMap *Map = S.findCallsiteTypeSamplesAt(Loc);
635 Map && !Map->empty()) {
636 OS.indent(NumSpaces: Indent);
637 Loc.print(OS);
638 OS << ": ";
639 OS << kVTableProfPrefix;
640 for (const auto [TypeId, Count] : *Map) {
641 OS << TypeId << ":" << Count << " ";
642 }
643 OS << "\n";
644 LineCount++;
645 }
646 }
647
648 Indent -= 1;
649
650 if (FunctionSamples::ProfileIsProbeBased) {
651 OS.indent(NumSpaces: Indent + 1);
652 OS << "!CFGChecksum: " << S.getFunctionHash() << "\n";
653 LineCount++;
654 }
655
656 if (S.getContext().getAllAttributes()) {
657 OS.indent(NumSpaces: Indent + 1);
658 OS << "!Attributes: " << S.getContext().getAllAttributes() << "\n";
659 LineCount++;
660 }
661
662 if (Indent == 0 && MarkFlatProfiles && S.getCallsiteSamples().size() == 0)
663 OS << " !Flat\n";
664
665 return sampleprof_error::success;
666}
667
668std::error_code
669SampleProfileWriterBinary::writeContextIdx(const SampleContext &Context) {
670 assert(!Context.hasContext() && "cs profile is not supported");
671 return writeNameIdx(FName: Context.getFunction());
672}
673
674std::error_code SampleProfileWriterBinary::writeNameIdx(FunctionId FName) {
675 auto &NTable = getNameTable();
676 const auto &Ret = NTable.find(Key: FName);
677 if (Ret == NTable.end())
678 return sampleprof_error::truncated_name_table;
679 encodeULEB128(Value: Ret->second, OS&: *OutputStream);
680 return sampleprof_error::success;
681}
682
683void SampleProfileWriterBinary::addName(FunctionId FName) {
684 auto &NTable = getNameTable();
685 NTable.insert(KV: std::make_pair(x&: FName, y: 0));
686}
687
688void SampleProfileWriterBinary::addContext(const SampleContext &Context) {
689 addName(FName: Context.getFunction());
690}
691
692void SampleProfileWriterBinary::addNames(const FunctionSamples &S) {
693 // Add all the names in indirect call targets.
694 for (const auto &I : S.getBodySamples()) {
695 const SampleRecord &Sample = I.second;
696 for (const auto &J : Sample.getCallTargets())
697 addName(FName: J.first);
698 }
699
700 // Recursively add all the names for inlined callsites.
701 for (const auto &J : S.getCallsiteSamples())
702 for (const auto &FS : J.second) {
703 const FunctionSamples &CalleeSamples = FS.second;
704 addName(FName: CalleeSamples.getFunction());
705 addNames(S: CalleeSamples);
706 }
707
708 if (!WriteVTableProf)
709 return;
710 // Add all the vtable names to NameTable.
711 for (const auto &VTableAccessCountMap :
712 llvm::make_second_range(c: S.getCallsiteTypeCounts())) {
713 // Add type name to NameTable.
714 for (const auto Type : llvm::make_first_range(c: VTableAccessCountMap)) {
715 addName(FName: Type);
716 }
717 }
718}
719
720void SampleProfileWriterExtBinaryBase::addContext(
721 const SampleContext &Context) {
722 if (Context.hasContext()) {
723 for (auto &Callsite : Context.getContextFrames())
724 SampleProfileWriterBinary::addName(FName: Callsite.Func);
725 CSNameTable.insert(KV: std::make_pair(x: Context, y: 0));
726 } else {
727 SampleProfileWriterBinary::addName(FName: Context.getFunction());
728 }
729}
730
731void SampleProfileWriterBinary::stablizeNameTable(
732 MapVector<FunctionId, uint32_t> &NameTable, std::set<FunctionId> &V) {
733 // Sort the names to make NameTable deterministic.
734 for (const auto &I : NameTable)
735 V.insert(x: I.first);
736 int i = 0;
737 for (const FunctionId &N : V)
738 NameTable[N] = i++;
739}
740
741std::error_code SampleProfileWriterBinary::writeNameTable() {
742 auto &OS = *OutputStream;
743 std::set<FunctionId> V;
744 stablizeNameTable(NameTable, V);
745
746 // Write out the name table.
747 encodeULEB128(Value: NameTable.size(), OS);
748 for (auto N : V) {
749 OS << N;
750 encodeULEB128(Value: 0, OS);
751 }
752 return sampleprof_error::success;
753}
754
755std::error_code
756SampleProfileWriterBinary::writeMagicIdent(SampleProfileFormat Format) {
757 auto &OS = *OutputStream;
758 // Write file magic identifier.
759 encodeULEB128(Value: SPMagic(Format), OS);
760 encodeULEB128(Value: SPVersion(), OS);
761 return sampleprof_error::success;
762}
763
764std::error_code
765SampleProfileWriterBinary::writeHeader(const SampleProfileMap &ProfileMap) {
766 // When calling write on a different profile map, existing names should be
767 // cleared.
768 NameTable.clear();
769
770 writeMagicIdent(Format);
771
772 computeSummary(ProfileMap);
773 if (auto EC = writeSummary())
774 return EC;
775
776 // Generate the name table for all the functions referenced in the profile.
777 for (const auto &I : ProfileMap) {
778 addContext(Context: I.second.getContext());
779 addNames(S: I.second);
780 }
781
782 writeNameTable();
783 return sampleprof_error::success;
784}
785
786void SampleProfileWriterExtBinaryBase::setToCompressAllSections() {
787 for (auto &Entry : SectionHdrLayout)
788 addSecFlag(Entry, Flag: SecCommonFlags::SecFlagCompress);
789}
790
791void SampleProfileWriterExtBinaryBase::setToCompressSection(SecType Type) {
792 addSectionFlag(Type, Flag: SecCommonFlags::SecFlagCompress);
793}
794
795void SampleProfileWriterExtBinaryBase::allocSecHdrTable() {
796 support::endian::Writer Writer(*OutputStream, llvm::endianness::little);
797
798 Writer.write(Val: static_cast<uint64_t>(SectionHdrLayout.size()));
799 SecHdrTableOffset = OutputStream->tell();
800 for (uint32_t i = 0; i < SectionHdrLayout.size(); i++) {
801 Writer.write(Val: static_cast<uint64_t>(-1));
802 Writer.write(Val: static_cast<uint64_t>(-1));
803 Writer.write(Val: static_cast<uint64_t>(-1));
804 Writer.write(Val: static_cast<uint64_t>(-1));
805 }
806}
807
808std::error_code SampleProfileWriterExtBinaryBase::writeSecHdrTable() {
809 assert(SecHdrTable.size() == SectionHdrLayout.size() &&
810 "SecHdrTable entries doesn't match SectionHdrLayout");
811 SmallVector<uint32_t, 16> IndexMap(SecHdrTable.size(), -1);
812 for (uint32_t TableIdx = 0; TableIdx < SecHdrTable.size(); TableIdx++) {
813 IndexMap[SecHdrTable[TableIdx].LayoutIndex] = TableIdx;
814 }
815
816 // Write the section header table in the order specified in
817 // SectionHdrLayout. SectionHdrLayout specifies the sections
818 // order in which profile reader expect to read, so the section
819 // header table should be written in the order in SectionHdrLayout.
820 // Note that the section order in SecHdrTable may be different
821 // from the order in SectionHdrLayout, for example, SecFuncOffsetTable
822 // needs to be computed after SecLBRProfile (the order in SecHdrTable),
823 // but it needs to be read before SecLBRProfile (the order in
824 // SectionHdrLayout). So we use IndexMap above to switch the order.
825 support::endian::SeekableWriter Writer(
826 static_cast<raw_pwrite_stream &>(*OutputStream),
827 llvm::endianness::little);
828 for (uint32_t LayoutIdx = 0; LayoutIdx < SectionHdrLayout.size();
829 LayoutIdx++) {
830 assert(IndexMap[LayoutIdx] < SecHdrTable.size() &&
831 "Incorrect LayoutIdx in SecHdrTable");
832 auto Entry = SecHdrTable[IndexMap[LayoutIdx]];
833 Writer.pwrite(Val: static_cast<uint64_t>(Entry.Type),
834 Offset: SecHdrTableOffset + 4 * LayoutIdx * sizeof(uint64_t));
835 Writer.pwrite(Val: static_cast<uint64_t>(Entry.Flags),
836 Offset: SecHdrTableOffset + (4 * LayoutIdx + 1) * sizeof(uint64_t));
837 Writer.pwrite(Val: static_cast<uint64_t>(Entry.Offset),
838 Offset: SecHdrTableOffset + (4 * LayoutIdx + 2) * sizeof(uint64_t));
839 Writer.pwrite(Val: static_cast<uint64_t>(Entry.Size),
840 Offset: SecHdrTableOffset + (4 * LayoutIdx + 3) * sizeof(uint64_t));
841 }
842
843 return sampleprof_error::success;
844}
845
846std::error_code SampleProfileWriterExtBinaryBase::writeHeader(
847 const SampleProfileMap &ProfileMap) {
848 auto &OS = *OutputStream;
849 FileStart = OS.tell();
850 writeMagicIdent(Format);
851
852 allocSecHdrTable();
853 return sampleprof_error::success;
854}
855
856std::error_code SampleProfileWriterBinary::writeCallsiteVTableProf(
857 const CallsiteTypeMap &CallsiteTypeMap, raw_ostream &OS) {
858 assert(WriteVTableProf &&
859 "writeCallsiteVTableProf should not be called if WriteVTableProf is "
860 "false");
861
862 encodeULEB128(Value: CallsiteTypeMap.size(), OS);
863 for (const auto &[Loc, TypeMap] : CallsiteTypeMap) {
864 Loc.serialize(OS);
865 if (std::error_code EC = serializeTypeMap(Map: TypeMap, NameTable: getNameTable(), OS))
866 return EC;
867 }
868
869 return sampleprof_error::success;
870}
871
872std::error_code SampleProfileWriterBinary::writeSummary() {
873 auto &OS = *OutputStream;
874 encodeULEB128(Value: Summary->getTotalCount(), OS);
875 encodeULEB128(Value: Summary->getMaxCount(), OS);
876 encodeULEB128(Value: Summary->getMaxFunctionCount(), OS);
877 encodeULEB128(Value: Summary->getNumCounts(), OS);
878 encodeULEB128(Value: Summary->getNumFunctions(), OS);
879 ArrayRef<ProfileSummaryEntry> Entries = Summary->getDetailedSummary();
880 encodeULEB128(Value: Entries.size(), OS);
881 for (auto Entry : Entries) {
882 encodeULEB128(Value: Entry.Cutoff, OS);
883 encodeULEB128(Value: Entry.MinCount, OS);
884 encodeULEB128(Value: Entry.NumCounts, OS);
885 }
886 return sampleprof_error::success;
887}
888std::error_code SampleProfileWriterBinary::writeBody(const FunctionSamples &S) {
889 auto &OS = *OutputStream;
890 if (std::error_code EC = writeContextIdx(Context: S.getContext()))
891 return EC;
892
893 encodeULEB128(Value: S.getTotalSamples(), OS);
894
895 // Emit all the body samples.
896 encodeULEB128(Value: S.getBodySamples().size(), OS);
897 for (const auto &I : S.getBodySamples()) {
898 LineLocation Loc = I.first;
899 const SampleRecord &Sample = I.second;
900 Loc.serialize(OS);
901 Sample.serialize(OS, NameTable: getNameTable());
902 }
903
904 // Recursively emit all the callsite samples.
905 uint64_t NumCallsites = 0;
906 for (const auto &J : S.getCallsiteSamples())
907 NumCallsites += J.second.size();
908 encodeULEB128(Value: NumCallsites, OS);
909 for (const auto &J : S.getCallsiteSamples())
910 for (const auto &FS : J.second) {
911 J.first.serialize(OS);
912 if (std::error_code EC = writeBody(S: FS.second))
913 return EC;
914 }
915
916 if (WriteVTableProf)
917 return writeCallsiteVTableProf(CallsiteTypeMap: S.getCallsiteTypeCounts(), OS);
918
919 return sampleprof_error::success;
920}
921
922/// Write samples of a top-level function to a binary file.
923///
924/// \returns true if the samples were written successfully, false otherwise.
925std::error_code
926SampleProfileWriterBinary::writeSample(const FunctionSamples &S) {
927 encodeULEB128(Value: S.getHeadSamples(), OS&: *OutputStream);
928 return writeBody(S);
929}
930
931/// Create a sample profile file writer based on the specified format.
932///
933/// \param Filename The file to create.
934///
935/// \param Format Encoding format for the profile file.
936///
937/// \returns an error code indicating the status of the created writer.
938ErrorOr<std::unique_ptr<SampleProfileWriter>>
939SampleProfileWriter::create(StringRef Filename, SampleProfileFormat Format) {
940 std::error_code EC;
941 std::unique_ptr<raw_ostream> OS;
942 if (Format == SPF_Binary || Format == SPF_Ext_Binary)
943 OS.reset(p: new raw_fd_ostream(Filename, EC, sys::fs::OF_None));
944 else
945 OS.reset(p: new raw_fd_ostream(Filename, EC, sys::fs::OF_TextWithCRLF));
946 if (EC)
947 return EC;
948
949 return create(OS, Format);
950}
951
952/// Create a sample profile stream writer based on the specified format.
953///
954/// \param OS The output stream to store the profile data to.
955///
956/// \param Format Encoding format for the profile file.
957///
958/// \returns an error code indicating the status of the created writer.
959ErrorOr<std::unique_ptr<SampleProfileWriter>>
960SampleProfileWriter::create(std::unique_ptr<raw_ostream> &OS,
961 SampleProfileFormat Format) {
962 std::error_code EC;
963 std::unique_ptr<SampleProfileWriter> Writer;
964
965 // Currently only Text and Extended Binary format are supported for CSSPGO.
966 if ((FunctionSamples::ProfileIsCS || FunctionSamples::ProfileIsProbeBased) &&
967 Format == SPF_Binary)
968 return sampleprof_error::unsupported_writing_format;
969
970 if (Format == SPF_Binary)
971 Writer.reset(p: new SampleProfileWriterRawBinary(OS));
972 else if (Format == SPF_Ext_Binary)
973 Writer.reset(p: new SampleProfileWriterExtBinary(OS));
974 else if (Format == SPF_Text)
975 Writer.reset(p: new SampleProfileWriterText(OS));
976 else if (Format == SPF_GCC)
977 EC = sampleprof_error::unsupported_writing_format;
978 else
979 EC = sampleprof_error::unrecognized_format;
980
981 if (EC)
982 return EC;
983
984 Writer->Format = Format;
985 return std::move(Writer);
986}
987
988void SampleProfileWriter::computeSummary(const SampleProfileMap &ProfileMap) {
989 SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs);
990 Summary = Builder.computeSummaryForProfiles(Profiles: ProfileMap);
991}
992