1//===- SampleProfWriter.cpp - Write LLVM sample profile data --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the class that writes LLVM sample profiles. It
10// supports two file formats: text and binary. The textual representation
11// is useful for debugging and testing purposes. The binary representation
12// is more compact, resulting in smaller file sizes. However, they can
13// both be used interchangeably.
14//
15// See lib/ProfileData/SampleProfReader.cpp for documentation on each of the
16// supported formats.
17//
18//===----------------------------------------------------------------------===//
19
20#include "llvm/ProfileData/SampleProfWriter.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/ProfileData/ProfileCommon.h"
23#include "llvm/ProfileData/SampleProf.h"
24#include "llvm/Support/Compression.h"
25#include "llvm/Support/EndianStream.h"
26#include "llvm/Support/ErrorOr.h"
27#include "llvm/Support/FileSystem.h"
28#include "llvm/Support/LEB128.h"
29#include "llvm/Support/MD5.h"
30#include "llvm/Support/raw_ostream.h"
31#include <cmath>
32#include <cstdint>
33#include <memory>
34#include <set>
35#include <system_error>
36#include <utility>
37#include <vector>
38
39#define DEBUG_TYPE "llvm-profdata"
40
41using namespace llvm;
42using namespace sampleprof;
43
44namespace llvm {
45namespace support {
46namespace endian {
47namespace {
48
49// Adapter class to llvm::support::endian::Writer for pwrite().
50struct SeekableWriter {
51 raw_pwrite_stream &OS;
52 endianness Endian;
53 SeekableWriter(raw_pwrite_stream &OS, endianness Endian)
54 : OS(OS), Endian(Endian) {}
55
56 template <typename ValueType>
57 void pwrite(ValueType Val, size_t Offset) {
58 std::string StringBuf;
59 raw_string_ostream SStream(StringBuf);
60 Writer(SStream, Endian).write(Val);
61 OS.pwrite(Ptr: StringBuf.data(), Size: StringBuf.size(), Offset);
62 }
63};
64
65} // namespace
66} // namespace endian
67} // namespace support
68} // namespace llvm
69
70DefaultFunctionPruningStrategy::DefaultFunctionPruningStrategy(
71 SampleProfileMap &ProfileMap, size_t OutputSizeLimit)
72 : FunctionPruningStrategy(ProfileMap, OutputSizeLimit) {
73 sortFuncProfiles(ProfileMap, SortedProfiles&: SortedFunctions);
74}
75
76void DefaultFunctionPruningStrategy::Erase(size_t CurrentOutputSize) {
77 double D = (double)OutputSizeLimit / CurrentOutputSize;
78 size_t NewSize = (size_t)round(x: ProfileMap.size() * D * D);
79 size_t NumToRemove = ProfileMap.size() - NewSize;
80 if (NumToRemove < 1)
81 NumToRemove = 1;
82
83 assert(NumToRemove <= SortedFunctions.size());
84 for (const NameFunctionSamples &E :
85 llvm::drop_begin(RangeOrContainer&: SortedFunctions, N: SortedFunctions.size() - NumToRemove))
86 ProfileMap.erase(Key: E.first);
87 SortedFunctions.resize(new_size: SortedFunctions.size() - NumToRemove);
88}
89
90std::error_code SampleProfileWriter::writeWithSizeLimitInternal(
91 SampleProfileMap &ProfileMap, size_t OutputSizeLimit,
92 FunctionPruningStrategy *Strategy) {
93 if (OutputSizeLimit == 0)
94 return write(ProfileMap);
95
96 size_t OriginalFunctionCount = ProfileMap.size();
97
98 std::unique_ptr<raw_ostream> OriginalOutputStream;
99 OutputStream.swap(u&: OriginalOutputStream);
100
101 size_t IterationCount = 0;
102 size_t TotalSize;
103
104 SmallVector<char> StringBuffer;
105 do {
106 StringBuffer.clear();
107 OutputStream.reset(p: new raw_svector_ostream(StringBuffer));
108 if (std::error_code EC = write(ProfileMap))
109 return EC;
110
111 TotalSize = StringBuffer.size();
112 // On Windows every "\n" is actually written as "\r\n" to disk but not to
113 // memory buffer, this difference should be added when considering the total
114 // output size.
115#ifdef _WIN32
116 if (Format == SPF_Text)
117 TotalSize += LineCount;
118#endif
119 if (TotalSize <= OutputSizeLimit)
120 break;
121
122 Strategy->Erase(CurrentOutputSize: TotalSize);
123 IterationCount++;
124 } while (ProfileMap.size() != 0);
125
126 if (ProfileMap.size() == 0)
127 return sampleprof_error::too_large;
128
129 OutputStream.swap(u&: OriginalOutputStream);
130 OutputStream->write(Ptr: StringBuffer.data(), Size: StringBuffer.size());
131 LLVM_DEBUG(dbgs() << "Profile originally has " << OriginalFunctionCount
132 << " functions, reduced to " << ProfileMap.size() << " in "
133 << IterationCount << " iterations\n");
134 // Silence warning on Release build.
135 (void)OriginalFunctionCount;
136 (void)IterationCount;
137 return sampleprof_error::success;
138}
139
140std::error_code
141SampleProfileWriter::writeFuncProfiles(const SampleProfileMap &ProfileMap) {
142 std::vector<NameFunctionSamples> V;
143 sortFuncProfiles(ProfileMap, SortedProfiles&: V);
144 for (const auto &I : V) {
145 if (std::error_code EC = writeSample(S: *I.second))
146 return EC;
147 }
148 return sampleprof_error::success;
149}
150
151std::error_code SampleProfileWriter::write(const SampleProfileMap &ProfileMap) {
152 if (std::error_code EC = writeHeader(ProfileMap))
153 return EC;
154
155 if (std::error_code EC = writeFuncProfiles(ProfileMap))
156 return EC;
157
158 return sampleprof_error::success;
159}
160
161/// Return the current position and prepare to use it as the start
162/// position of a section given the section type \p Type and its position
163/// \p LayoutIdx in SectionHdrLayout.
164uint64_t
165SampleProfileWriterExtBinaryBase::markSectionStart(SecType Type,
166 uint32_t LayoutIdx) {
167 uint64_t SectionStart = OutputStream->tell();
168 assert(LayoutIdx < SectionHdrLayout.size() && "LayoutIdx out of range");
169 const auto &Entry = SectionHdrLayout[LayoutIdx];
170 assert(Entry.Type == Type && "Unexpected section type");
171 // Use LocalBuf as a temporary output for writting data.
172 if (hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagCompress))
173 LocalBufStream.swap(u&: OutputStream);
174 return SectionStart;
175}
176
177std::error_code SampleProfileWriterExtBinaryBase::compressAndOutput() {
178 if (!llvm::compression::zlib::isAvailable())
179 return sampleprof_error::zlib_unavailable;
180 std::string &UncompressedStrings =
181 static_cast<raw_string_ostream *>(LocalBufStream.get())->str();
182 if (UncompressedStrings.size() == 0)
183 return sampleprof_error::success;
184 auto &OS = *OutputStream;
185 SmallVector<uint8_t, 128> CompressedStrings;
186 compression::zlib::compress(Input: arrayRefFromStringRef(Input: UncompressedStrings),
187 CompressedBuffer&: CompressedStrings,
188 Level: compression::zlib::BestSizeCompression);
189 encodeULEB128(Value: UncompressedStrings.size(), OS);
190 encodeULEB128(Value: CompressedStrings.size(), OS);
191 OS << toStringRef(Input: CompressedStrings);
192 UncompressedStrings.clear();
193 return sampleprof_error::success;
194}
195
196/// Add a new section into section header table given the section type
197/// \p Type, its position \p LayoutIdx in SectionHdrLayout and the
198/// location \p SectionStart where the section should be written to.
199std::error_code SampleProfileWriterExtBinaryBase::addNewSection(
200 SecType Type, uint32_t LayoutIdx, uint64_t SectionStart) {
201 assert(LayoutIdx < SectionHdrLayout.size() && "LayoutIdx out of range");
202 const auto &Entry = SectionHdrLayout[LayoutIdx];
203 assert(Entry.Type == Type && "Unexpected section type");
204 if (hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagCompress)) {
205 LocalBufStream.swap(u&: OutputStream);
206 if (std::error_code EC = compressAndOutput())
207 return EC;
208 }
209 SecHdrTable.push_back(x: {.Type: Type, .Flags: Entry.Flags, .Offset: SectionStart - FileStart,
210 .Size: OutputStream->tell() - SectionStart, .LayoutIndex: LayoutIdx});
211 return sampleprof_error::success;
212}
213
214std::error_code
215SampleProfileWriterExtBinaryBase::write(const SampleProfileMap &ProfileMap) {
216 // When calling write on a different profile map, existing states should be
217 // cleared.
218 NameTable.clear();
219 CSNameTable.clear();
220 SecHdrTable.clear();
221
222 if (std::error_code EC = writeHeader(ProfileMap))
223 return EC;
224
225 std::string LocalBuf;
226 LocalBufStream = std::make_unique<raw_string_ostream>(args&: LocalBuf);
227 if (std::error_code EC = writeSections(ProfileMap))
228 return EC;
229
230 if (std::error_code EC = writeSecHdrTable())
231 return EC;
232
233 return sampleprof_error::success;
234}
235
236std::error_code SampleProfileWriterExtBinaryBase::writeContextIdx(
237 const SampleContext &Context) {
238 if (Context.hasContext())
239 return writeCSNameIdx(Context);
240 else
241 return SampleProfileWriterBinary::writeNameIdx(FName: Context.getFunction());
242}
243
244std::error_code
245SampleProfileWriterExtBinaryBase::writeCSNameIdx(const SampleContext &Context) {
246 const auto &Ret = CSNameTable.find(Key: Context);
247 if (Ret == CSNameTable.end())
248 return sampleprof_error::truncated_name_table;
249 encodeULEB128(Value: Ret->second, OS&: *OutputStream);
250 return sampleprof_error::success;
251}
252
253std::error_code
254SampleProfileWriterExtBinaryBase::writeSample(const FunctionSamples &S) {
255 uint64_t Offset = OutputStream->tell();
256 auto &Context = S.getContext();
257 FuncOffsetTable[Context] = Offset - SecLBRProfileStart;
258 encodeULEB128(Value: S.getHeadSamples(), OS&: *OutputStream);
259 return writeBody(S);
260}
261
262std::error_code SampleProfileWriterExtBinaryBase::writeFuncOffsetTable() {
263 auto &OS = *OutputStream;
264
265 // Write out the table size.
266 encodeULEB128(Value: FuncOffsetTable.size(), OS);
267
268 // Write out FuncOffsetTable.
269 auto WriteItem = [&](const SampleContext &Context, uint64_t Offset) {
270 if (std::error_code EC = writeContextIdx(Context))
271 return EC;
272 encodeULEB128(Value: Offset, OS);
273 return (std::error_code)sampleprof_error::success;
274 };
275
276 if (FunctionSamples::ProfileIsCS) {
277 // Sort the contexts before writing them out. This is to help fast load all
278 // context profiles for a function as well as their callee contexts which
279 // can help profile-guided importing for ThinLTO.
280 std::map<SampleContext, uint64_t> OrderedFuncOffsetTable(
281 FuncOffsetTable.begin(), FuncOffsetTable.end());
282 for (const auto &Entry : OrderedFuncOffsetTable) {
283 if (std::error_code EC = WriteItem(Entry.first, Entry.second))
284 return EC;
285 }
286 addSectionFlag(Type: SecFuncOffsetTable, Flag: SecFuncOffsetFlags::SecFlagOrdered);
287 } else {
288 for (const auto &Entry : FuncOffsetTable) {
289 if (std::error_code EC = WriteItem(Entry.first, Entry.second))
290 return EC;
291 }
292 }
293
294 FuncOffsetTable.clear();
295 return sampleprof_error::success;
296}
297
298std::error_code SampleProfileWriterExtBinaryBase::writeFuncMetadata(
299 const FunctionSamples &FunctionProfile) {
300 auto &OS = *OutputStream;
301 if (std::error_code EC = writeContextIdx(Context: FunctionProfile.getContext()))
302 return EC;
303
304 if (FunctionSamples::ProfileIsProbeBased)
305 encodeULEB128(Value: FunctionProfile.getFunctionHash(), OS);
306 if (FunctionSamples::ProfileIsCS || FunctionSamples::ProfileIsPreInlined) {
307 encodeULEB128(Value: FunctionProfile.getContext().getAllAttributes(), OS);
308 }
309
310 if (!FunctionSamples::ProfileIsCS) {
311 // Recursively emit attributes for all callee samples.
312 uint64_t NumCallsites = 0;
313 for (const auto &J : FunctionProfile.getCallsiteSamples())
314 NumCallsites += J.second.size();
315 encodeULEB128(Value: NumCallsites, OS);
316 for (const auto &J : FunctionProfile.getCallsiteSamples()) {
317 for (const auto &FS : J.second) {
318 LineLocation Loc = J.first;
319 encodeULEB128(Value: Loc.LineOffset, OS);
320 encodeULEB128(Value: Loc.Discriminator, OS);
321 if (std::error_code EC = writeFuncMetadata(FunctionProfile: FS.second))
322 return EC;
323 }
324 }
325 }
326
327 return sampleprof_error::success;
328}
329
330std::error_code SampleProfileWriterExtBinaryBase::writeFuncMetadata(
331 const SampleProfileMap &Profiles) {
332 if (!FunctionSamples::ProfileIsProbeBased && !FunctionSamples::ProfileIsCS &&
333 !FunctionSamples::ProfileIsPreInlined)
334 return sampleprof_error::success;
335 for (const auto &Entry : Profiles) {
336 if (std::error_code EC = writeFuncMetadata(FunctionProfile: Entry.second))
337 return EC;
338 }
339 return sampleprof_error::success;
340}
341
342std::error_code SampleProfileWriterExtBinaryBase::writeNameTable() {
343 if (!UseMD5)
344 return SampleProfileWriterBinary::writeNameTable();
345
346 auto &OS = *OutputStream;
347 std::set<FunctionId> V;
348 stablizeNameTable(NameTable, V);
349
350 // Write out the MD5 name table. We wrote unencoded MD5 so reader can
351 // retrieve the name using the name index without having to read the
352 // whole name table.
353 encodeULEB128(Value: NameTable.size(), OS);
354 support::endian::Writer Writer(OS, llvm::endianness::little);
355 for (auto N : V)
356 Writer.write(Val: N.getHashCode());
357 return sampleprof_error::success;
358}
359
360std::error_code SampleProfileWriterExtBinaryBase::writeNameTableSection(
361 const SampleProfileMap &ProfileMap) {
362 for (const auto &I : ProfileMap) {
363 addContext(Context: I.second.getContext());
364 addNames(S: I.second);
365 }
366
367 // If NameTable contains ".__uniq." suffix, set SecFlagUniqSuffix flag
368 // so compiler won't strip the suffix during profile matching after
369 // seeing the flag in the profile.
370 // Original names are unavailable if using MD5, so this option has no use.
371 if (!UseMD5) {
372 for (const auto &I : NameTable) {
373 if (I.first.stringRef().contains(Other: FunctionSamples::UniqSuffix)) {
374 addSectionFlag(Type: SecNameTable, Flag: SecNameTableFlags::SecFlagUniqSuffix);
375 break;
376 }
377 }
378 }
379
380 if (auto EC = writeNameTable())
381 return EC;
382 return sampleprof_error::success;
383}
384
385std::error_code SampleProfileWriterExtBinaryBase::writeCSNameTableSection() {
386 // Sort the names to make CSNameTable deterministic.
387 std::set<SampleContext> OrderedContexts;
388 for (const auto &I : CSNameTable)
389 OrderedContexts.insert(x: I.first);
390 assert(OrderedContexts.size() == CSNameTable.size() &&
391 "Unmatched ordered and unordered contexts");
392 uint64_t I = 0;
393 for (auto &Context : OrderedContexts)
394 CSNameTable[Context] = I++;
395
396 auto &OS = *OutputStream;
397 encodeULEB128(Value: OrderedContexts.size(), OS);
398 support::endian::Writer Writer(OS, llvm::endianness::little);
399 for (auto Context : OrderedContexts) {
400 auto Frames = Context.getContextFrames();
401 encodeULEB128(Value: Frames.size(), OS);
402 for (auto &Callsite : Frames) {
403 if (std::error_code EC = writeNameIdx(FName: Callsite.Func))
404 return EC;
405 encodeULEB128(Value: Callsite.Location.LineOffset, OS);
406 encodeULEB128(Value: Callsite.Location.Discriminator, OS);
407 }
408 }
409
410 return sampleprof_error::success;
411}
412
413std::error_code
414SampleProfileWriterExtBinaryBase::writeProfileSymbolListSection() {
415 if (ProfSymList && ProfSymList->size() > 0)
416 if (std::error_code EC = ProfSymList->write(OS&: *OutputStream))
417 return EC;
418
419 return sampleprof_error::success;
420}
421
422std::error_code SampleProfileWriterExtBinaryBase::writeOneSection(
423 SecType Type, uint32_t LayoutIdx, const SampleProfileMap &ProfileMap) {
424 // The setting of SecFlagCompress should happen before markSectionStart.
425 if (Type == SecProfileSymbolList && ProfSymList && ProfSymList->toCompress())
426 setToCompressSection(SecProfileSymbolList);
427 if (Type == SecFuncMetadata && FunctionSamples::ProfileIsProbeBased)
428 addSectionFlag(Type: SecFuncMetadata, Flag: SecFuncMetadataFlags::SecFlagIsProbeBased);
429 if (Type == SecFuncMetadata &&
430 (FunctionSamples::ProfileIsCS || FunctionSamples::ProfileIsPreInlined))
431 addSectionFlag(Type: SecFuncMetadata, Flag: SecFuncMetadataFlags::SecFlagHasAttribute);
432 if (Type == SecProfSummary && FunctionSamples::ProfileIsCS)
433 addSectionFlag(Type: SecProfSummary, Flag: SecProfSummaryFlags::SecFlagFullContext);
434 if (Type == SecProfSummary && FunctionSamples::ProfileIsPreInlined)
435 addSectionFlag(Type: SecProfSummary, Flag: SecProfSummaryFlags::SecFlagIsPreInlined);
436 if (Type == SecProfSummary && FunctionSamples::ProfileIsFS)
437 addSectionFlag(Type: SecProfSummary, Flag: SecProfSummaryFlags::SecFlagFSDiscriminator);
438
439 uint64_t SectionStart = markSectionStart(Type, LayoutIdx);
440 switch (Type) {
441 case SecProfSummary:
442 computeSummary(ProfileMap);
443 if (auto EC = writeSummary())
444 return EC;
445 break;
446 case SecNameTable:
447 if (auto EC = writeNameTableSection(ProfileMap))
448 return EC;
449 break;
450 case SecCSNameTable:
451 if (auto EC = writeCSNameTableSection())
452 return EC;
453 break;
454 case SecLBRProfile:
455 SecLBRProfileStart = OutputStream->tell();
456 if (std::error_code EC = writeFuncProfiles(ProfileMap))
457 return EC;
458 break;
459 case SecFuncOffsetTable:
460 if (auto EC = writeFuncOffsetTable())
461 return EC;
462 break;
463 case SecFuncMetadata:
464 if (std::error_code EC = writeFuncMetadata(Profiles: ProfileMap))
465 return EC;
466 break;
467 case SecProfileSymbolList:
468 if (auto EC = writeProfileSymbolListSection())
469 return EC;
470 break;
471 default:
472 if (auto EC = writeCustomSection(Type))
473 return EC;
474 break;
475 }
476 if (std::error_code EC = addNewSection(Type, LayoutIdx, SectionStart))
477 return EC;
478 return sampleprof_error::success;
479}
480
481std::error_code SampleProfileWriterExtBinary::writeDefaultLayout(
482 const SampleProfileMap &ProfileMap) {
483 // The const indices passed to writeOneSection below are specifying the
484 // positions of the sections in SectionHdrLayout. Look at
485 // initSectionHdrLayout to find out where each section is located in
486 // SectionHdrLayout.
487 if (auto EC = writeOneSection(Type: SecProfSummary, LayoutIdx: 0, ProfileMap))
488 return EC;
489 if (auto EC = writeOneSection(Type: SecNameTable, LayoutIdx: 1, ProfileMap))
490 return EC;
491 if (auto EC = writeOneSection(Type: SecCSNameTable, LayoutIdx: 2, ProfileMap))
492 return EC;
493 if (auto EC = writeOneSection(Type: SecLBRProfile, LayoutIdx: 4, ProfileMap))
494 return EC;
495 if (auto EC = writeOneSection(Type: SecProfileSymbolList, LayoutIdx: 5, ProfileMap))
496 return EC;
497 if (auto EC = writeOneSection(Type: SecFuncOffsetTable, LayoutIdx: 3, ProfileMap))
498 return EC;
499 if (auto EC = writeOneSection(Type: SecFuncMetadata, LayoutIdx: 6, ProfileMap))
500 return EC;
501 return sampleprof_error::success;
502}
503
504static void splitProfileMapToTwo(const SampleProfileMap &ProfileMap,
505 SampleProfileMap &ContextProfileMap,
506 SampleProfileMap &NoContextProfileMap) {
507 for (const auto &I : ProfileMap) {
508 if (I.second.getCallsiteSamples().size())
509 ContextProfileMap.insert(x: {I.first, I.second});
510 else
511 NoContextProfileMap.insert(x: {I.first, I.second});
512 }
513}
514
515std::error_code SampleProfileWriterExtBinary::writeCtxSplitLayout(
516 const SampleProfileMap &ProfileMap) {
517 SampleProfileMap ContextProfileMap, NoContextProfileMap;
518 splitProfileMapToTwo(ProfileMap, ContextProfileMap, NoContextProfileMap);
519
520 if (auto EC = writeOneSection(Type: SecProfSummary, LayoutIdx: 0, ProfileMap))
521 return EC;
522 if (auto EC = writeOneSection(Type: SecNameTable, LayoutIdx: 1, ProfileMap))
523 return EC;
524 if (auto EC = writeOneSection(Type: SecLBRProfile, LayoutIdx: 3, ProfileMap: ContextProfileMap))
525 return EC;
526 if (auto EC = writeOneSection(Type: SecFuncOffsetTable, LayoutIdx: 2, ProfileMap: ContextProfileMap))
527 return EC;
528 // Mark the section to have no context. Note section flag needs to be set
529 // before writing the section.
530 addSectionFlag(SectionIdx: 5, Flag: SecCommonFlags::SecFlagFlat);
531 if (auto EC = writeOneSection(Type: SecLBRProfile, LayoutIdx: 5, ProfileMap: NoContextProfileMap))
532 return EC;
533 // Mark the section to have no context. Note section flag needs to be set
534 // before writing the section.
535 addSectionFlag(SectionIdx: 4, Flag: SecCommonFlags::SecFlagFlat);
536 if (auto EC = writeOneSection(Type: SecFuncOffsetTable, LayoutIdx: 4, ProfileMap: NoContextProfileMap))
537 return EC;
538 if (auto EC = writeOneSection(Type: SecProfileSymbolList, LayoutIdx: 6, ProfileMap))
539 return EC;
540 if (auto EC = writeOneSection(Type: SecFuncMetadata, LayoutIdx: 7, ProfileMap))
541 return EC;
542
543 return sampleprof_error::success;
544}
545
546std::error_code SampleProfileWriterExtBinary::writeSections(
547 const SampleProfileMap &ProfileMap) {
548 std::error_code EC;
549 if (SecLayout == DefaultLayout)
550 EC = writeDefaultLayout(ProfileMap);
551 else if (SecLayout == CtxSplitLayout)
552 EC = writeCtxSplitLayout(ProfileMap);
553 else
554 llvm_unreachable("Unsupported layout");
555 return EC;
556}
557
558/// Write samples to a text file.
559///
560/// Note: it may be tempting to implement this in terms of
561/// FunctionSamples::print(). Please don't. The dump functionality is intended
562/// for debugging and has no specified form.
563///
564/// The format used here is more structured and deliberate because
565/// it needs to be parsed by the SampleProfileReaderText class.
566std::error_code SampleProfileWriterText::writeSample(const FunctionSamples &S) {
567 auto &OS = *OutputStream;
568 if (FunctionSamples::ProfileIsCS)
569 OS << "[" << S.getContext().toString() << "]:" << S.getTotalSamples();
570 else
571 OS << S.getFunction() << ":" << S.getTotalSamples();
572
573 if (Indent == 0)
574 OS << ":" << S.getHeadSamples();
575 OS << "\n";
576 LineCount++;
577
578 SampleSorter<LineLocation, SampleRecord> SortedSamples(S.getBodySamples());
579 for (const auto &I : SortedSamples.get()) {
580 LineLocation Loc = I->first;
581 const SampleRecord &Sample = I->second;
582 OS.indent(NumSpaces: Indent + 1);
583 Loc.print(OS);
584 OS << ": " << Sample.getSamples();
585
586 for (const auto &J : Sample.getSortedCallTargets())
587 OS << " " << J.first << ":" << J.second;
588 OS << "\n";
589 LineCount++;
590 }
591
592 SampleSorter<LineLocation, FunctionSamplesMap> SortedCallsiteSamples(
593 S.getCallsiteSamples());
594 Indent += 1;
595 for (const auto &I : SortedCallsiteSamples.get())
596 for (const auto &FS : I->second) {
597 LineLocation Loc = I->first;
598 const FunctionSamples &CalleeSamples = FS.second;
599 OS.indent(NumSpaces: Indent);
600 Loc.print(OS);
601 OS << ": ";
602 if (std::error_code EC = writeSample(S: CalleeSamples))
603 return EC;
604 }
605 Indent -= 1;
606
607 if (FunctionSamples::ProfileIsProbeBased) {
608 OS.indent(NumSpaces: Indent + 1);
609 OS << "!CFGChecksum: " << S.getFunctionHash() << "\n";
610 LineCount++;
611 }
612
613 if (S.getContext().getAllAttributes()) {
614 OS.indent(NumSpaces: Indent + 1);
615 OS << "!Attributes: " << S.getContext().getAllAttributes() << "\n";
616 LineCount++;
617 }
618
619 if (Indent == 0 && MarkFlatProfiles && S.getCallsiteSamples().size() == 0)
620 OS << " !Flat\n";
621
622 return sampleprof_error::success;
623}
624
625std::error_code
626SampleProfileWriterBinary::writeContextIdx(const SampleContext &Context) {
627 assert(!Context.hasContext() && "cs profile is not supported");
628 return writeNameIdx(FName: Context.getFunction());
629}
630
631std::error_code SampleProfileWriterBinary::writeNameIdx(FunctionId FName) {
632 auto &NTable = getNameTable();
633 const auto &Ret = NTable.find(Key: FName);
634 if (Ret == NTable.end())
635 return sampleprof_error::truncated_name_table;
636 encodeULEB128(Value: Ret->second, OS&: *OutputStream);
637 return sampleprof_error::success;
638}
639
640void SampleProfileWriterBinary::addName(FunctionId FName) {
641 auto &NTable = getNameTable();
642 NTable.insert(KV: std::make_pair(x&: FName, y: 0));
643}
644
645void SampleProfileWriterBinary::addContext(const SampleContext &Context) {
646 addName(FName: Context.getFunction());
647}
648
649void SampleProfileWriterBinary::addNames(const FunctionSamples &S) {
650 // Add all the names in indirect call targets.
651 for (const auto &I : S.getBodySamples()) {
652 const SampleRecord &Sample = I.second;
653 for (const auto &J : Sample.getCallTargets())
654 addName(FName: J.first);
655 }
656
657 // Recursively add all the names for inlined callsites.
658 for (const auto &J : S.getCallsiteSamples())
659 for (const auto &FS : J.second) {
660 const FunctionSamples &CalleeSamples = FS.second;
661 addName(FName: CalleeSamples.getFunction());
662 addNames(S: CalleeSamples);
663 }
664}
665
666void SampleProfileWriterExtBinaryBase::addContext(
667 const SampleContext &Context) {
668 if (Context.hasContext()) {
669 for (auto &Callsite : Context.getContextFrames())
670 SampleProfileWriterBinary::addName(FName: Callsite.Func);
671 CSNameTable.insert(KV: std::make_pair(x: Context, y: 0));
672 } else {
673 SampleProfileWriterBinary::addName(FName: Context.getFunction());
674 }
675}
676
677void SampleProfileWriterBinary::stablizeNameTable(
678 MapVector<FunctionId, uint32_t> &NameTable, std::set<FunctionId> &V) {
679 // Sort the names to make NameTable deterministic.
680 for (const auto &I : NameTable)
681 V.insert(x: I.first);
682 int i = 0;
683 for (const FunctionId &N : V)
684 NameTable[N] = i++;
685}
686
687std::error_code SampleProfileWriterBinary::writeNameTable() {
688 auto &OS = *OutputStream;
689 std::set<FunctionId> V;
690 stablizeNameTable(NameTable, V);
691
692 // Write out the name table.
693 encodeULEB128(Value: NameTable.size(), OS);
694 for (auto N : V) {
695 OS << N;
696 encodeULEB128(Value: 0, OS);
697 }
698 return sampleprof_error::success;
699}
700
701std::error_code
702SampleProfileWriterBinary::writeMagicIdent(SampleProfileFormat Format) {
703 auto &OS = *OutputStream;
704 // Write file magic identifier.
705 encodeULEB128(Value: SPMagic(Format), OS);
706 encodeULEB128(Value: SPVersion(), OS);
707 return sampleprof_error::success;
708}
709
710std::error_code
711SampleProfileWriterBinary::writeHeader(const SampleProfileMap &ProfileMap) {
712 // When calling write on a different profile map, existing names should be
713 // cleared.
714 NameTable.clear();
715
716 writeMagicIdent(Format);
717
718 computeSummary(ProfileMap);
719 if (auto EC = writeSummary())
720 return EC;
721
722 // Generate the name table for all the functions referenced in the profile.
723 for (const auto &I : ProfileMap) {
724 addContext(Context: I.second.getContext());
725 addNames(S: I.second);
726 }
727
728 writeNameTable();
729 return sampleprof_error::success;
730}
731
732void SampleProfileWriterExtBinaryBase::setToCompressAllSections() {
733 for (auto &Entry : SectionHdrLayout)
734 addSecFlag(Entry, Flag: SecCommonFlags::SecFlagCompress);
735}
736
737void SampleProfileWriterExtBinaryBase::setToCompressSection(SecType Type) {
738 addSectionFlag(Type, Flag: SecCommonFlags::SecFlagCompress);
739}
740
741void SampleProfileWriterExtBinaryBase::allocSecHdrTable() {
742 support::endian::Writer Writer(*OutputStream, llvm::endianness::little);
743
744 Writer.write(Val: static_cast<uint64_t>(SectionHdrLayout.size()));
745 SecHdrTableOffset = OutputStream->tell();
746 for (uint32_t i = 0; i < SectionHdrLayout.size(); i++) {
747 Writer.write(Val: static_cast<uint64_t>(-1));
748 Writer.write(Val: static_cast<uint64_t>(-1));
749 Writer.write(Val: static_cast<uint64_t>(-1));
750 Writer.write(Val: static_cast<uint64_t>(-1));
751 }
752}
753
754std::error_code SampleProfileWriterExtBinaryBase::writeSecHdrTable() {
755 assert(SecHdrTable.size() == SectionHdrLayout.size() &&
756 "SecHdrTable entries doesn't match SectionHdrLayout");
757 SmallVector<uint32_t, 16> IndexMap(SecHdrTable.size(), -1);
758 for (uint32_t TableIdx = 0; TableIdx < SecHdrTable.size(); TableIdx++) {
759 IndexMap[SecHdrTable[TableIdx].LayoutIndex] = TableIdx;
760 }
761
762 // Write the section header table in the order specified in
763 // SectionHdrLayout. SectionHdrLayout specifies the sections
764 // order in which profile reader expect to read, so the section
765 // header table should be written in the order in SectionHdrLayout.
766 // Note that the section order in SecHdrTable may be different
767 // from the order in SectionHdrLayout, for example, SecFuncOffsetTable
768 // needs to be computed after SecLBRProfile (the order in SecHdrTable),
769 // but it needs to be read before SecLBRProfile (the order in
770 // SectionHdrLayout). So we use IndexMap above to switch the order.
771 support::endian::SeekableWriter Writer(
772 static_cast<raw_pwrite_stream &>(*OutputStream),
773 llvm::endianness::little);
774 for (uint32_t LayoutIdx = 0; LayoutIdx < SectionHdrLayout.size();
775 LayoutIdx++) {
776 assert(IndexMap[LayoutIdx] < SecHdrTable.size() &&
777 "Incorrect LayoutIdx in SecHdrTable");
778 auto Entry = SecHdrTable[IndexMap[LayoutIdx]];
779 Writer.pwrite(Val: static_cast<uint64_t>(Entry.Type),
780 Offset: SecHdrTableOffset + 4 * LayoutIdx * sizeof(uint64_t));
781 Writer.pwrite(Val: static_cast<uint64_t>(Entry.Flags),
782 Offset: SecHdrTableOffset + (4 * LayoutIdx + 1) * sizeof(uint64_t));
783 Writer.pwrite(Val: static_cast<uint64_t>(Entry.Offset),
784 Offset: SecHdrTableOffset + (4 * LayoutIdx + 2) * sizeof(uint64_t));
785 Writer.pwrite(Val: static_cast<uint64_t>(Entry.Size),
786 Offset: SecHdrTableOffset + (4 * LayoutIdx + 3) * sizeof(uint64_t));
787 }
788
789 return sampleprof_error::success;
790}
791
792std::error_code SampleProfileWriterExtBinaryBase::writeHeader(
793 const SampleProfileMap &ProfileMap) {
794 auto &OS = *OutputStream;
795 FileStart = OS.tell();
796 writeMagicIdent(Format);
797
798 allocSecHdrTable();
799 return sampleprof_error::success;
800}
801
802std::error_code SampleProfileWriterBinary::writeSummary() {
803 auto &OS = *OutputStream;
804 encodeULEB128(Value: Summary->getTotalCount(), OS);
805 encodeULEB128(Value: Summary->getMaxCount(), OS);
806 encodeULEB128(Value: Summary->getMaxFunctionCount(), OS);
807 encodeULEB128(Value: Summary->getNumCounts(), OS);
808 encodeULEB128(Value: Summary->getNumFunctions(), OS);
809 ArrayRef<ProfileSummaryEntry> Entries = Summary->getDetailedSummary();
810 encodeULEB128(Value: Entries.size(), OS);
811 for (auto Entry : Entries) {
812 encodeULEB128(Value: Entry.Cutoff, OS);
813 encodeULEB128(Value: Entry.MinCount, OS);
814 encodeULEB128(Value: Entry.NumCounts, OS);
815 }
816 return sampleprof_error::success;
817}
818std::error_code SampleProfileWriterBinary::writeBody(const FunctionSamples &S) {
819 auto &OS = *OutputStream;
820 if (std::error_code EC = writeContextIdx(Context: S.getContext()))
821 return EC;
822
823 encodeULEB128(Value: S.getTotalSamples(), OS);
824
825 // Emit all the body samples.
826 encodeULEB128(Value: S.getBodySamples().size(), OS);
827 for (const auto &I : S.getBodySamples()) {
828 LineLocation Loc = I.first;
829 const SampleRecord &Sample = I.second;
830 Loc.serialize(OS);
831 Sample.serialize(OS, NameTable: getNameTable());
832 }
833
834 // Recursively emit all the callsite samples.
835 uint64_t NumCallsites = 0;
836 for (const auto &J : S.getCallsiteSamples())
837 NumCallsites += J.second.size();
838 encodeULEB128(Value: NumCallsites, OS);
839 for (const auto &J : S.getCallsiteSamples())
840 for (const auto &FS : J.second) {
841 LineLocation Loc = J.first;
842 const FunctionSamples &CalleeSamples = FS.second;
843 Loc.serialize(OS);
844 if (std::error_code EC = writeBody(S: CalleeSamples))
845 return EC;
846 }
847
848 return sampleprof_error::success;
849}
850
851/// Write samples of a top-level function to a binary file.
852///
853/// \returns true if the samples were written successfully, false otherwise.
854std::error_code
855SampleProfileWriterBinary::writeSample(const FunctionSamples &S) {
856 encodeULEB128(Value: S.getHeadSamples(), OS&: *OutputStream);
857 return writeBody(S);
858}
859
860/// Create a sample profile file writer based on the specified format.
861///
862/// \param Filename The file to create.
863///
864/// \param Format Encoding format for the profile file.
865///
866/// \returns an error code indicating the status of the created writer.
867ErrorOr<std::unique_ptr<SampleProfileWriter>>
868SampleProfileWriter::create(StringRef Filename, SampleProfileFormat Format) {
869 std::error_code EC;
870 std::unique_ptr<raw_ostream> OS;
871 if (Format == SPF_Binary || Format == SPF_Ext_Binary)
872 OS.reset(p: new raw_fd_ostream(Filename, EC, sys::fs::OF_None));
873 else
874 OS.reset(p: new raw_fd_ostream(Filename, EC, sys::fs::OF_TextWithCRLF));
875 if (EC)
876 return EC;
877
878 return create(OS, Format);
879}
880
881/// Create a sample profile stream writer based on the specified format.
882///
883/// \param OS The output stream to store the profile data to.
884///
885/// \param Format Encoding format for the profile file.
886///
887/// \returns an error code indicating the status of the created writer.
888ErrorOr<std::unique_ptr<SampleProfileWriter>>
889SampleProfileWriter::create(std::unique_ptr<raw_ostream> &OS,
890 SampleProfileFormat Format) {
891 std::error_code EC;
892 std::unique_ptr<SampleProfileWriter> Writer;
893
894 // Currently only Text and Extended Binary format are supported for CSSPGO.
895 if ((FunctionSamples::ProfileIsCS || FunctionSamples::ProfileIsProbeBased) &&
896 Format == SPF_Binary)
897 return sampleprof_error::unsupported_writing_format;
898
899 if (Format == SPF_Binary)
900 Writer.reset(p: new SampleProfileWriterRawBinary(OS));
901 else if (Format == SPF_Ext_Binary)
902 Writer.reset(p: new SampleProfileWriterExtBinary(OS));
903 else if (Format == SPF_Text)
904 Writer.reset(p: new SampleProfileWriterText(OS));
905 else if (Format == SPF_GCC)
906 EC = sampleprof_error::unsupported_writing_format;
907 else
908 EC = sampleprof_error::unrecognized_format;
909
910 if (EC)
911 return EC;
912
913 Writer->Format = Format;
914 return std::move(Writer);
915}
916
917void SampleProfileWriter::computeSummary(const SampleProfileMap &ProfileMap) {
918 SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs);
919 Summary = Builder.computeSummaryForProfiles(Profiles: ProfileMap);
920}
921