1//===- SampleProfWriter.cpp - Write LLVM sample profile data --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the class that writes LLVM sample profiles. It
10// supports two file formats: text and binary. The textual representation
11// is useful for debugging and testing purposes. The binary representation
12// is more compact, resulting in smaller file sizes. However, they can
13// both be used interchangeably.
14//
15// See lib/ProfileData/SampleProfReader.cpp for documentation on each of the
16// supported formats.
17//
18//===----------------------------------------------------------------------===//
19
20#include "llvm/ProfileData/SampleProfWriter.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/ProfileData/ProfileCommon.h"
23#include "llvm/ProfileData/SampleProf.h"
24#include "llvm/Support/Compression.h"
25#include "llvm/Support/Endian.h"
26#include "llvm/Support/EndianStream.h"
27#include "llvm/Support/ErrorOr.h"
28#include "llvm/Support/FileSystem.h"
29#include "llvm/Support/LEB128.h"
30#include "llvm/Support/MD5.h"
31#include "llvm/Support/raw_ostream.h"
32#include <algorithm>
33#include <cmath>
34#include <cstdint>
35#include <memory>
36#include <set>
37#include <system_error>
38#include <utility>
39#include <vector>
40
41#define DEBUG_TYPE "llvm-profdata"
42
43using namespace llvm;
44using namespace sampleprof;
45
46namespace llvm {
47namespace support {
48namespace endian {
49namespace {
50
51// Adapter class to llvm::support::endian::Writer for pwrite().
52struct SeekableWriter {
53 raw_pwrite_stream &OS;
54 endianness Endian;
55 SeekableWriter(raw_pwrite_stream &OS, endianness Endian)
56 : OS(OS), Endian(Endian) {}
57
58 template <typename ValueType>
59 void pwrite(ValueType Val, size_t Offset) {
60 std::string StringBuf;
61 raw_string_ostream SStream(StringBuf);
62 Writer(SStream, Endian).write(Val);
63 OS.pwrite(Ptr: StringBuf.data(), Size: StringBuf.size(), Offset);
64 }
65};
66
67} // namespace
68} // namespace endian
69} // namespace support
70} // namespace llvm
71
72DefaultFunctionPruningStrategy::DefaultFunctionPruningStrategy(
73 SampleProfileMap &ProfileMap, size_t OutputSizeLimit)
74 : FunctionPruningStrategy(ProfileMap, OutputSizeLimit) {
75 sortFuncProfiles(ProfileMap, SortedProfiles&: SortedFunctions);
76}
77
78void DefaultFunctionPruningStrategy::Erase(size_t CurrentOutputSize) {
79 double D = (double)OutputSizeLimit / CurrentOutputSize;
80 size_t NewSize = (size_t)round(x: ProfileMap.size() * D * D);
81 size_t NumToRemove = ProfileMap.size() - NewSize;
82 if (NumToRemove < 1)
83 NumToRemove = 1;
84
85 assert(NumToRemove <= SortedFunctions.size());
86 for (const NameFunctionSamples &E :
87 llvm::drop_begin(RangeOrContainer&: SortedFunctions, N: SortedFunctions.size() - NumToRemove))
88 ProfileMap.erase(Key: E.first);
89 SortedFunctions.resize(new_size: SortedFunctions.size() - NumToRemove);
90}
91
92std::error_code SampleProfileWriter::writeWithSizeLimitInternal(
93 SampleProfileMap &ProfileMap, size_t OutputSizeLimit,
94 FunctionPruningStrategy *Strategy) {
95 if (OutputSizeLimit == 0)
96 return write(ProfileMap);
97
98 size_t OriginalFunctionCount = ProfileMap.size();
99
100 std::unique_ptr<raw_ostream> OriginalOutputStream;
101 OutputStream.swap(u&: OriginalOutputStream);
102
103 size_t IterationCount = 0;
104 size_t TotalSize;
105
106 SmallVector<char> StringBuffer;
107 do {
108 StringBuffer.clear();
109 OutputStream.reset(p: new raw_svector_ostream(StringBuffer));
110 if (std::error_code EC = write(ProfileMap))
111 return EC;
112
113 TotalSize = StringBuffer.size();
114 // On Windows every "\n" is actually written as "\r\n" to disk but not to
115 // memory buffer, this difference should be added when considering the total
116 // output size.
117#ifdef _WIN32
118 if (Format == SPF_Text)
119 TotalSize += LineCount;
120#endif
121 if (TotalSize <= OutputSizeLimit)
122 break;
123
124 Strategy->Erase(CurrentOutputSize: TotalSize);
125 IterationCount++;
126 } while (ProfileMap.size() != 0);
127
128 if (ProfileMap.size() == 0)
129 return sampleprof_error::too_large;
130
131 OutputStream.swap(u&: OriginalOutputStream);
132 OutputStream->write(Ptr: StringBuffer.data(), Size: StringBuffer.size());
133 LLVM_DEBUG(dbgs() << "Profile originally has " << OriginalFunctionCount
134 << " functions, reduced to " << ProfileMap.size() << " in "
135 << IterationCount << " iterations\n");
136 // Silence warning on Release build.
137 (void)OriginalFunctionCount;
138 (void)IterationCount;
139 return sampleprof_error::success;
140}
141
142std::error_code
143SampleProfileWriter::writeFuncProfiles(const SampleProfileMap &ProfileMap) {
144 std::vector<NameFunctionSamples> V;
145 sortFuncProfiles(ProfileMap, SortedProfiles&: V);
146 for (const auto &I : V) {
147 if (std::error_code EC = writeSample(S: *I.second))
148 return EC;
149 }
150 return sampleprof_error::success;
151}
152
153std::error_code SampleProfileWriter::write(const SampleProfileMap &ProfileMap) {
154 if (std::error_code EC = writeHeader(ProfileMap))
155 return EC;
156
157 if (std::error_code EC = writeFuncProfiles(ProfileMap))
158 return EC;
159
160 return sampleprof_error::success;
161}
162
163/// Return the current position and prepare to use it as the start
164/// position of a section given the section type \p Type and its position
165/// \p LayoutIdx in SectionHdrLayout.
166uint64_t
167SampleProfileWriterExtBinaryBase::markSectionStart(SecType Type,
168 uint32_t LayoutIdx) {
169 uint64_t SectionStart = OutputStream->tell();
170 assert(LayoutIdx < SectionHdrLayout.size() && "LayoutIdx out of range");
171 const auto &Entry = SectionHdrLayout[LayoutIdx];
172 assert(Entry.Type == Type && "Unexpected section type");
173 // Use LocalBuf as a temporary output for writting data.
174 if (hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagCompress))
175 LocalBufStream.swap(u&: OutputStream);
176 return SectionStart;
177}
178
179std::error_code SampleProfileWriterExtBinaryBase::compressAndOutput() {
180 if (!llvm::compression::zlib::isAvailable())
181 return sampleprof_error::zlib_unavailable;
182 std::string &UncompressedStrings =
183 static_cast<raw_string_ostream *>(LocalBufStream.get())->str();
184 if (UncompressedStrings.size() == 0)
185 return sampleprof_error::success;
186 auto &OS = *OutputStream;
187 SmallVector<uint8_t, 128> CompressedStrings;
188 compression::zlib::compress(Input: arrayRefFromStringRef(Input: UncompressedStrings),
189 CompressedBuffer&: CompressedStrings,
190 Level: compression::zlib::BestSizeCompression);
191 encodeULEB128(Value: UncompressedStrings.size(), OS);
192 encodeULEB128(Value: CompressedStrings.size(), OS);
193 OS << toStringRef(Input: CompressedStrings);
194 UncompressedStrings.clear();
195 return sampleprof_error::success;
196}
197
198/// Add a new section into section header table given the section type
199/// \p Type, its position \p LayoutIdx in SectionHdrLayout and the
200/// location \p SectionStart where the section should be written to.
201std::error_code SampleProfileWriterExtBinaryBase::addNewSection(
202 SecType Type, uint32_t LayoutIdx, uint64_t SectionStart) {
203 assert(LayoutIdx < SectionHdrLayout.size() && "LayoutIdx out of range");
204 const auto &Entry = SectionHdrLayout[LayoutIdx];
205 assert(Entry.Type == Type && "Unexpected section type");
206 if (hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagCompress)) {
207 LocalBufStream.swap(u&: OutputStream);
208 if (std::error_code EC = compressAndOutput())
209 return EC;
210 }
211 SecHdrTable.push_back(x: {.Type: Type, .Flags: Entry.Flags, .Offset: SectionStart - FileStart,
212 .Size: OutputStream->tell() - SectionStart, .LayoutIndex: LayoutIdx});
213 return sampleprof_error::success;
214}
215
216std::error_code
217SampleProfileWriterExtBinaryBase::write(const SampleProfileMap &ProfileMap) {
218 // When calling write on a different profile map, existing states should be
219 // cleared.
220 NameTable.clear();
221 CSNameTable.clear();
222 SecHdrTable.clear();
223
224 if (std::error_code EC = writeHeader(ProfileMap))
225 return EC;
226
227 std::string LocalBuf;
228 LocalBufStream = std::make_unique<raw_string_ostream>(args&: LocalBuf);
229 if (std::error_code EC = writeSections(ProfileMap))
230 return EC;
231
232 if (std::error_code EC = writeSecHdrTable())
233 return EC;
234
235 return sampleprof_error::success;
236}
237
238std::error_code SampleProfileWriterExtBinaryBase::writeContextIdx(
239 const SampleContext &Context) {
240 if (Context.hasContext())
241 return writeCSNameIdx(Context);
242 else
243 return SampleProfileWriterBinary::writeNameIdx(FName: Context.getFunction());
244}
245
246std::error_code
247SampleProfileWriterExtBinaryBase::writeCSNameIdx(const SampleContext &Context) {
248 const auto &Ret = CSNameTable.find(Key: Context);
249 if (Ret == CSNameTable.end())
250 return sampleprof_error::truncated_name_table;
251 encodeULEB128(Value: Ret->second, OS&: *OutputStream);
252 return sampleprof_error::success;
253}
254
255std::error_code
256SampleProfileWriterExtBinaryBase::writeSample(const FunctionSamples &S) {
257 uint64_t Offset = OutputStream->tell();
258 auto &Context = S.getContext();
259 FuncOffsetTable[Context] = Offset - SecLBRProfileStart;
260 encodeULEB128(Value: S.getHeadSamples(), OS&: *OutputStream);
261 return writeBody(S);
262}
263
264std::error_code SampleProfileWriterExtBinaryBase::writeFuncOffsetTable() {
265 auto &OS = *OutputStream;
266
267 // Write out the table size.
268 encodeULEB128(Value: FuncOffsetTable.size(), OS);
269
270 // Write out FuncOffsetTable.
271 auto WriteItem = [&](const SampleContext &Context, uint64_t Offset) {
272 if (std::error_code EC = writeContextIdx(Context))
273 return EC;
274 encodeULEB128(Value: Offset, OS);
275 return (std::error_code)sampleprof_error::success;
276 };
277
278 if (FunctionSamples::ProfileIsCS) {
279 // Sort the contexts before writing them out. This is to help fast load all
280 // context profiles for a function as well as their callee contexts which
281 // can help profile-guided importing for ThinLTO.
282 std::map<SampleContext, uint64_t> OrderedFuncOffsetTable(
283 FuncOffsetTable.begin(), FuncOffsetTable.end());
284 for (const auto &Entry : OrderedFuncOffsetTable) {
285 if (std::error_code EC = WriteItem(Entry.first, Entry.second))
286 return EC;
287 }
288 addSectionFlag(Type: SecFuncOffsetTable, Flag: SecFuncOffsetFlags::SecFlagOrdered);
289 } else {
290 for (const auto &Entry : FuncOffsetTable) {
291 if (std::error_code EC = WriteItem(Entry.first, Entry.second))
292 return EC;
293 }
294 }
295
296 FuncOffsetTable.clear();
297 return sampleprof_error::success;
298}
299
300std::error_code SampleProfileWriterExtBinaryBase::writeFuncMetadata(
301 const FunctionSamples &FunctionProfile) {
302 auto &OS = *OutputStream;
303 if (std::error_code EC = writeContextIdx(Context: FunctionProfile.getContext()))
304 return EC;
305
306 if (FunctionSamples::ProfileIsProbeBased)
307 encodeULEB128(Value: FunctionProfile.getFunctionHash(), OS);
308 if (FunctionSamples::ProfileIsCS || FunctionSamples::ProfileIsPreInlined) {
309 encodeULEB128(Value: FunctionProfile.getContext().getAllAttributes(), OS);
310 }
311
312 if (!FunctionSamples::ProfileIsCS) {
313 // Recursively emit attributes for all callee samples.
314 uint64_t NumCallsites = 0;
315 for (const auto &J : FunctionProfile.getCallsiteSamples())
316 NumCallsites += J.second.size();
317 encodeULEB128(Value: NumCallsites, OS);
318 for (const auto &J : FunctionProfile.getCallsiteSamples()) {
319 for (const auto &FS : J.second) {
320 LineLocation Loc = J.first;
321 encodeULEB128(Value: Loc.LineOffset, OS);
322 encodeULEB128(Value: Loc.Discriminator, OS);
323 if (std::error_code EC = writeFuncMetadata(FunctionProfile: FS.second))
324 return EC;
325 }
326 }
327 }
328
329 return sampleprof_error::success;
330}
331
332std::error_code SampleProfileWriterExtBinaryBase::writeFuncMetadata(
333 const SampleProfileMap &Profiles) {
334 if (!FunctionSamples::ProfileIsProbeBased && !FunctionSamples::ProfileIsCS &&
335 !FunctionSamples::ProfileIsPreInlined)
336 return sampleprof_error::success;
337 for (const auto &Entry : Profiles) {
338 if (std::error_code EC = writeFuncMetadata(FunctionProfile: Entry.second))
339 return EC;
340 }
341 return sampleprof_error::success;
342}
343
344std::error_code SampleProfileWriterExtBinaryBase::writeNameTable() {
345 if (!UseMD5)
346 return SampleProfileWriterBinary::writeNameTable();
347
348 auto &OS = *OutputStream;
349 std::set<FunctionId> V;
350 stablizeNameTable(NameTable, V);
351
352 // Write out the MD5 name table. We wrote unencoded MD5 so reader can
353 // retrieve the name using the name index without having to read the
354 // whole name table.
355 encodeULEB128(Value: NameTable.size(), OS);
356 support::endian::Writer Writer(OS, llvm::endianness::little);
357 for (auto N : V)
358 Writer.write(Val: N.getHashCode());
359 return sampleprof_error::success;
360}
361
362std::error_code SampleProfileWriterExtBinaryBase::writeNameTableSection(
363 const SampleProfileMap &ProfileMap) {
364 for (const auto &I : ProfileMap) {
365 addContext(Context: I.second.getContext());
366 addNames(S: I.second);
367 }
368
369 // If NameTable contains ".__uniq." suffix, set SecFlagUniqSuffix flag
370 // so compiler won't strip the suffix during profile matching after
371 // seeing the flag in the profile.
372 // Original names are unavailable if using MD5, so this option has no use.
373 if (!UseMD5) {
374 for (const auto &I : NameTable) {
375 if (I.first.stringRef().contains(Other: FunctionSamples::UniqSuffix)) {
376 addSectionFlag(Type: SecNameTable, Flag: SecNameTableFlags::SecFlagUniqSuffix);
377 break;
378 }
379 }
380 }
381
382 if (auto EC = writeNameTable())
383 return EC;
384 return sampleprof_error::success;
385}
386
387std::error_code SampleProfileWriterExtBinaryBase::writeCSNameTableSection() {
388 // Sort the names to make CSNameTable deterministic.
389 std::set<SampleContext> OrderedContexts;
390 for (const auto &I : CSNameTable)
391 OrderedContexts.insert(x: I.first);
392 assert(OrderedContexts.size() == CSNameTable.size() &&
393 "Unmatched ordered and unordered contexts");
394 uint64_t I = 0;
395 for (auto &Context : OrderedContexts)
396 CSNameTable[Context] = I++;
397
398 auto &OS = *OutputStream;
399 encodeULEB128(Value: OrderedContexts.size(), OS);
400 support::endian::Writer Writer(OS, llvm::endianness::little);
401 for (auto Context : OrderedContexts) {
402 auto Frames = Context.getContextFrames();
403 encodeULEB128(Value: Frames.size(), OS);
404 for (auto &Callsite : Frames) {
405 if (std::error_code EC = writeNameIdx(FName: Callsite.Func))
406 return EC;
407 encodeULEB128(Value: Callsite.Location.LineOffset, OS);
408 encodeULEB128(Value: Callsite.Location.Discriminator, OS);
409 }
410 }
411
412 return sampleprof_error::success;
413}
414
415std::error_code
416SampleProfileWriterExtBinaryBase::writeProfileSymbolListSection() {
417 if (ProfSymList && ProfSymList->size() > 0)
418 if (std::error_code EC = ProfSymList->write(OS&: *OutputStream))
419 return EC;
420
421 return sampleprof_error::success;
422}
423
424std::error_code SampleProfileWriterExtBinaryBase::writeOneSection(
425 SecType Type, uint32_t LayoutIdx, const SampleProfileMap &ProfileMap) {
426 // The setting of SecFlagCompress should happen before markSectionStart.
427 if (Type == SecProfileSymbolList && ProfSymList && ProfSymList->toCompress())
428 setToCompressSection(SecProfileSymbolList);
429 if (Type == SecFuncMetadata && FunctionSamples::ProfileIsProbeBased)
430 addSectionFlag(Type: SecFuncMetadata, Flag: SecFuncMetadataFlags::SecFlagIsProbeBased);
431 if (Type == SecFuncMetadata &&
432 (FunctionSamples::ProfileIsCS || FunctionSamples::ProfileIsPreInlined))
433 addSectionFlag(Type: SecFuncMetadata, Flag: SecFuncMetadataFlags::SecFlagHasAttribute);
434 if (Type == SecProfSummary && FunctionSamples::ProfileIsCS)
435 addSectionFlag(Type: SecProfSummary, Flag: SecProfSummaryFlags::SecFlagFullContext);
436 if (Type == SecProfSummary && FunctionSamples::ProfileIsPreInlined)
437 addSectionFlag(Type: SecProfSummary, Flag: SecProfSummaryFlags::SecFlagIsPreInlined);
438 if (Type == SecProfSummary && FunctionSamples::ProfileIsFS)
439 addSectionFlag(Type: SecProfSummary, Flag: SecProfSummaryFlags::SecFlagFSDiscriminator);
440
441 uint64_t SectionStart = markSectionStart(Type, LayoutIdx);
442 switch (Type) {
443 case SecProfSummary:
444 computeSummary(ProfileMap);
445 if (auto EC = writeSummary())
446 return EC;
447 break;
448 case SecNameTable:
449 if (auto EC = writeNameTableSection(ProfileMap))
450 return EC;
451 break;
452 case SecCSNameTable:
453 if (auto EC = writeCSNameTableSection())
454 return EC;
455 break;
456 case SecLBRProfile:
457 SecLBRProfileStart = OutputStream->tell();
458 if (std::error_code EC = writeFuncProfiles(ProfileMap))
459 return EC;
460 break;
461 case SecFuncOffsetTable:
462 if (auto EC = writeFuncOffsetTable())
463 return EC;
464 break;
465 case SecFuncMetadata:
466 if (std::error_code EC = writeFuncMetadata(Profiles: ProfileMap))
467 return EC;
468 break;
469 case SecProfileSymbolList:
470 if (auto EC = writeProfileSymbolListSection())
471 return EC;
472 break;
473 default:
474 if (auto EC = writeCustomSection(Type))
475 return EC;
476 break;
477 }
478 if (std::error_code EC = addNewSection(Type, LayoutIdx, SectionStart))
479 return EC;
480 return sampleprof_error::success;
481}
482
483std::error_code SampleProfileWriterExtBinary::writeDefaultLayout(
484 const SampleProfileMap &ProfileMap) {
485 // The const indices passed to writeOneSection below are specifying the
486 // positions of the sections in SectionHdrLayout. Look at
487 // initSectionHdrLayout to find out where each section is located in
488 // SectionHdrLayout.
489 if (auto EC = writeOneSection(Type: SecProfSummary, LayoutIdx: 0, ProfileMap))
490 return EC;
491 if (auto EC = writeOneSection(Type: SecNameTable, LayoutIdx: 1, ProfileMap))
492 return EC;
493 if (auto EC = writeOneSection(Type: SecCSNameTable, LayoutIdx: 2, ProfileMap))
494 return EC;
495 if (auto EC = writeOneSection(Type: SecLBRProfile, LayoutIdx: 4, ProfileMap))
496 return EC;
497 if (auto EC = writeOneSection(Type: SecProfileSymbolList, LayoutIdx: 5, ProfileMap))
498 return EC;
499 if (auto EC = writeOneSection(Type: SecFuncOffsetTable, LayoutIdx: 3, ProfileMap))
500 return EC;
501 if (auto EC = writeOneSection(Type: SecFuncMetadata, LayoutIdx: 6, ProfileMap))
502 return EC;
503 return sampleprof_error::success;
504}
505
506static void splitProfileMapToTwo(const SampleProfileMap &ProfileMap,
507 SampleProfileMap &ContextProfileMap,
508 SampleProfileMap &NoContextProfileMap) {
509 for (const auto &I : ProfileMap) {
510 if (I.second.getCallsiteSamples().size())
511 ContextProfileMap.insert(x: {I.first, I.second});
512 else
513 NoContextProfileMap.insert(x: {I.first, I.second});
514 }
515}
516
517std::error_code SampleProfileWriterExtBinary::writeCtxSplitLayout(
518 const SampleProfileMap &ProfileMap) {
519 SampleProfileMap ContextProfileMap, NoContextProfileMap;
520 splitProfileMapToTwo(ProfileMap, ContextProfileMap, NoContextProfileMap);
521
522 if (auto EC = writeOneSection(Type: SecProfSummary, LayoutIdx: 0, ProfileMap))
523 return EC;
524 if (auto EC = writeOneSection(Type: SecNameTable, LayoutIdx: 1, ProfileMap))
525 return EC;
526 if (auto EC = writeOneSection(Type: SecLBRProfile, LayoutIdx: 3, ProfileMap: ContextProfileMap))
527 return EC;
528 if (auto EC = writeOneSection(Type: SecFuncOffsetTable, LayoutIdx: 2, ProfileMap: ContextProfileMap))
529 return EC;
530 // Mark the section to have no context. Note section flag needs to be set
531 // before writing the section.
532 addSectionFlag(SectionIdx: 5, Flag: SecCommonFlags::SecFlagFlat);
533 if (auto EC = writeOneSection(Type: SecLBRProfile, LayoutIdx: 5, ProfileMap: NoContextProfileMap))
534 return EC;
535 // Mark the section to have no context. Note section flag needs to be set
536 // before writing the section.
537 addSectionFlag(SectionIdx: 4, Flag: SecCommonFlags::SecFlagFlat);
538 if (auto EC = writeOneSection(Type: SecFuncOffsetTable, LayoutIdx: 4, ProfileMap: NoContextProfileMap))
539 return EC;
540 if (auto EC = writeOneSection(Type: SecProfileSymbolList, LayoutIdx: 6, ProfileMap))
541 return EC;
542 if (auto EC = writeOneSection(Type: SecFuncMetadata, LayoutIdx: 7, ProfileMap))
543 return EC;
544
545 return sampleprof_error::success;
546}
547
548std::error_code SampleProfileWriterExtBinary::writeSections(
549 const SampleProfileMap &ProfileMap) {
550 std::error_code EC;
551 if (SecLayout == DefaultLayout)
552 EC = writeDefaultLayout(ProfileMap);
553 else if (SecLayout == CtxSplitLayout)
554 EC = writeCtxSplitLayout(ProfileMap);
555 else
556 llvm_unreachable("Unsupported layout");
557 return EC;
558}
559
560/// Write samples to a text file.
561///
562/// Note: it may be tempting to implement this in terms of
563/// FunctionSamples::print(). Please don't. The dump functionality is intended
564/// for debugging and has no specified form.
565///
566/// The format used here is more structured and deliberate because
567/// it needs to be parsed by the SampleProfileReaderText class.
568std::error_code SampleProfileWriterText::writeSample(const FunctionSamples &S) {
569 auto &OS = *OutputStream;
570 if (FunctionSamples::ProfileIsCS)
571 OS << "[" << S.getContext().toString() << "]:" << S.getTotalSamples();
572 else
573 OS << S.getFunction() << ":" << S.getTotalSamples();
574
575 if (Indent == 0)
576 OS << ":" << S.getHeadSamples();
577 OS << "\n";
578 LineCount++;
579
580 SampleSorter<LineLocation, SampleRecord> SortedSamples(S.getBodySamples());
581 for (const auto &I : SortedSamples.get()) {
582 LineLocation Loc = I->first;
583 const SampleRecord &Sample = I->second;
584 OS.indent(NumSpaces: Indent + 1);
585 if (Loc.Discriminator == 0)
586 OS << Loc.LineOffset << ": ";
587 else
588 OS << Loc.LineOffset << "." << Loc.Discriminator << ": ";
589
590 OS << Sample.getSamples();
591
592 for (const auto &J : Sample.getSortedCallTargets())
593 OS << " " << J.first << ":" << J.second;
594 OS << "\n";
595 LineCount++;
596 }
597
598 SampleSorter<LineLocation, FunctionSamplesMap> SortedCallsiteSamples(
599 S.getCallsiteSamples());
600 Indent += 1;
601 for (const auto &I : SortedCallsiteSamples.get())
602 for (const auto &FS : I->second) {
603 LineLocation Loc = I->first;
604 const FunctionSamples &CalleeSamples = FS.second;
605 OS.indent(NumSpaces: Indent);
606 if (Loc.Discriminator == 0)
607 OS << Loc.LineOffset << ": ";
608 else
609 OS << Loc.LineOffset << "." << Loc.Discriminator << ": ";
610 if (std::error_code EC = writeSample(S: CalleeSamples))
611 return EC;
612 }
613 Indent -= 1;
614
615 if (FunctionSamples::ProfileIsProbeBased) {
616 OS.indent(NumSpaces: Indent + 1);
617 OS << "!CFGChecksum: " << S.getFunctionHash() << "\n";
618 LineCount++;
619 }
620
621 if (S.getContext().getAllAttributes()) {
622 OS.indent(NumSpaces: Indent + 1);
623 OS << "!Attributes: " << S.getContext().getAllAttributes() << "\n";
624 LineCount++;
625 }
626
627 return sampleprof_error::success;
628}
629
630std::error_code
631SampleProfileWriterBinary::writeContextIdx(const SampleContext &Context) {
632 assert(!Context.hasContext() && "cs profile is not supported");
633 return writeNameIdx(FName: Context.getFunction());
634}
635
636std::error_code SampleProfileWriterBinary::writeNameIdx(FunctionId FName) {
637 auto &NTable = getNameTable();
638 const auto &Ret = NTable.find(Key: FName);
639 if (Ret == NTable.end())
640 return sampleprof_error::truncated_name_table;
641 encodeULEB128(Value: Ret->second, OS&: *OutputStream);
642 return sampleprof_error::success;
643}
644
645void SampleProfileWriterBinary::addName(FunctionId FName) {
646 auto &NTable = getNameTable();
647 NTable.insert(KV: std::make_pair(x&: FName, y: 0));
648}
649
650void SampleProfileWriterBinary::addContext(const SampleContext &Context) {
651 addName(FName: Context.getFunction());
652}
653
654void SampleProfileWriterBinary::addNames(const FunctionSamples &S) {
655 // Add all the names in indirect call targets.
656 for (const auto &I : S.getBodySamples()) {
657 const SampleRecord &Sample = I.second;
658 for (const auto &J : Sample.getCallTargets())
659 addName(FName: J.first);
660 }
661
662 // Recursively add all the names for inlined callsites.
663 for (const auto &J : S.getCallsiteSamples())
664 for (const auto &FS : J.second) {
665 const FunctionSamples &CalleeSamples = FS.second;
666 addName(FName: CalleeSamples.getFunction());
667 addNames(S: CalleeSamples);
668 }
669}
670
671void SampleProfileWriterExtBinaryBase::addContext(
672 const SampleContext &Context) {
673 if (Context.hasContext()) {
674 for (auto &Callsite : Context.getContextFrames())
675 SampleProfileWriterBinary::addName(FName: Callsite.Func);
676 CSNameTable.insert(KV: std::make_pair(x: Context, y: 0));
677 } else {
678 SampleProfileWriterBinary::addName(FName: Context.getFunction());
679 }
680}
681
682void SampleProfileWriterBinary::stablizeNameTable(
683 MapVector<FunctionId, uint32_t> &NameTable, std::set<FunctionId> &V) {
684 // Sort the names to make NameTable deterministic.
685 for (const auto &I : NameTable)
686 V.insert(x: I.first);
687 int i = 0;
688 for (const FunctionId &N : V)
689 NameTable[N] = i++;
690}
691
692std::error_code SampleProfileWriterBinary::writeNameTable() {
693 auto &OS = *OutputStream;
694 std::set<FunctionId> V;
695 stablizeNameTable(NameTable, V);
696
697 // Write out the name table.
698 encodeULEB128(Value: NameTable.size(), OS);
699 for (auto N : V) {
700 OS << N;
701 encodeULEB128(Value: 0, OS);
702 }
703 return sampleprof_error::success;
704}
705
706std::error_code
707SampleProfileWriterBinary::writeMagicIdent(SampleProfileFormat Format) {
708 auto &OS = *OutputStream;
709 // Write file magic identifier.
710 encodeULEB128(Value: SPMagic(Format), OS);
711 encodeULEB128(Value: SPVersion(), OS);
712 return sampleprof_error::success;
713}
714
715std::error_code
716SampleProfileWriterBinary::writeHeader(const SampleProfileMap &ProfileMap) {
717 // When calling write on a different profile map, existing names should be
718 // cleared.
719 NameTable.clear();
720
721 writeMagicIdent(Format);
722
723 computeSummary(ProfileMap);
724 if (auto EC = writeSummary())
725 return EC;
726
727 // Generate the name table for all the functions referenced in the profile.
728 for (const auto &I : ProfileMap) {
729 addContext(Context: I.second.getContext());
730 addNames(S: I.second);
731 }
732
733 writeNameTable();
734 return sampleprof_error::success;
735}
736
737void SampleProfileWriterExtBinaryBase::setToCompressAllSections() {
738 for (auto &Entry : SectionHdrLayout)
739 addSecFlag(Entry, Flag: SecCommonFlags::SecFlagCompress);
740}
741
742void SampleProfileWriterExtBinaryBase::setToCompressSection(SecType Type) {
743 addSectionFlag(Type, Flag: SecCommonFlags::SecFlagCompress);
744}
745
746void SampleProfileWriterExtBinaryBase::allocSecHdrTable() {
747 support::endian::Writer Writer(*OutputStream, llvm::endianness::little);
748
749 Writer.write(Val: static_cast<uint64_t>(SectionHdrLayout.size()));
750 SecHdrTableOffset = OutputStream->tell();
751 for (uint32_t i = 0; i < SectionHdrLayout.size(); i++) {
752 Writer.write(Val: static_cast<uint64_t>(-1));
753 Writer.write(Val: static_cast<uint64_t>(-1));
754 Writer.write(Val: static_cast<uint64_t>(-1));
755 Writer.write(Val: static_cast<uint64_t>(-1));
756 }
757}
758
759std::error_code SampleProfileWriterExtBinaryBase::writeSecHdrTable() {
760 assert(SecHdrTable.size() == SectionHdrLayout.size() &&
761 "SecHdrTable entries doesn't match SectionHdrLayout");
762 SmallVector<uint32_t, 16> IndexMap(SecHdrTable.size(), -1);
763 for (uint32_t TableIdx = 0; TableIdx < SecHdrTable.size(); TableIdx++) {
764 IndexMap[SecHdrTable[TableIdx].LayoutIndex] = TableIdx;
765 }
766
767 // Write the section header table in the order specified in
768 // SectionHdrLayout. SectionHdrLayout specifies the sections
769 // order in which profile reader expect to read, so the section
770 // header table should be written in the order in SectionHdrLayout.
771 // Note that the section order in SecHdrTable may be different
772 // from the order in SectionHdrLayout, for example, SecFuncOffsetTable
773 // needs to be computed after SecLBRProfile (the order in SecHdrTable),
774 // but it needs to be read before SecLBRProfile (the order in
775 // SectionHdrLayout). So we use IndexMap above to switch the order.
776 support::endian::SeekableWriter Writer(
777 static_cast<raw_pwrite_stream &>(*OutputStream),
778 llvm::endianness::little);
779 for (uint32_t LayoutIdx = 0; LayoutIdx < SectionHdrLayout.size();
780 LayoutIdx++) {
781 assert(IndexMap[LayoutIdx] < SecHdrTable.size() &&
782 "Incorrect LayoutIdx in SecHdrTable");
783 auto Entry = SecHdrTable[IndexMap[LayoutIdx]];
784 Writer.pwrite(Val: static_cast<uint64_t>(Entry.Type),
785 Offset: SecHdrTableOffset + 4 * LayoutIdx * sizeof(uint64_t));
786 Writer.pwrite(Val: static_cast<uint64_t>(Entry.Flags),
787 Offset: SecHdrTableOffset + (4 * LayoutIdx + 1) * sizeof(uint64_t));
788 Writer.pwrite(Val: static_cast<uint64_t>(Entry.Offset),
789 Offset: SecHdrTableOffset + (4 * LayoutIdx + 2) * sizeof(uint64_t));
790 Writer.pwrite(Val: static_cast<uint64_t>(Entry.Size),
791 Offset: SecHdrTableOffset + (4 * LayoutIdx + 3) * sizeof(uint64_t));
792 }
793
794 return sampleprof_error::success;
795}
796
797std::error_code SampleProfileWriterExtBinaryBase::writeHeader(
798 const SampleProfileMap &ProfileMap) {
799 auto &OS = *OutputStream;
800 FileStart = OS.tell();
801 writeMagicIdent(Format);
802
803 allocSecHdrTable();
804 return sampleprof_error::success;
805}
806
807std::error_code SampleProfileWriterBinary::writeSummary() {
808 auto &OS = *OutputStream;
809 encodeULEB128(Value: Summary->getTotalCount(), OS);
810 encodeULEB128(Value: Summary->getMaxCount(), OS);
811 encodeULEB128(Value: Summary->getMaxFunctionCount(), OS);
812 encodeULEB128(Value: Summary->getNumCounts(), OS);
813 encodeULEB128(Value: Summary->getNumFunctions(), OS);
814 ArrayRef<ProfileSummaryEntry> Entries = Summary->getDetailedSummary();
815 encodeULEB128(Value: Entries.size(), OS);
816 for (auto Entry : Entries) {
817 encodeULEB128(Value: Entry.Cutoff, OS);
818 encodeULEB128(Value: Entry.MinCount, OS);
819 encodeULEB128(Value: Entry.NumCounts, OS);
820 }
821 return sampleprof_error::success;
822}
823std::error_code SampleProfileWriterBinary::writeBody(const FunctionSamples &S) {
824 auto &OS = *OutputStream;
825 if (std::error_code EC = writeContextIdx(Context: S.getContext()))
826 return EC;
827
828 encodeULEB128(Value: S.getTotalSamples(), OS);
829
830 // Emit all the body samples.
831 encodeULEB128(Value: S.getBodySamples().size(), OS);
832 for (const auto &I : S.getBodySamples()) {
833 LineLocation Loc = I.first;
834 const SampleRecord &Sample = I.second;
835 encodeULEB128(Value: Loc.LineOffset, OS);
836 encodeULEB128(Value: Loc.Discriminator, OS);
837 encodeULEB128(Value: Sample.getSamples(), OS);
838 encodeULEB128(Value: Sample.getCallTargets().size(), OS);
839 for (const auto &J : Sample.getSortedCallTargets()) {
840 FunctionId Callee = J.first;
841 uint64_t CalleeSamples = J.second;
842 if (std::error_code EC = writeNameIdx(FName: Callee))
843 return EC;
844 encodeULEB128(Value: CalleeSamples, OS);
845 }
846 }
847
848 // Recursively emit all the callsite samples.
849 uint64_t NumCallsites = 0;
850 for (const auto &J : S.getCallsiteSamples())
851 NumCallsites += J.second.size();
852 encodeULEB128(Value: NumCallsites, OS);
853 for (const auto &J : S.getCallsiteSamples())
854 for (const auto &FS : J.second) {
855 LineLocation Loc = J.first;
856 const FunctionSamples &CalleeSamples = FS.second;
857 encodeULEB128(Value: Loc.LineOffset, OS);
858 encodeULEB128(Value: Loc.Discriminator, OS);
859 if (std::error_code EC = writeBody(S: CalleeSamples))
860 return EC;
861 }
862
863 return sampleprof_error::success;
864}
865
866/// Write samples of a top-level function to a binary file.
867///
868/// \returns true if the samples were written successfully, false otherwise.
869std::error_code
870SampleProfileWriterBinary::writeSample(const FunctionSamples &S) {
871 encodeULEB128(Value: S.getHeadSamples(), OS&: *OutputStream);
872 return writeBody(S);
873}
874
875/// Create a sample profile file writer based on the specified format.
876///
877/// \param Filename The file to create.
878///
879/// \param Format Encoding format for the profile file.
880///
881/// \returns an error code indicating the status of the created writer.
882ErrorOr<std::unique_ptr<SampleProfileWriter>>
883SampleProfileWriter::create(StringRef Filename, SampleProfileFormat Format) {
884 std::error_code EC;
885 std::unique_ptr<raw_ostream> OS;
886 if (Format == SPF_Binary || Format == SPF_Ext_Binary)
887 OS.reset(p: new raw_fd_ostream(Filename, EC, sys::fs::OF_None));
888 else
889 OS.reset(p: new raw_fd_ostream(Filename, EC, sys::fs::OF_TextWithCRLF));
890 if (EC)
891 return EC;
892
893 return create(OS, Format);
894}
895
896/// Create a sample profile stream writer based on the specified format.
897///
898/// \param OS The output stream to store the profile data to.
899///
900/// \param Format Encoding format for the profile file.
901///
902/// \returns an error code indicating the status of the created writer.
903ErrorOr<std::unique_ptr<SampleProfileWriter>>
904SampleProfileWriter::create(std::unique_ptr<raw_ostream> &OS,
905 SampleProfileFormat Format) {
906 std::error_code EC;
907 std::unique_ptr<SampleProfileWriter> Writer;
908
909 // Currently only Text and Extended Binary format are supported for CSSPGO.
910 if ((FunctionSamples::ProfileIsCS || FunctionSamples::ProfileIsProbeBased) &&
911 Format == SPF_Binary)
912 return sampleprof_error::unsupported_writing_format;
913
914 if (Format == SPF_Binary)
915 Writer.reset(p: new SampleProfileWriterRawBinary(OS));
916 else if (Format == SPF_Ext_Binary)
917 Writer.reset(p: new SampleProfileWriterExtBinary(OS));
918 else if (Format == SPF_Text)
919 Writer.reset(p: new SampleProfileWriterText(OS));
920 else if (Format == SPF_GCC)
921 EC = sampleprof_error::unsupported_writing_format;
922 else
923 EC = sampleprof_error::unrecognized_format;
924
925 if (EC)
926 return EC;
927
928 Writer->Format = Format;
929 return std::move(Writer);
930}
931
932void SampleProfileWriter::computeSummary(const SampleProfileMap &ProfileMap) {
933 SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs);
934 Summary = Builder.computeSummaryForProfiles(Profiles: ProfileMap);
935}
936