| 1 | //===- SampleProfWriter.cpp - Write LLVM sample profile data --------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file implements the class that writes LLVM sample profiles. It |
| 10 | // supports two file formats: text and binary. The textual representation |
| 11 | // is useful for debugging and testing purposes. The binary representation |
| 12 | // is more compact, resulting in smaller file sizes. However, they can |
| 13 | // both be used interchangeably. |
| 14 | // |
| 15 | // See lib/ProfileData/SampleProfReader.cpp for documentation on each of the |
| 16 | // supported formats. |
| 17 | // |
| 18 | //===----------------------------------------------------------------------===// |
| 19 | |
| 20 | #include "llvm/ProfileData/SampleProfWriter.h" |
| 21 | #include "llvm/ADT/Eytzinger.h" |
| 22 | #include "llvm/ADT/StringRef.h" |
| 23 | #include "llvm/ProfileData/ProfileCommon.h" |
| 24 | #include "llvm/ProfileData/SampleProf.h" |
| 25 | #include "llvm/Support/Compression.h" |
| 26 | #include "llvm/Support/EndianStream.h" |
| 27 | #include "llvm/Support/ErrorOr.h" |
| 28 | #include "llvm/Support/FileSystem.h" |
| 29 | #include "llvm/Support/LEB128.h" |
| 30 | #include "llvm/Support/MD5.h" |
| 31 | #include "llvm/Support/raw_ostream.h" |
| 32 | #include <array> |
| 33 | #include <cmath> |
| 34 | #include <cstdint> |
| 35 | #include <memory> |
| 36 | #include <system_error> |
| 37 | #include <utility> |
| 38 | #include <vector> |
| 39 | |
| 40 | #define DEBUG_TYPE "llvm-profdata" |
| 41 | |
| 42 | using namespace llvm; |
| 43 | using namespace sampleprof; |
| 44 | |
| 45 | // To begin with, make this option off by default. |
| 46 | static cl::opt<bool> ExtBinaryWriteVTableTypeProf( |
| 47 | "extbinary-write-vtable-type-prof" , cl::init(Val: false), cl::Hidden, |
| 48 | cl::desc("Write vtable type profile in ext-binary sample profile writer" )); |
| 49 | |
| 50 | static cl::opt<uint64_t> RequestedVersion( |
| 51 | "sample-profile-format-version" , cl::init(Val: DefaultVersion), cl::Hidden, |
| 52 | cl::desc("Format version to write for extensible binary profiles" )); |
| 53 | |
| 54 | namespace llvm { |
| 55 | namespace support { |
| 56 | namespace endian { |
| 57 | namespace { |
| 58 | |
| 59 | // Adapter class to llvm::support::endian::Writer for pwrite(). |
| 60 | struct SeekableWriter { |
| 61 | raw_pwrite_stream &OS; |
| 62 | endianness Endian; |
| 63 | SeekableWriter(raw_pwrite_stream &OS, endianness Endian) |
| 64 | : OS(OS), Endian(Endian) {} |
| 65 | |
| 66 | template <typename ValueType> void pwrite(ValueType Val, size_t Offset) { |
| 67 | std::string StringBuf; |
| 68 | raw_string_ostream SStream(StringBuf); |
| 69 | Writer(SStream, Endian).write(Val); |
| 70 | OS.pwrite(Ptr: StringBuf.data(), Size: StringBuf.size(), Offset); |
| 71 | } |
| 72 | }; |
| 73 | |
| 74 | } // namespace |
| 75 | } // namespace endian |
| 76 | } // namespace support |
| 77 | } // namespace llvm |
| 78 | |
| 79 | DefaultFunctionPruningStrategy::DefaultFunctionPruningStrategy( |
| 80 | SampleProfileMap &ProfileMap, size_t OutputSizeLimit) |
| 81 | : FunctionPruningStrategy(ProfileMap, OutputSizeLimit) { |
| 82 | sortFuncProfiles(ProfileMap, SortedProfiles&: SortedFunctions); |
| 83 | } |
| 84 | |
| 85 | void DefaultFunctionPruningStrategy::Erase(size_t CurrentOutputSize) { |
| 86 | double D = (double)OutputSizeLimit / CurrentOutputSize; |
| 87 | size_t NewSize = (size_t)round(x: ProfileMap.size() * D * D); |
| 88 | size_t NumToRemove = ProfileMap.size() - NewSize; |
| 89 | if (NumToRemove < 1) |
| 90 | NumToRemove = 1; |
| 91 | |
| 92 | assert(NumToRemove <= SortedFunctions.size()); |
| 93 | for (const NameFunctionSamples &E : |
| 94 | llvm::drop_begin(RangeOrContainer&: SortedFunctions, N: SortedFunctions.size() - NumToRemove)) |
| 95 | ProfileMap.erase(Key: E.first); |
| 96 | SortedFunctions.resize(new_size: SortedFunctions.size() - NumToRemove); |
| 97 | } |
| 98 | |
| 99 | std::error_code SampleProfileWriter::writeWithSizeLimitInternal( |
| 100 | SampleProfileMap &ProfileMap, size_t OutputSizeLimit, |
| 101 | FunctionPruningStrategy *Strategy) { |
| 102 | if (OutputSizeLimit == 0) |
| 103 | return write(ProfileMap); |
| 104 | |
| 105 | size_t OriginalFunctionCount = ProfileMap.size(); |
| 106 | |
| 107 | std::unique_ptr<raw_ostream> OriginalOutputStream; |
| 108 | OutputStream.swap(u&: OriginalOutputStream); |
| 109 | |
| 110 | size_t IterationCount = 0; |
| 111 | size_t TotalSize; |
| 112 | |
| 113 | SmallVector<char> StringBuffer; |
| 114 | do { |
| 115 | StringBuffer.clear(); |
| 116 | OutputStream.reset(p: new raw_svector_ostream(StringBuffer)); |
| 117 | if (std::error_code EC = write(ProfileMap)) |
| 118 | return EC; |
| 119 | |
| 120 | TotalSize = StringBuffer.size(); |
| 121 | // On Windows every "\n" is actually written as "\r\n" to disk but not to |
| 122 | // memory buffer, this difference should be added when considering the total |
| 123 | // output size. |
| 124 | #ifdef _WIN32 |
| 125 | if (Format == SPF_Text) |
| 126 | TotalSize += LineCount; |
| 127 | #endif |
| 128 | if (TotalSize <= OutputSizeLimit) |
| 129 | break; |
| 130 | |
| 131 | Strategy->Erase(CurrentOutputSize: TotalSize); |
| 132 | IterationCount++; |
| 133 | } while (ProfileMap.size() != 0); |
| 134 | |
| 135 | if (ProfileMap.size() == 0) |
| 136 | return sampleprof_error::too_large; |
| 137 | |
| 138 | OutputStream.swap(u&: OriginalOutputStream); |
| 139 | OutputStream->write(Ptr: StringBuffer.data(), Size: StringBuffer.size()); |
| 140 | LLVM_DEBUG(dbgs() << "Profile originally has " << OriginalFunctionCount |
| 141 | << " functions, reduced to " << ProfileMap.size() << " in " |
| 142 | << IterationCount << " iterations\n" ); |
| 143 | // Silence warning on Release build. |
| 144 | (void)OriginalFunctionCount; |
| 145 | (void)IterationCount; |
| 146 | return sampleprof_error::success; |
| 147 | } |
| 148 | |
| 149 | std::error_code |
| 150 | SampleProfileWriter::writeFuncProfiles(const SampleProfileMap &ProfileMap) { |
| 151 | std::vector<NameFunctionSamples> V; |
| 152 | sortFuncProfiles(ProfileMap, SortedProfiles&: V); |
| 153 | for (const auto &I : V) { |
| 154 | if (std::error_code EC = writeSample(S: *I.second)) |
| 155 | return EC; |
| 156 | } |
| 157 | return sampleprof_error::success; |
| 158 | } |
| 159 | |
| 160 | std::error_code SampleProfileWriter::write(const SampleProfileMap &ProfileMap) { |
| 161 | if (std::error_code EC = writeHeader(ProfileMap)) |
| 162 | return EC; |
| 163 | |
| 164 | if (std::error_code EC = writeFuncProfiles(ProfileMap)) |
| 165 | return EC; |
| 166 | |
| 167 | return sampleprof_error::success; |
| 168 | } |
| 169 | |
| 170 | /// Return the current position and prepare to use it as the start |
| 171 | /// position of a section given the section type \p Type and its position |
| 172 | /// \p LayoutIdx in SectionHdrLayout. |
| 173 | uint64_t |
| 174 | SampleProfileWriterExtBinaryBase::markSectionStart(SecType Type, |
| 175 | uint32_t LayoutIdx) { |
| 176 | uint64_t SectionStart = OutputStream->tell(); |
| 177 | assert(LayoutIdx < SectionHdrLayout.size() && "LayoutIdx out of range" ); |
| 178 | const auto &Entry = SectionHdrLayout[LayoutIdx]; |
| 179 | assert(Entry.Type == Type && "Unexpected section type" ); |
| 180 | // Use LocalBuf as a temporary output for writing data. |
| 181 | if (hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagCompress)) |
| 182 | LocalBufStream.swap(u&: OutputStream); |
| 183 | return SectionStart; |
| 184 | } |
| 185 | |
| 186 | std::error_code SampleProfileWriterExtBinaryBase::compressAndOutput() { |
| 187 | if (!llvm::compression::zlib::isAvailable()) |
| 188 | return sampleprof_error::zlib_unavailable; |
| 189 | std::string &UncompressedStrings = |
| 190 | static_cast<raw_string_ostream *>(LocalBufStream.get())->str(); |
| 191 | if (UncompressedStrings.empty()) |
| 192 | return sampleprof_error::success; |
| 193 | auto &OS = *OutputStream; |
| 194 | SmallVector<uint8_t, 128> CompressedStrings; |
| 195 | compression::zlib::compress(Input: arrayRefFromStringRef(Input: UncompressedStrings), |
| 196 | CompressedBuffer&: CompressedStrings, |
| 197 | Level: compression::zlib::BestSizeCompression); |
| 198 | encodeULEB128(Value: UncompressedStrings.size(), OS); |
| 199 | encodeULEB128(Value: CompressedStrings.size(), OS); |
| 200 | OS << toStringRef(Input: CompressedStrings); |
| 201 | UncompressedStrings.clear(); |
| 202 | return sampleprof_error::success; |
| 203 | } |
| 204 | |
| 205 | /// Add a new section into section header table given the section type |
| 206 | /// \p Type, its position \p LayoutIdx in SectionHdrLayout and the |
| 207 | /// location \p SectionStart where the section should be written to. |
| 208 | std::error_code SampleProfileWriterExtBinaryBase::addNewSection( |
| 209 | SecType Type, uint32_t LayoutIdx, uint64_t SectionStart) { |
| 210 | assert(LayoutIdx < SectionHdrLayout.size() && "LayoutIdx out of range" ); |
| 211 | const auto &Entry = SectionHdrLayout[LayoutIdx]; |
| 212 | assert(Entry.Type == Type && "Unexpected section type" ); |
| 213 | if (hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagCompress)) { |
| 214 | LocalBufStream.swap(u&: OutputStream); |
| 215 | if (std::error_code EC = compressAndOutput()) |
| 216 | return EC; |
| 217 | } |
| 218 | SecHdrTable.push_back(x: {.Type: Type, .Flags: Entry.Flags, .Offset: SectionStart - FileStart, |
| 219 | .Size: OutputStream->tell() - SectionStart, .LayoutIndex: LayoutIdx}); |
| 220 | return sampleprof_error::success; |
| 221 | } |
| 222 | |
| 223 | std::error_code |
| 224 | SampleProfileWriterExtBinaryBase::write(const SampleProfileMap &ProfileMap) { |
| 225 | // When calling write on a different profile map, existing states should be |
| 226 | // cleared. |
| 227 | NameTable.clear(); |
| 228 | CSNameTable.clear(); |
| 229 | SecHdrTable.clear(); |
| 230 | |
| 231 | if (std::error_code EC = writeHeader(ProfileMap)) |
| 232 | return EC; |
| 233 | |
| 234 | std::string LocalBuf; |
| 235 | LocalBufStream = std::make_unique<raw_string_ostream>(args&: LocalBuf); |
| 236 | if (std::error_code EC = writeSections(ProfileMap)) |
| 237 | return EC; |
| 238 | |
| 239 | if (std::error_code EC = writeSecHdrTable()) |
| 240 | return EC; |
| 241 | |
| 242 | return sampleprof_error::success; |
| 243 | } |
| 244 | |
| 245 | std::error_code SampleProfileWriterExtBinaryBase::writeContextIdx( |
| 246 | const SampleContext &Context) { |
| 247 | if (Context.hasContext()) |
| 248 | return writeCSNameIdx(Context); |
| 249 | else |
| 250 | return SampleProfileWriterBinary::writeNameIdx(FName: Context.getFunction()); |
| 251 | } |
| 252 | |
| 253 | std::error_code |
| 254 | SampleProfileWriterExtBinaryBase::writeCSNameIdx(const SampleContext &Context) { |
| 255 | const auto &Ret = CSNameTable.find(Key: Context); |
| 256 | if (Ret == CSNameTable.end()) |
| 257 | return sampleprof_error::truncated_name_table; |
| 258 | encodeULEB128(Value: Ret->second, OS&: *OutputStream); |
| 259 | return sampleprof_error::success; |
| 260 | } |
| 261 | |
| 262 | std::error_code |
| 263 | SampleProfileWriterExtBinaryBase::writeSample(const FunctionSamples &S) { |
| 264 | uint64_t Offset = OutputStream->tell(); |
| 265 | auto &Context = S.getContext(); |
| 266 | FuncOffsetTable[Context] = Offset - SecLBRProfileStart; |
| 267 | encodeULEB128(Value: S.getHeadSamples(), OS&: *OutputStream); |
| 268 | return writeBody(S); |
| 269 | } |
| 270 | |
| 271 | std::error_code |
| 272 | SampleProfileWriterExtBinaryBase::writeFuncOffsetTable(bool IsNested) { |
| 273 | if (UseMD5IndexedTables) { |
| 274 | // Eytzinger layout requires MD5 representation and does not support |
| 275 | // multi-context Context-Sensitive profiles. |
| 276 | if (!UseMD5 || FunctionSamples::ProfileIsCS) |
| 277 | return sampleprof_error::unsupported_writing_format; |
| 278 | return writeEytzingerFuncOffsetTable(IsNested); |
| 279 | } |
| 280 | return writeLegacyFuncOffsetTable(); |
| 281 | } |
| 282 | |
| 283 | std::error_code |
| 284 | SampleProfileWriterExtBinaryBase::writeEytzingerFuncOffsetTable(bool IsNested) { |
| 285 | assert((NumNested + NumFlat > 0 || FuncOffsetTable.empty()) && |
| 286 | "SecNameTable must be written before SecFuncOffsetTable to establish " |
| 287 | "Eytzinger indices!" ); |
| 288 | |
| 289 | size_t SpanSize = IsNested ? NumNested : NumFlat; |
| 290 | size_t BaseIdx = IsNested ? 0 : NumNested; |
| 291 | |
| 292 | std::vector<support::ulittle32_t> FuncOffsets( |
| 293 | SpanSize, support::ulittle32_t(UINT32_MAX)); |
| 294 | |
| 295 | // Populate the function offset array parallel to the Eytzinger span. |
| 296 | for (const auto &[Context, RelativeOffset] : FuncOffsetTable) { |
| 297 | if (RelativeOffset >= UINT32_MAX) |
| 298 | return sampleprof_error::too_large; |
| 299 | |
| 300 | FunctionId FId = Context.getFunction(); |
| 301 | auto It = NameTable.find(Key: FId); |
| 302 | if (It == NameTable.end()) |
| 303 | continue; |
| 304 | |
| 305 | size_t GlobalIdx = It->second; |
| 306 | if (GlobalIdx < BaseIdx || (GlobalIdx - BaseIdx) >= SpanSize) |
| 307 | continue; |
| 308 | |
| 309 | size_t LocalIdx = GlobalIdx - BaseIdx; |
| 310 | assert( |
| 311 | FuncOffsets[LocalIdx] == UINT32_MAX && |
| 312 | "Function offset slot already populated; duplicate GUID or collision!" ); |
| 313 | FuncOffsets[LocalIdx] = static_cast<uint32_t>(RelativeOffset); |
| 314 | } |
| 315 | |
| 316 | assert(!llvm::is_contained(FuncOffsets, support::ulittle32_t(UINT32_MAX)) && |
| 317 | "Unpopulated slot in Eytzinger function offset array!" ); |
| 318 | |
| 319 | OutputStream->write(Ptr: reinterpret_cast<const char *>(FuncOffsets.data()), |
| 320 | Size: SpanSize * sizeof(support::ulittle32_t)); |
| 321 | addSectionFlag(Type: SecFuncOffsetTable, Flag: SecFuncOffsetFlags::SecFlagEytzinger); |
| 322 | FuncOffsetTable.clear(); |
| 323 | return sampleprof_error::success; |
| 324 | } |
| 325 | |
| 326 | std::error_code SampleProfileWriterExtBinaryBase::writeLegacyFuncOffsetTable() { |
| 327 | auto &OS = *OutputStream; |
| 328 | |
| 329 | // Write out the table size. |
| 330 | encodeULEB128(Value: FuncOffsetTable.size(), OS); |
| 331 | |
| 332 | // Write out FuncOffsetTable. |
| 333 | auto WriteItem = [&](const SampleContext &Context, uint64_t Offset) { |
| 334 | if (std::error_code EC = writeContextIdx(Context)) |
| 335 | return EC; |
| 336 | encodeULEB128(Value: Offset, OS); |
| 337 | return (std::error_code)sampleprof_error::success; |
| 338 | }; |
| 339 | |
| 340 | if (FunctionSamples::ProfileIsCS) { |
| 341 | // Sort the contexts before writing them out. This is to help fast load all |
| 342 | // context profiles for a function as well as their callee contexts which |
| 343 | // can help profile-guided importing for ThinLTO. |
| 344 | std::map<SampleContext, uint64_t> OrderedFuncOffsetTable( |
| 345 | FuncOffsetTable.begin(), FuncOffsetTable.end()); |
| 346 | for (const auto &Entry : OrderedFuncOffsetTable) { |
| 347 | if (std::error_code EC = WriteItem(Entry.first, Entry.second)) |
| 348 | return EC; |
| 349 | } |
| 350 | addSectionFlag(Type: SecFuncOffsetTable, Flag: SecFuncOffsetFlags::SecFlagOrdered); |
| 351 | } else { |
| 352 | for (const auto &Entry : FuncOffsetTable) { |
| 353 | if (std::error_code EC = WriteItem(Entry.first, Entry.second)) |
| 354 | return EC; |
| 355 | } |
| 356 | } |
| 357 | |
| 358 | FuncOffsetTable.clear(); |
| 359 | return sampleprof_error::success; |
| 360 | } |
| 361 | |
| 362 | std::error_code SampleProfileWriterExtBinaryBase::writeFuncMetadata( |
| 363 | const FunctionSamples &FunctionProfile) { |
| 364 | auto &OS = *OutputStream; |
| 365 | if (std::error_code EC = writeContextIdx(Context: FunctionProfile.getContext())) |
| 366 | return EC; |
| 367 | |
| 368 | if (FunctionSamples::ProfileIsProbeBased) |
| 369 | encodeULEB128(Value: FunctionProfile.getFunctionHash(), OS); |
| 370 | if (FunctionSamples::ProfileIsCS || FunctionSamples::ProfileIsPreInlined) { |
| 371 | encodeULEB128(Value: FunctionProfile.getContext().getAllAttributes(), OS); |
| 372 | } |
| 373 | |
| 374 | if (!FunctionSamples::ProfileIsCS) { |
| 375 | // Recursively emit attributes for all callee samples. |
| 376 | uint64_t NumCallsites = 0; |
| 377 | for (const auto &J : FunctionProfile.getCallsiteSamples()) |
| 378 | NumCallsites += J.second.size(); |
| 379 | encodeULEB128(Value: NumCallsites, OS); |
| 380 | for (const auto &J : FunctionProfile.getCallsiteSamples()) { |
| 381 | for (const auto &FS : J.second) { |
| 382 | LineLocation Loc = J.first; |
| 383 | encodeULEB128(Value: Loc.LineOffset, OS); |
| 384 | encodeULEB128(Value: Loc.Discriminator, OS); |
| 385 | if (std::error_code EC = writeFuncMetadata(FunctionProfile: FS.second)) |
| 386 | return EC; |
| 387 | } |
| 388 | } |
| 389 | } |
| 390 | |
| 391 | return sampleprof_error::success; |
| 392 | } |
| 393 | |
| 394 | std::error_code SampleProfileWriterExtBinaryBase::writeFuncMetadata( |
| 395 | const SampleProfileMap &Profiles) { |
| 396 | if (!FunctionSamples::ProfileIsProbeBased && !FunctionSamples::ProfileIsCS && |
| 397 | !FunctionSamples::ProfileIsPreInlined) |
| 398 | return sampleprof_error::success; |
| 399 | for (const auto &Entry : Profiles) { |
| 400 | if (std::error_code EC = writeFuncMetadata(FunctionProfile: Entry.second)) |
| 401 | return EC; |
| 402 | } |
| 403 | return sampleprof_error::success; |
| 404 | } |
| 405 | |
| 406 | template <class KeyT, class ValT> |
| 407 | static SmallVector<std::pair<KeyT, ValT> *, 0> |
| 408 | stabilizeTable(MapVector<KeyT, ValT> &Table) { |
| 409 | SmallVector<std::pair<KeyT, ValT> *, 0> Entries( |
| 410 | llvm::make_pointer_range(Table)); |
| 411 | |
| 412 | llvm::sort(Entries, |
| 413 | [](const auto *L, const auto *R) { return L->first < R->first; }); |
| 414 | |
| 415 | for (const auto &[I, Entry] : llvm::enumerate(Entries)) |
| 416 | Entry->second = I; |
| 417 | |
| 418 | return Entries; |
| 419 | } |
| 420 | |
| 421 | std::error_code SampleProfileWriterExtBinaryBase::writeNameTable() { |
| 422 | if (!UseMD5) |
| 423 | return SampleProfileWriterBinary::writeNameTable(); |
| 424 | |
| 425 | auto &OS = *OutputStream; |
| 426 | |
| 427 | // Write out the MD5 name table. We wrote unencoded MD5 so reader can |
| 428 | // retrieve the name using the name index without having to read the |
| 429 | // whole name table. |
| 430 | encodeULEB128(Value: NameTable.size(), OS); |
| 431 | support::endian::Writer Writer(OS, llvm::endianness::little); |
| 432 | for (const auto *Entry : stabilizeTable(Table&: NameTable)) |
| 433 | Writer.write(Val: Entry->first.getHashCode()); |
| 434 | return sampleprof_error::success; |
| 435 | } |
| 436 | |
| 437 | std::error_code SampleProfileWriterExtBinaryBase::writeNameTableSection( |
| 438 | const SampleProfileMap &ProfileMap) { |
| 439 | for (const auto &I : ProfileMap) { |
| 440 | addContext(Context: I.second.getContext()); |
| 441 | addNames(S: I.second); |
| 442 | } |
| 443 | |
| 444 | // If NameTable contains ".__uniq." suffix, set SecFlagUniqSuffix flag |
| 445 | // so compiler won't strip the suffix during profile matching after |
| 446 | // seeing the flag in the profile. |
| 447 | // Original names are unavailable if using MD5, so this option has no use. |
| 448 | if (!UseMD5) { |
| 449 | for (const auto &I : NameTable) { |
| 450 | if (I.first.stringRef().contains(Other: FunctionSamples::UniqSuffix)) { |
| 451 | addSectionFlag(Type: SecNameTable, Flag: SecNameTableFlags::SecFlagUniqSuffix); |
| 452 | break; |
| 453 | } |
| 454 | } |
| 455 | } |
| 456 | |
| 457 | if (UseMD5 && UseMD5IndexedTables) { |
| 458 | // Eytzinger name tables do not support CSSPGO profiles |
| 459 | // (FunctionSamples::ProfileIsCS). |
| 460 | if (FunctionSamples::ProfileIsCS) |
| 461 | return sampleprof_error::unsupported_writing_format; |
| 462 | if (auto EC = writeEytzingerNameTableSection(ProfileMap)) |
| 463 | return EC; |
| 464 | return sampleprof_error::success; |
| 465 | } |
| 466 | |
| 467 | if (auto EC = writeNameTable()) |
| 468 | return EC; |
| 469 | return sampleprof_error::success; |
| 470 | } |
| 471 | |
| 472 | namespace { |
| 473 | |
| 474 | // Helper class to construct and write the SecNameTable section in Eytzinger |
| 475 | // layout for ExtBinary MD5 profiles. |
| 476 | // |
| 477 | // The on-disk layout of the Eytzinger name table section consists of symbol |
| 478 | // counts followed by three contiguous Eytzinger hash arrays: |
| 479 | // - ULEB128 count of Nested top-level profile symbol keys |
| 480 | // - ULEB128 count of Flat top-level profile symbol keys |
| 481 | // - ULEB128 count of Inlinee and auxiliary profile symbol keys |
| 482 | // - Array of 64-bit little-endian MD5 hash keys for Nested profiles in |
| 483 | // Eytzinger order |
| 484 | // - Array of 64-bit little-endian MD5 hash keys for Flat profiles in Eytzinger |
| 485 | // order |
| 486 | // - Array of 64-bit little-endian MD5 hash keys for Inlinees in Eytzinger order |
| 487 | class EytzingerNameTable { |
| 488 | using TableT = llvm::EytzingerTable<support::ulittle64_t>; |
| 489 | std::array<TableT, static_cast<size_t>(EytzingerSpan::NumSpans)> Spans; |
| 490 | |
| 491 | public: |
| 492 | EytzingerNameTable(std::vector<support::ulittle64_t> NestedKeys, |
| 493 | std::vector<support::ulittle64_t> FlatKeys, |
| 494 | std::vector<support::ulittle64_t> InlineeKeys) |
| 495 | : Spans{TableT::create(Keys: std::move(NestedKeys)), |
| 496 | TableT::create(Keys: std::move(FlatKeys)), |
| 497 | TableT::create(Keys: std::move(InlineeKeys))} {} |
| 498 | |
| 499 | // Find the global index of GUID across the three Eytzinger table spans. |
| 500 | uint64_t findGlobalIdx(uint64_t GUID) const { |
| 501 | uint64_t BaseIdx = 0; |
| 502 | for (const auto &Table : Spans) { |
| 503 | if (std::optional<size_t> LocalIdx = Table.findIndex(Target: GUID)) |
| 504 | return BaseIdx + *LocalIdx; |
| 505 | BaseIdx += Table.size(); |
| 506 | } |
| 507 | llvm_unreachable("Symbol in NameTable missing from Eytzinger spans" ); |
| 508 | } |
| 509 | |
| 510 | void write(raw_ostream &OS) const { |
| 511 | for (const auto &Table : Spans) |
| 512 | encodeULEB128(Value: uint64_t(Table.size()), OS); |
| 513 | for (const auto &Table : Spans) |
| 514 | OS.write(Ptr: reinterpret_cast<const char *>(Table.data()), |
| 515 | Size: Table.size() * sizeof(support::ulittle64_t)); |
| 516 | } |
| 517 | |
| 518 | size_t size(EytzingerSpan S) const { |
| 519 | return Spans[static_cast<size_t>(S)].size(); |
| 520 | } |
| 521 | }; |
| 522 | |
| 523 | } // end anonymous namespace |
| 524 | |
| 525 | std::error_code |
| 526 | SampleProfileWriterExtBinaryBase::writeEytzingerNameTableSection( |
| 527 | const SampleProfileMap &ProfileMap) { |
| 528 | DenseSet<uint64_t> TopLevelGUIDs; |
| 529 | std::vector<support::ulittle64_t> NestedKeys, FlatKeys, InlineeKeys; |
| 530 | |
| 531 | // Collect top-level Nested and Flat keys directly from ProfileMap. |
| 532 | for (const auto &I : ProfileMap) { |
| 533 | const SampleContext &Ctx = I.second.getContext(); |
| 534 | uint64_t GUID = Ctx.getFunction().getHashCode(); |
| 535 | if (TopLevelGUIDs.insert(V: GUID).second) { |
| 536 | // In single-table default layouts, unify all top-level symbols in the |
| 537 | // Nested partition so they match the single unflagged function offset |
| 538 | // table. |
| 539 | if (SecLayout != CtxSplitLayout || I.second.hasCallsiteSamples()) |
| 540 | NestedKeys.emplace_back(args&: GUID); |
| 541 | else |
| 542 | FlatKeys.emplace_back(args&: GUID); |
| 543 | } |
| 544 | } |
| 545 | |
| 546 | // Collect remaining non-top-level symbols (inlinees, targets, vtables) from |
| 547 | // NameTable. |
| 548 | for (const auto &Entry : NameTable) { |
| 549 | uint64_t GUID = Entry.first.getHashCode(); |
| 550 | if (!TopLevelGUIDs.contains(V: GUID)) |
| 551 | InlineeKeys.emplace_back(args&: GUID); |
| 552 | } |
| 553 | |
| 554 | EytzingerNameTable Tables(std::move(NestedKeys), std::move(FlatKeys), |
| 555 | std::move(InlineeKeys)); |
| 556 | |
| 557 | // Assign each symbol its corresponding index in the Eytzinger layout. |
| 558 | for (auto &[FId, Idx] : NameTable) |
| 559 | Idx = Tables.findGlobalIdx(GUID: FId.getHashCode()); |
| 560 | |
| 561 | Tables.write(OS&: *OutputStream); |
| 562 | NumNested = Tables.size(S: EytzingerSpan::Nested); |
| 563 | NumFlat = Tables.size(S: EytzingerSpan::Flat); |
| 564 | |
| 565 | return sampleprof_error::success; |
| 566 | } |
| 567 | |
| 568 | std::error_code SampleProfileWriterExtBinaryBase::writeCSNameTableSection() { |
| 569 | auto &OS = *OutputStream; |
| 570 | encodeULEB128(Value: CSNameTable.size(), OS); |
| 571 | support::endian::Writer Writer(OS, llvm::endianness::little); |
| 572 | for (const auto *Entry : stabilizeTable(Table&: CSNameTable)) { |
| 573 | auto Frames = Entry->first.getContextFrames(); |
| 574 | encodeULEB128(Value: Frames.size(), OS); |
| 575 | for (auto &Callsite : Frames) { |
| 576 | if (std::error_code EC = writeNameIdx(FName: Callsite.Func)) |
| 577 | return EC; |
| 578 | encodeULEB128(Value: Callsite.Location.LineOffset, OS); |
| 579 | encodeULEB128(Value: Callsite.Location.Discriminator, OS); |
| 580 | } |
| 581 | } |
| 582 | |
| 583 | return sampleprof_error::success; |
| 584 | } |
| 585 | |
| 586 | std::error_code |
| 587 | SampleProfileWriterExtBinaryBase::writeProfileSymbolListSection() { |
| 588 | if (UseMD5ProfSymList) |
| 589 | return writeMD5ProfileSymbolListSection(); |
| 590 | return writeStringBasedProfileSymbolListSection(); |
| 591 | } |
| 592 | |
| 593 | std::error_code |
| 594 | SampleProfileWriterExtBinaryBase::writeStringBasedProfileSymbolListSection() { |
| 595 | assert((!ProfSymList || !ProfSymList->isMD5()) && |
| 596 | "Writing string-based ProfileSymbolListSection from MD5 table " |
| 597 | "not yet implemented" ); |
| 598 | if (ProfSymList && ProfSymList->size() > 0) |
| 599 | if (std::error_code EC = ProfSymList->write(OS&: *OutputStream)) |
| 600 | return EC; |
| 601 | |
| 602 | return sampleprof_error::success; |
| 603 | } |
| 604 | |
| 605 | std::error_code |
| 606 | SampleProfileWriterExtBinaryBase::writeMD5ProfileSymbolListSection() { |
| 607 | if (!ProfSymList || ProfSymList->size() == 0) |
| 608 | return sampleprof_error::success; |
| 609 | assert(!ProfSymList->isMD5() && |
| 610 | "Writing MD5 ProfileSymbolListSection from existing MD5 " |
| 611 | "table not yet implemented" ); |
| 612 | |
| 613 | auto &OS = *OutputStream; |
| 614 | std::vector<uint64_t> Keys = ProfSymList->collectGUIDs(); |
| 615 | |
| 616 | auto Table = |
| 617 | llvm::EytzingerTable<support::ulittle64_t>::create(Keys: std::move(Keys)); |
| 618 | |
| 619 | OS.write(Ptr: reinterpret_cast<const char *>(Table.data()), |
| 620 | Size: Table.size() * sizeof(support::ulittle64_t)); |
| 621 | return sampleprof_error::success; |
| 622 | } |
| 623 | |
| 624 | unsigned SampleProfileWriterExtBinaryBase::findUnwrittenEntry(SecType Type) { |
| 625 | auto WrittenIndices = |
| 626 | llvm::map_range(C&: SecHdrTable, F: &SecHdrTableEntry::LayoutIndex); |
| 627 | for (auto [I, Entry] : llvm::enumerate(First&: SectionHdrLayout)) |
| 628 | if (Entry.Type == Type && !llvm::is_contained(Range&: WrittenIndices, Element: I)) |
| 629 | return I; |
| 630 | llvm_unreachable("Matching section not found in SectionHdrLayout" ); |
| 631 | } |
| 632 | |
| 633 | std::error_code SampleProfileWriterExtBinaryBase::writeOneSection( |
| 634 | SecType Type, const SampleProfileMap &ProfileMap) { |
| 635 | unsigned LayoutIdx = findUnwrittenEntry(Type); |
| 636 | SecHdrTableEntry &Entry = SectionHdrLayout[LayoutIdx]; |
| 637 | |
| 638 | // The setting of SecFlagCompress should happen before markSectionStart. |
| 639 | if (Type == SecFuncMetadata && FunctionSamples::ProfileIsProbeBased) |
| 640 | addSectionFlag(Type: SecFuncMetadata, Flag: SecFuncMetadataFlags::SecFlagIsProbeBased); |
| 641 | if (Type == SecFuncMetadata && |
| 642 | (FunctionSamples::ProfileIsCS || FunctionSamples::ProfileIsPreInlined)) |
| 643 | addSectionFlag(Type: SecFuncMetadata, Flag: SecFuncMetadataFlags::SecFlagHasAttribute); |
| 644 | if (Type == SecProfSummary && FunctionSamples::ProfileIsCS) |
| 645 | addSectionFlag(Type: SecProfSummary, Flag: SecProfSummaryFlags::SecFlagFullContext); |
| 646 | if (Type == SecProfSummary && FunctionSamples::ProfileIsPreInlined) |
| 647 | addSectionFlag(Type: SecProfSummary, Flag: SecProfSummaryFlags::SecFlagIsPreInlined); |
| 648 | if (Type == SecProfSummary && FunctionSamples::ProfileIsFS) |
| 649 | addSectionFlag(Type: SecProfSummary, Flag: SecProfSummaryFlags::SecFlagFSDiscriminator); |
| 650 | if (Type == SecProfSummary && ExtBinaryWriteVTableTypeProf) |
| 651 | addSectionFlag(Type: SecProfSummary, |
| 652 | Flag: SecProfSummaryFlags::SecFlagHasVTableTypeProf); |
| 653 | if (Type == SecProfileSymbolList && UseMD5ProfSymList) |
| 654 | addSectionFlag(Type: SecProfileSymbolList, Flag: SecProfileSymbolListFlags::SecFlagMD5); |
| 655 | if (Type == SecNameTable && UseMD5IndexedTables && UseMD5) |
| 656 | addSectionFlag(Type: SecNameTable, Flag: SecNameTableFlags::SecFlagEytzinger); |
| 657 | |
| 658 | uint64_t SectionStart = markSectionStart(Type, LayoutIdx); |
| 659 | switch (Type) { |
| 660 | case SecProfSummary: |
| 661 | computeSummary(ProfileMap); |
| 662 | if (auto EC = writeSummary()) |
| 663 | return EC; |
| 664 | break; |
| 665 | case SecNameTable: |
| 666 | if (auto EC = writeNameTableSection(ProfileMap)) |
| 667 | return EC; |
| 668 | break; |
| 669 | case SecCSNameTable: |
| 670 | if (auto EC = writeCSNameTableSection()) |
| 671 | return EC; |
| 672 | break; |
| 673 | case SecLBRProfile: |
| 674 | SecLBRProfileStart = OutputStream->tell(); |
| 675 | if (std::error_code EC = writeFuncProfiles(ProfileMap)) |
| 676 | return EC; |
| 677 | break; |
| 678 | case SecFuncOffsetTable: { |
| 679 | bool IsFlat = hasSecFlag(Entry, Flag: SecCommonFlags::SecFlagFlat); |
| 680 | // An unflagged function offset table inherently indexes the primary |
| 681 | // Nested symbol span. |
| 682 | bool IsNested = !IsFlat; |
| 683 | if (auto EC = writeFuncOffsetTable(IsNested)) |
| 684 | return EC; |
| 685 | break; |
| 686 | } |
| 687 | case SecFuncMetadata: |
| 688 | if (std::error_code EC = writeFuncMetadata(Profiles: ProfileMap)) |
| 689 | return EC; |
| 690 | break; |
| 691 | case SecProfileSymbolList: |
| 692 | if (auto EC = writeProfileSymbolListSection()) |
| 693 | return EC; |
| 694 | break; |
| 695 | default: |
| 696 | if (auto EC = writeCustomSection(Type)) |
| 697 | return EC; |
| 698 | break; |
| 699 | } |
| 700 | if (std::error_code EC = addNewSection(Type, LayoutIdx, SectionStart)) |
| 701 | return EC; |
| 702 | return sampleprof_error::success; |
| 703 | } |
| 704 | |
| 705 | SampleProfileWriterExtBinary::SampleProfileWriterExtBinary( |
| 706 | std::unique_ptr<raw_ostream> &OS) |
| 707 | : SampleProfileWriterExtBinaryBase(OS) { |
| 708 | WriteVTableProf = ExtBinaryWriteVTableTypeProf; |
| 709 | } |
| 710 | |
| 711 | std::error_code SampleProfileWriterExtBinary::writeDefaultLayout( |
| 712 | const SampleProfileMap &ProfileMap) { |
| 713 | static constexpr SecType Sections[] = { |
| 714 | SecProfSummary, SecNameTable, SecCSNameTable, SecLBRProfile, |
| 715 | SecProfileSymbolList, SecFuncOffsetTable, SecFuncMetadata, |
| 716 | }; |
| 717 | for (SecType Type : Sections) |
| 718 | if (std::error_code EC = writeOneSection(Type, ProfileMap)) |
| 719 | return EC; |
| 720 | return sampleprof_error::success; |
| 721 | } |
| 722 | |
| 723 | static void splitProfileMapToTwo(const SampleProfileMap &ProfileMap, |
| 724 | SampleProfileMap &NestedProfileMap, |
| 725 | SampleProfileMap &FlatProfileMap) { |
| 726 | for (const auto &I : ProfileMap) { |
| 727 | if (I.second.hasCallsiteSamples()) |
| 728 | NestedProfileMap.insert(x: {I.first, I.second}); |
| 729 | else |
| 730 | FlatProfileMap.insert(x: {I.first, I.second}); |
| 731 | } |
| 732 | } |
| 733 | |
| 734 | std::error_code SampleProfileWriterExtBinary::writeCtxSplitLayout( |
| 735 | const SampleProfileMap &ProfileMap) { |
| 736 | SampleProfileMap NestedProfileMap, FlatProfileMap; |
| 737 | splitProfileMapToTwo(ProfileMap, NestedProfileMap, FlatProfileMap); |
| 738 | |
| 739 | const std::pair<SecType, const SampleProfileMap &> Sections[] = { |
| 740 | {SecProfSummary, ProfileMap}, |
| 741 | {SecNameTable, ProfileMap}, |
| 742 | {SecLBRProfile, NestedProfileMap}, |
| 743 | {SecFuncOffsetTable, NestedProfileMap}, |
| 744 | {SecLBRProfile, FlatProfileMap}, |
| 745 | {SecFuncOffsetTable, FlatProfileMap}, |
| 746 | {SecProfileSymbolList, ProfileMap}, |
| 747 | {SecFuncMetadata, ProfileMap}, |
| 748 | }; |
| 749 | for (const auto &[Type, Map] : Sections) |
| 750 | if (std::error_code EC = writeOneSection(Type, ProfileMap: Map)) |
| 751 | return EC; |
| 752 | |
| 753 | return sampleprof_error::success; |
| 754 | } |
| 755 | |
| 756 | std::error_code SampleProfileWriterExtBinary::writeSections( |
| 757 | const SampleProfileMap &ProfileMap) { |
| 758 | std::error_code EC; |
| 759 | if (SecLayout == DefaultLayout) |
| 760 | EC = writeDefaultLayout(ProfileMap); |
| 761 | else if (SecLayout == CtxSplitLayout) |
| 762 | EC = writeCtxSplitLayout(ProfileMap); |
| 763 | else |
| 764 | llvm_unreachable("Unsupported layout" ); |
| 765 | return EC; |
| 766 | } |
| 767 | |
| 768 | /// Write samples to a text file. |
| 769 | /// |
| 770 | /// Note: it may be tempting to implement this in terms of |
| 771 | /// FunctionSamples::print(). Please don't. The dump functionality is intended |
| 772 | /// for debugging and has no specified form. |
| 773 | /// |
| 774 | /// The format used here is more structured and deliberate because |
| 775 | /// it needs to be parsed by the SampleProfileReaderText class. |
| 776 | std::error_code SampleProfileWriterText::writeSample(const FunctionSamples &S) { |
| 777 | auto &OS = *OutputStream; |
| 778 | if (FunctionSamples::ProfileIsCS) |
| 779 | OS << "[" << S.getContext().toString() << "]:" << S.getTotalSamples(); |
| 780 | else |
| 781 | OS << S.getFunction() << ":" << S.getTotalSamples(); |
| 782 | |
| 783 | if (Indent == 0) |
| 784 | OS << ":" << S.getHeadSamples(); |
| 785 | OS << "\n" ; |
| 786 | LineCount++; |
| 787 | |
| 788 | for (const auto &[Loc, Sample] : S.getBodySamples()) { |
| 789 | OS.indent(NumSpaces: Indent + 1); |
| 790 | Loc.print(OS); |
| 791 | OS << ": " << Sample.getSamples(); |
| 792 | |
| 793 | for (const auto &J : Sample.getSortedCallTargets()) |
| 794 | OS << " " << J.first << ":" << J.second; |
| 795 | OS << "\n" ; |
| 796 | LineCount++; |
| 797 | |
| 798 | if (const TypeCountMap *Map = S.findCallsiteTypeSamplesAt(Loc); |
| 799 | Map && !Map->empty()) { |
| 800 | OS.indent(NumSpaces: Indent + 1); |
| 801 | Loc.print(OS); |
| 802 | OS << ": " ; |
| 803 | OS << kVTableProfPrefix; |
| 804 | for (const auto [TypeName, Count] : *Map) { |
| 805 | OS << TypeName << ":" << Count << " " ; |
| 806 | } |
| 807 | OS << "\n" ; |
| 808 | LineCount++; |
| 809 | } |
| 810 | } |
| 811 | |
| 812 | Indent += 1; |
| 813 | for (const auto &[Loc, FunctionSamplesMap] : S.getCallsiteSamples()) { |
| 814 | for (const FunctionSamples &CalleeSamples : |
| 815 | make_second_range(c: FunctionSamplesMap)) { |
| 816 | OS.indent(NumSpaces: Indent); |
| 817 | Loc.print(OS); |
| 818 | OS << ": " ; |
| 819 | if (std::error_code EC = writeSample(S: CalleeSamples)) |
| 820 | return EC; |
| 821 | } |
| 822 | |
| 823 | if (const TypeCountMap *Map = S.findCallsiteTypeSamplesAt(Loc); |
| 824 | Map && !Map->empty()) { |
| 825 | OS.indent(NumSpaces: Indent); |
| 826 | Loc.print(OS); |
| 827 | OS << ": " ; |
| 828 | OS << kVTableProfPrefix; |
| 829 | for (const auto [TypeId, Count] : *Map) { |
| 830 | OS << TypeId << ":" << Count << " " ; |
| 831 | } |
| 832 | OS << "\n" ; |
| 833 | LineCount++; |
| 834 | } |
| 835 | } |
| 836 | |
| 837 | Indent -= 1; |
| 838 | |
| 839 | if (FunctionSamples::ProfileIsProbeBased) { |
| 840 | OS.indent(NumSpaces: Indent + 1); |
| 841 | OS << "!CFGChecksum: " << S.getFunctionHash() << "\n" ; |
| 842 | LineCount++; |
| 843 | } |
| 844 | |
| 845 | if (S.getContext().getAllAttributes()) { |
| 846 | OS.indent(NumSpaces: Indent + 1); |
| 847 | OS << "!Attributes: " << S.getContext().getAllAttributes() << "\n" ; |
| 848 | LineCount++; |
| 849 | } |
| 850 | |
| 851 | if (Indent == 0 && MarkFlatProfiles && S.getCallsiteSamples().size() == 0) |
| 852 | OS << " !Flat\n" ; |
| 853 | |
| 854 | return sampleprof_error::success; |
| 855 | } |
| 856 | |
| 857 | std::error_code |
| 858 | SampleProfileWriterBinary::writeContextIdx(const SampleContext &Context) { |
| 859 | assert(!Context.hasContext() && "cs profile is not supported" ); |
| 860 | return writeNameIdx(FName: Context.getFunction()); |
| 861 | } |
| 862 | |
| 863 | std::error_code SampleProfileWriterBinary::writeNameIdx(FunctionId FName) { |
| 864 | auto &NTable = getNameTable(); |
| 865 | const auto &Ret = NTable.find(Key: FName); |
| 866 | if (Ret == NTable.end()) |
| 867 | return sampleprof_error::truncated_name_table; |
| 868 | encodeULEB128(Value: Ret->second, OS&: *OutputStream); |
| 869 | return sampleprof_error::success; |
| 870 | } |
| 871 | |
| 872 | void SampleProfileWriterBinary::addName(FunctionId FName) { |
| 873 | auto &NTable = getNameTable(); |
| 874 | NTable.insert(KV: std::make_pair(x&: FName, y: 0)); |
| 875 | } |
| 876 | |
| 877 | void SampleProfileWriterBinary::addContext(const SampleContext &Context) { |
| 878 | addName(FName: Context.getFunction()); |
| 879 | } |
| 880 | |
| 881 | void SampleProfileWriterBinary::addNames(const FunctionSamples &S) { |
| 882 | // Add all the names in indirect call targets. |
| 883 | for (const auto &I : S.getBodySamples()) { |
| 884 | const SampleRecord &Sample = I.second; |
| 885 | for (const auto &J : Sample.getCallTargets()) |
| 886 | addName(FName: J.first); |
| 887 | } |
| 888 | |
| 889 | // Recursively add all the names for inlined callsites. |
| 890 | for (const auto &J : S.getCallsiteSamples()) |
| 891 | for (const auto &FS : J.second) { |
| 892 | const FunctionSamples &CalleeSamples = FS.second; |
| 893 | addName(FName: CalleeSamples.getFunction()); |
| 894 | addNames(S: CalleeSamples); |
| 895 | } |
| 896 | |
| 897 | if (!WriteVTableProf) |
| 898 | return; |
| 899 | // Add all the vtable names to NameTable. |
| 900 | for (const auto &VTableAccessCountMap : |
| 901 | llvm::make_second_range(c: S.getCallsiteTypeCounts())) { |
| 902 | // Add type name to NameTable. |
| 903 | for (const auto Type : llvm::make_first_range(c: VTableAccessCountMap)) { |
| 904 | addName(FName: Type); |
| 905 | } |
| 906 | } |
| 907 | } |
| 908 | |
| 909 | void SampleProfileWriterExtBinaryBase::addContext( |
| 910 | const SampleContext &Context) { |
| 911 | if (Context.hasContext()) { |
| 912 | for (auto &Callsite : Context.getContextFrames()) |
| 913 | SampleProfileWriterBinary::addName(FName: Callsite.Func); |
| 914 | CSNameTable.insert(KV: std::make_pair(x: Context, y: 0)); |
| 915 | } else { |
| 916 | SampleProfileWriterBinary::addName(FName: Context.getFunction()); |
| 917 | } |
| 918 | } |
| 919 | |
| 920 | std::error_code SampleProfileWriterBinary::writeNameTable() { |
| 921 | auto &OS = *OutputStream; |
| 922 | |
| 923 | // Write out the name table. |
| 924 | encodeULEB128(Value: NameTable.size(), OS); |
| 925 | for (const auto *Entry : stabilizeTable(Table&: NameTable)) { |
| 926 | OS << Entry->first; |
| 927 | encodeULEB128(Value: 0, OS); |
| 928 | } |
| 929 | return sampleprof_error::success; |
| 930 | } |
| 931 | |
| 932 | std::error_code |
| 933 | SampleProfileWriterBinary::writeMagicIdent(SampleProfileFormat Format) { |
| 934 | auto &OS = *OutputStream; |
| 935 | // Write file magic identifier. |
| 936 | encodeULEB128(Value: SPMagic(Format), OS); |
| 937 | encodeULEB128(Value: FormatVersion, OS); |
| 938 | return sampleprof_error::success; |
| 939 | } |
| 940 | |
| 941 | std::error_code |
| 942 | SampleProfileWriterBinary::(const SampleProfileMap &ProfileMap) { |
| 943 | // When calling write on a different profile map, existing names should be |
| 944 | // cleared. |
| 945 | NameTable.clear(); |
| 946 | |
| 947 | writeMagicIdent(Format); |
| 948 | |
| 949 | computeSummary(ProfileMap); |
| 950 | if (auto EC = writeSummary()) |
| 951 | return EC; |
| 952 | |
| 953 | // Generate the name table for all the functions referenced in the profile. |
| 954 | for (const auto &I : ProfileMap) { |
| 955 | addContext(Context: I.second.getContext()); |
| 956 | addNames(S: I.second); |
| 957 | } |
| 958 | |
| 959 | writeNameTable(); |
| 960 | return sampleprof_error::success; |
| 961 | } |
| 962 | |
| 963 | void SampleProfileWriterExtBinaryBase::setToCompressAllSections() { |
| 964 | for (auto &Entry : SectionHdrLayout) |
| 965 | addSecFlag(Entry, Flag: SecCommonFlags::SecFlagCompress); |
| 966 | } |
| 967 | |
| 968 | void SampleProfileWriterExtBinaryBase::setToCompressSection(SecType Type) { |
| 969 | addSectionFlag(Type, Flag: SecCommonFlags::SecFlagCompress); |
| 970 | } |
| 971 | |
| 972 | void SampleProfileWriterExtBinaryBase::allocSecHdrTable() { |
| 973 | support::endian::Writer Writer(*OutputStream, llvm::endianness::little); |
| 974 | |
| 975 | Writer.write(Val: static_cast<uint64_t>(SectionHdrLayout.size())); |
| 976 | SecHdrTableOffset = OutputStream->tell(); |
| 977 | for (uint32_t i = 0; i < SectionHdrLayout.size(); i++) { |
| 978 | Writer.write(Val: static_cast<uint64_t>(-1)); |
| 979 | Writer.write(Val: static_cast<uint64_t>(-1)); |
| 980 | Writer.write(Val: static_cast<uint64_t>(-1)); |
| 981 | Writer.write(Val: static_cast<uint64_t>(-1)); |
| 982 | } |
| 983 | } |
| 984 | |
| 985 | std::error_code SampleProfileWriterExtBinaryBase::writeSecHdrTable() { |
| 986 | assert(SecHdrTable.size() == SectionHdrLayout.size() && |
| 987 | "SecHdrTable entries doesn't match SectionHdrLayout" ); |
| 988 | SmallVector<uint32_t, 16> IndexMap(SecHdrTable.size(), -1); |
| 989 | for (uint32_t TableIdx = 0; TableIdx < SecHdrTable.size(); TableIdx++) { |
| 990 | IndexMap[SecHdrTable[TableIdx].LayoutIndex] = TableIdx; |
| 991 | } |
| 992 | |
| 993 | // Write the section header table in the order specified in |
| 994 | // SectionHdrLayout. SectionHdrLayout specifies the sections |
| 995 | // order in which profile reader expect to read, so the section |
| 996 | // header table should be written in the order in SectionHdrLayout. |
| 997 | // Note that the section order in SecHdrTable may be different |
| 998 | // from the order in SectionHdrLayout, for example, SecFuncOffsetTable |
| 999 | // needs to be computed after SecLBRProfile (the order in SecHdrTable), |
| 1000 | // but it needs to be read before SecLBRProfile (the order in |
| 1001 | // SectionHdrLayout). So we use IndexMap above to switch the order. |
| 1002 | support::endian::SeekableWriter Writer( |
| 1003 | static_cast<raw_pwrite_stream &>(*OutputStream), |
| 1004 | llvm::endianness::little); |
| 1005 | for (uint32_t LayoutIdx = 0; LayoutIdx < SectionHdrLayout.size(); |
| 1006 | LayoutIdx++) { |
| 1007 | assert(IndexMap[LayoutIdx] < SecHdrTable.size() && |
| 1008 | "Incorrect LayoutIdx in SecHdrTable" ); |
| 1009 | auto Entry = SecHdrTable[IndexMap[LayoutIdx]]; |
| 1010 | Writer.pwrite(Val: static_cast<uint64_t>(Entry.Type), |
| 1011 | Offset: SecHdrTableOffset + 4 * LayoutIdx * sizeof(uint64_t)); |
| 1012 | Writer.pwrite(Val: static_cast<uint64_t>(Entry.Flags), |
| 1013 | Offset: SecHdrTableOffset + (4 * LayoutIdx + 1) * sizeof(uint64_t)); |
| 1014 | Writer.pwrite(Val: static_cast<uint64_t>(Entry.Offset), |
| 1015 | Offset: SecHdrTableOffset + (4 * LayoutIdx + 2) * sizeof(uint64_t)); |
| 1016 | Writer.pwrite(Val: static_cast<uint64_t>(Entry.Size), |
| 1017 | Offset: SecHdrTableOffset + (4 * LayoutIdx + 3) * sizeof(uint64_t)); |
| 1018 | } |
| 1019 | |
| 1020 | return sampleprof_error::success; |
| 1021 | } |
| 1022 | |
| 1023 | std::error_code SampleProfileWriterExtBinaryBase::( |
| 1024 | const SampleProfileMap &ProfileMap) { |
| 1025 | auto &OS = *OutputStream; |
| 1026 | FileStart = OS.tell(); |
| 1027 | writeMagicIdent(Format); |
| 1028 | |
| 1029 | allocSecHdrTable(); |
| 1030 | return sampleprof_error::success; |
| 1031 | } |
| 1032 | |
| 1033 | std::error_code SampleProfileWriterBinary::writeCallsiteVTableProf( |
| 1034 | const CallsiteTypeMap &CallsiteTypeMap, raw_ostream &OS) { |
| 1035 | assert(WriteVTableProf && |
| 1036 | "writeCallsiteVTableProf should not be called if WriteVTableProf is " |
| 1037 | "false" ); |
| 1038 | |
| 1039 | encodeULEB128(Value: CallsiteTypeMap.size(), OS); |
| 1040 | for (const auto &[Loc, TypeMap] : CallsiteTypeMap) { |
| 1041 | Loc.serialize(OS); |
| 1042 | if (std::error_code EC = serializeTypeMap(Map: TypeMap, NameTable: getNameTable(), OS)) |
| 1043 | return EC; |
| 1044 | } |
| 1045 | |
| 1046 | return sampleprof_error::success; |
| 1047 | } |
| 1048 | |
| 1049 | std::error_code SampleProfileWriterBinary::writeSummary() { |
| 1050 | auto &OS = *OutputStream; |
| 1051 | encodeULEB128(Value: Summary->getTotalCount(), OS); |
| 1052 | encodeULEB128(Value: Summary->getMaxCount(), OS); |
| 1053 | encodeULEB128(Value: Summary->getMaxFunctionCount(), OS); |
| 1054 | encodeULEB128(Value: Summary->getNumCounts(), OS); |
| 1055 | encodeULEB128(Value: Summary->getNumFunctions(), OS); |
| 1056 | ArrayRef<ProfileSummaryEntry> Entries = Summary->getDetailedSummary(); |
| 1057 | encodeULEB128(Value: Entries.size(), OS); |
| 1058 | for (auto Entry : Entries) { |
| 1059 | encodeULEB128(Value: Entry.Cutoff, OS); |
| 1060 | encodeULEB128(Value: Entry.MinCount, OS); |
| 1061 | encodeULEB128(Value: Entry.NumCounts, OS); |
| 1062 | } |
| 1063 | return sampleprof_error::success; |
| 1064 | } |
| 1065 | std::error_code SampleProfileWriterBinary::writeBody(const FunctionSamples &S) { |
| 1066 | auto &OS = *OutputStream; |
| 1067 | if (std::error_code EC = writeContextIdx(Context: S.getContext())) |
| 1068 | return EC; |
| 1069 | |
| 1070 | encodeULEB128(Value: S.getTotalSamples(), OS); |
| 1071 | |
| 1072 | // Emit all the body samples. |
| 1073 | encodeULEB128(Value: S.getBodySamples().size(), OS); |
| 1074 | for (const auto &I : S.getBodySamples()) { |
| 1075 | LineLocation Loc = I.first; |
| 1076 | const SampleRecord &Sample = I.second; |
| 1077 | Loc.serialize(OS); |
| 1078 | Sample.serialize(OS, NameTable: getNameTable()); |
| 1079 | } |
| 1080 | |
| 1081 | // Recursively emit all the callsite samples. |
| 1082 | uint64_t NumCallsites = 0; |
| 1083 | for (const auto &J : S.getCallsiteSamples()) |
| 1084 | NumCallsites += J.second.size(); |
| 1085 | encodeULEB128(Value: NumCallsites, OS); |
| 1086 | for (const auto &J : S.getCallsiteSamples()) |
| 1087 | for (const auto &FS : J.second) { |
| 1088 | J.first.serialize(OS); |
| 1089 | if (std::error_code EC = writeBody(S: FS.second)) |
| 1090 | return EC; |
| 1091 | } |
| 1092 | |
| 1093 | if (WriteVTableProf) |
| 1094 | return writeCallsiteVTableProf(CallsiteTypeMap: S.getCallsiteTypeCounts(), OS); |
| 1095 | |
| 1096 | return sampleprof_error::success; |
| 1097 | } |
| 1098 | |
| 1099 | /// Write samples of a top-level function to a binary file. |
| 1100 | /// |
| 1101 | /// \returns true if the samples were written successfully, false otherwise. |
| 1102 | std::error_code |
| 1103 | SampleProfileWriterBinary::writeSample(const FunctionSamples &S) { |
| 1104 | encodeULEB128(Value: S.getHeadSamples(), OS&: *OutputStream); |
| 1105 | return writeBody(S); |
| 1106 | } |
| 1107 | |
| 1108 | /// Create a sample profile file writer based on the specified format. |
| 1109 | /// |
| 1110 | /// \param Filename The file to create. |
| 1111 | /// |
| 1112 | /// \param Format Encoding format for the profile file. |
| 1113 | /// |
| 1114 | /// \returns an error code indicating the status of the created writer. |
| 1115 | ErrorOr<std::unique_ptr<SampleProfileWriter>> |
| 1116 | SampleProfileWriter::create(StringRef Filename, SampleProfileFormat Format) { |
| 1117 | std::error_code EC; |
| 1118 | std::unique_ptr<raw_ostream> OS; |
| 1119 | if (Format == SPF_Binary || Format == SPF_Ext_Binary) |
| 1120 | OS.reset(p: new raw_fd_ostream(Filename, EC, sys::fs::OF_None)); |
| 1121 | else |
| 1122 | OS.reset(p: new raw_fd_ostream(Filename, EC, sys::fs::OF_TextWithCRLF)); |
| 1123 | if (EC) |
| 1124 | return EC; |
| 1125 | |
| 1126 | return create(OS, Format); |
| 1127 | } |
| 1128 | |
| 1129 | /// Create a sample profile stream writer based on the specified format. |
| 1130 | /// |
| 1131 | /// \param OS The output stream to store the profile data to. |
| 1132 | /// |
| 1133 | /// \param Format Encoding format for the profile file. |
| 1134 | /// |
| 1135 | /// \returns an error code indicating the status of the created writer. |
| 1136 | ErrorOr<std::unique_ptr<SampleProfileWriter>> |
| 1137 | SampleProfileWriter::create(std::unique_ptr<raw_ostream> &OS, |
| 1138 | SampleProfileFormat Format) { |
| 1139 | std::error_code EC; |
| 1140 | std::unique_ptr<SampleProfileWriter> Writer; |
| 1141 | |
| 1142 | // Currently only Text and Extended Binary format are supported for CSSPGO. |
| 1143 | if ((FunctionSamples::ProfileIsCS || FunctionSamples::ProfileIsProbeBased) && |
| 1144 | Format == SPF_Binary) |
| 1145 | return sampleprof_error::unsupported_writing_format; |
| 1146 | |
| 1147 | if (Format == SPF_Binary) |
| 1148 | Writer.reset(p: new SampleProfileWriterRawBinary(OS)); |
| 1149 | else if (Format == SPF_Ext_Binary) |
| 1150 | Writer.reset(p: new SampleProfileWriterExtBinary(OS)); |
| 1151 | else if (Format == SPF_Text) |
| 1152 | Writer.reset(p: new SampleProfileWriterText(OS)); |
| 1153 | else if (Format == SPF_GCC) |
| 1154 | EC = sampleprof_error::unsupported_writing_format; |
| 1155 | else |
| 1156 | EC = sampleprof_error::unrecognized_format; |
| 1157 | |
| 1158 | if (EC) |
| 1159 | return EC; |
| 1160 | |
| 1161 | Writer->Format = Format; |
| 1162 | if (Format != SPF_Ext_Binary) |
| 1163 | Writer->setFormatVersion(DefaultVersion); |
| 1164 | else if (formatVersionIsSupported(Version: RequestedVersion)) |
| 1165 | Writer->setFormatVersion(RequestedVersion); |
| 1166 | else |
| 1167 | return sampleprof_error::unsupported_version; |
| 1168 | return std::move(Writer); |
| 1169 | } |
| 1170 | |
| 1171 | void SampleProfileWriter::computeSummary(const SampleProfileMap &ProfileMap) { |
| 1172 | SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs); |
| 1173 | Summary = Builder.computeSummaryForProfiles(Profiles: ProfileMap); |
| 1174 | } |
| 1175 | |