1//===- InstrProfWriter.cpp - Instrumented profiling writer ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains support for writing profiling data for clang's
10// instrumentation based PGO and coverage.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/ProfileData/InstrProfWriter.h"
15#include "llvm/ADT/STLExtras.h"
16#include "llvm/ADT/StringRef.h"
17#include "llvm/IR/ProfileSummary.h"
18#include "llvm/ProfileData/DataAccessProf.h"
19#include "llvm/ProfileData/IndexedMemProfData.h"
20#include "llvm/ProfileData/InstrProf.h"
21#include "llvm/ProfileData/ProfileCommon.h"
22#include "llvm/Support/Compression.h"
23#include "llvm/Support/EndianStream.h"
24#include "llvm/Support/Error.h"
25#include "llvm/Support/MemoryBuffer.h"
26#include "llvm/Support/OnDiskHashTable.h"
27#include "llvm/Support/raw_ostream.h"
28#include <cstdint>
29#include <memory>
30#include <string>
31#include <tuple>
32#include <utility>
33#include <vector>
34
35using namespace llvm;
36
37namespace llvm {
38
39class InstrProfRecordWriterTrait {
40public:
41 using key_type = StringRef;
42 using key_type_ref = StringRef;
43
44 using data_type = const InstrProfWriter::ProfilingData *const;
45 using data_type_ref = const InstrProfWriter::ProfilingData *const;
46
47 using hash_value_type = uint64_t;
48 using offset_type = uint64_t;
49
50 llvm::endianness ValueProfDataEndianness = llvm::endianness::little;
51 InstrProfSummaryBuilder *SummaryBuilder;
52 InstrProfSummaryBuilder *CSSummaryBuilder;
53 bool WritePrevVersion = false;
54
55 InstrProfRecordWriterTrait() = default;
56
57 static hash_value_type ComputeHash(key_type_ref K) {
58 return IndexedInstrProf::ComputeHash(K);
59 }
60
61 std::pair<offset_type, offset_type>
62 EmitKeyDataLength(raw_ostream &Out, key_type_ref K, data_type_ref V) {
63 using namespace support;
64
65 endian::Writer LE(Out, llvm::endianness::little);
66
67 offset_type N = K.size();
68 LE.write<offset_type>(Val: N);
69
70 offset_type M = 0;
71 for (const auto &ProfileData : *V) {
72 const InstrProfRecord &ProfRecord = ProfileData.second;
73 M += sizeof(uint64_t); // The function hash
74 M += sizeof(uint64_t); // The size of the Counts vector
75 M += ProfRecord.Counts.size() * sizeof(uint64_t);
76 M += sizeof(uint64_t); // The size of the Bitmap vector
77 if (WritePrevVersion) {
78 // Compatibility mode: each bitmap byte is stored as a uint64_t.
79 M += ProfRecord.BitmapBytes.size() * sizeof(uint64_t);
80 } else {
81 // Version 14+: bitmap bytes as uint8_t with padding, plus
82 // uniformity bits.
83 M += alignTo(Value: ProfRecord.BitmapBytes.size(), Align: sizeof(uint64_t));
84 M += sizeof(uint64_t); // The size of the UniformityBits vector
85 M += alignTo(Value: ProfRecord.UniformityBits.size(), Align: sizeof(uint64_t));
86 }
87
88 // Value data
89 M += ValueProfData::getSize(Record: ProfileData.second);
90 }
91 LE.write<offset_type>(Val: M);
92
93 return std::make_pair(x&: N, y&: M);
94 }
95
96 void EmitKey(raw_ostream &Out, key_type_ref K, offset_type N) {
97 Out.write(Ptr: K.data(), Size: N);
98 }
99
100 void EmitData(raw_ostream &Out, key_type_ref K, data_type_ref V,
101 offset_type) {
102 using namespace support;
103
104 endian::Writer LE(Out, llvm::endianness::little);
105 for (const auto &ProfileData : *V) {
106 const InstrProfRecord &ProfRecord = ProfileData.second;
107 if (NamedInstrProfRecord::hasCSFlagInHash(FuncHash: ProfileData.first))
108 CSSummaryBuilder->addRecord(ProfRecord);
109 else
110 SummaryBuilder->addRecord(ProfRecord);
111
112 LE.write<uint64_t>(Val: ProfileData.first); // Function hash
113 LE.write<uint64_t>(Val: ProfRecord.Counts.size());
114 for (uint64_t I : ProfRecord.Counts)
115 LE.write<uint64_t>(Val: I);
116
117 LE.write<uint64_t>(Val: ProfRecord.BitmapBytes.size());
118 if (WritePrevVersion) {
119 // Compatibility mode: each bitmap byte is stored as a uint64_t.
120 for (uint8_t I : ProfRecord.BitmapBytes)
121 LE.write<uint64_t>(Val: I);
122 } else {
123 // Version 14+: bitmap bytes as uint8_t with padding.
124 for (uint8_t I : ProfRecord.BitmapBytes)
125 LE.write<uint8_t>(Val: I);
126 for (size_t I = ProfRecord.BitmapBytes.size();
127 I < alignTo(Value: ProfRecord.BitmapBytes.size(), Align: sizeof(uint64_t)); ++I)
128 LE.write<uint8_t>(Val: 0);
129
130 // Write uniformity bits (AMDGPU offload profiling).
131 LE.write<uint64_t>(Val: ProfRecord.UniformityBits.size());
132 for (uint8_t I : ProfRecord.UniformityBits)
133 LE.write<uint8_t>(Val: I);
134 for (size_t I = ProfRecord.UniformityBits.size();
135 I < alignTo(Value: ProfRecord.UniformityBits.size(), Align: sizeof(uint64_t));
136 ++I)
137 LE.write<uint8_t>(Val: 0);
138 }
139
140 // Write value data
141 std::unique_ptr<ValueProfData> VDataPtr =
142 ValueProfData::serializeFrom(Record: ProfileData.second);
143 uint32_t S = VDataPtr->getSize();
144 VDataPtr->swapBytesFromHost(Endianness: ValueProfDataEndianness);
145 Out.write(Ptr: (const char *)VDataPtr.get(), Size: S);
146 }
147 }
148};
149
150} // end namespace llvm
151
152InstrProfWriter::InstrProfWriter(
153 bool Sparse, uint64_t TemporalProfTraceReservoirSize,
154 uint64_t MaxTemporalProfTraceLength, bool WritePrevVersion,
155 memprof::IndexedVersion MemProfVersionRequested, bool MemProfFullSchema,
156 bool MemprofGenerateRandomHotness, unsigned RandomSeed)
157 : Sparse(Sparse), MaxTemporalProfTraceLength(MaxTemporalProfTraceLength),
158 TemporalProfTraceReservoirSize(TemporalProfTraceReservoirSize),
159 InfoObj(new InstrProfRecordWriterTrait()),
160 WritePrevVersion(WritePrevVersion),
161 MemProfVersionRequested(MemProfVersionRequested),
162 MemProfFullSchema(MemProfFullSchema),
163 MemprofGenerateRandomHotness(MemprofGenerateRandomHotness) {
164 if (RandomSeed)
165 RNG.seed(sd: RandomSeed);
166}
167
168InstrProfWriter::~InstrProfWriter() { delete InfoObj; }
169
170// Internal interface for testing purpose only.
171void InstrProfWriter::setValueProfDataEndianness(llvm::endianness Endianness) {
172 InfoObj->ValueProfDataEndianness = Endianness;
173}
174
175void InstrProfWriter::setOutputSparse(bool Sparse) { this->Sparse = Sparse; }
176
177void InstrProfWriter::addRecord(NamedInstrProfRecord &&I, uint64_t Weight,
178 function_ref<void(Error)> Warn) {
179 auto Name = I.Name;
180 auto Hash = I.Hash;
181 addRecord(Name, Hash, I: std::move(I), Weight, Warn);
182}
183
184void InstrProfWriter::overlapRecord(NamedInstrProfRecord &&Other,
185 OverlapStats &Overlap,
186 OverlapStats &FuncLevelOverlap,
187 const OverlapFuncFilters &FuncFilter) {
188 auto Name = Other.Name;
189 auto Hash = Other.Hash;
190 Other.accumulateCounts(Sum&: FuncLevelOverlap.Test);
191 auto It = FunctionData.find(Key: Name);
192 if (It == FunctionData.end()) {
193 Overlap.addOneUnique(UniqueFunc: FuncLevelOverlap.Test);
194 return;
195 }
196 if (FuncLevelOverlap.Test.CountSum < 1.0f) {
197 Overlap.Overlap.NumEntries += 1;
198 return;
199 }
200 auto &ProfileDataMap = It->second;
201 auto [Where, NewFunc] = ProfileDataMap.try_emplace(Key: Hash);
202 if (NewFunc) {
203 Overlap.addOneMismatch(MismatchFunc: FuncLevelOverlap.Test);
204 return;
205 }
206 InstrProfRecord &Dest = Where->second;
207
208 uint64_t ValueCutoff = FuncFilter.ValueCutoff;
209 if (!FuncFilter.NameFilter.empty() && Name.contains(Other: FuncFilter.NameFilter))
210 ValueCutoff = 0;
211
212 Dest.overlap(Other, Overlap, FuncLevelOverlap, ValueCutoff);
213}
214
215void InstrProfWriter::addRecord(StringRef Name, uint64_t Hash,
216 InstrProfRecord &&I, uint64_t Weight,
217 function_ref<void(Error)> Warn) {
218 I.computeBlockUniformity();
219
220 auto &ProfileDataMap = FunctionData[Name];
221
222 auto [Where, NewFunc] = ProfileDataMap.try_emplace(Key: Hash);
223 InstrProfRecord &Dest = Where->second;
224
225 auto MapWarn = [&](instrprof_error E) {
226 Warn(make_error<InstrProfError>(Args&: E));
227 };
228
229 if (NewFunc) {
230 // We've never seen a function with this name and hash, add it.
231 Dest = std::move(I);
232 if (Weight > 1)
233 Dest.scale(N: Weight, D: 1, Warn: MapWarn);
234 } else {
235 // We're updating a function we've seen before.
236 Dest.merge(Other&: I, Weight, Warn: MapWarn);
237 }
238
239 Dest.sortValueData();
240}
241
242void InstrProfWriter::addMemProfRecord(
243 const Function::GUID Id, const memprof::IndexedMemProfRecord &Record) {
244 auto NewRecord = Record;
245 // Provoke random hotness values if requested. We specify the lifetime access
246 // density and lifetime length that will result in a cold or not cold hotness.
247 // See the logic in getAllocType() in Analysis/MemoryProfileInfo.cpp.
248 if (MemprofGenerateRandomHotness) {
249 for (auto &Alloc : NewRecord.AllocSites) {
250 // To get a not cold context, set the lifetime access density to the
251 // maximum value and the lifetime to 0.
252 uint64_t NewTLAD = std::numeric_limits<uint64_t>::max();
253 uint64_t NewTL = 0;
254 std::bernoulli_distribution IsCold;
255 if (IsCold(RNG)) {
256 // To get a cold context, set the lifetime access density to 0 and the
257 // lifetime to the maximum value.
258 NewTLAD = 0;
259 NewTL = std::numeric_limits<uint64_t>::max();
260 }
261 Alloc.Info.setTotalLifetimeAccessDensity(NewTLAD);
262 Alloc.Info.setTotalLifetime(NewTL);
263 }
264 }
265 MemProfSumBuilder.addRecord(NewRecord);
266 auto [Iter, Inserted] = MemProfData.Records.insert(KV: {Id, NewRecord});
267 // If we inserted a new record then we are done.
268 if (Inserted) {
269 return;
270 }
271 memprof::IndexedMemProfRecord &Existing = Iter->second;
272 Existing.merge(Other: NewRecord);
273}
274
275bool InstrProfWriter::addMemProfFrame(const memprof::FrameId Id,
276 const memprof::Frame &Frame,
277 function_ref<void(Error)> Warn) {
278 auto [Iter, Inserted] = MemProfData.Frames.insert(KV: {Id, Frame});
279 // If a mapping already exists for the current frame id and it does not
280 // match the new mapping provided then reset the existing contents and bail
281 // out. We don't support the merging of memprof data whose Frame -> Id
282 // mapping across profiles is inconsistent.
283 if (!Inserted && Iter->second != Frame) {
284 Warn(make_error<InstrProfError>(Args: instrprof_error::malformed,
285 Args: "frame to id mapping mismatch"));
286 return false;
287 }
288 return true;
289}
290
291bool InstrProfWriter::addMemProfCallStack(
292 const memprof::CallStackId CSId,
293 const llvm::SmallVector<memprof::FrameId> &CallStack,
294 function_ref<void(Error)> Warn) {
295 auto [Iter, Inserted] = MemProfData.CallStacks.insert(KV: {CSId, CallStack});
296 // If a mapping already exists for the current call stack id and it does not
297 // match the new mapping provided then reset the existing contents and bail
298 // out. We don't support the merging of memprof data whose CallStack -> Id
299 // mapping across profiles is inconsistent.
300 if (!Inserted && Iter->second != CallStack) {
301 Warn(make_error<InstrProfError>(Args: instrprof_error::malformed,
302 Args: "call stack to id mapping mismatch"));
303 return false;
304 }
305 return true;
306}
307
308bool InstrProfWriter::addMemProfData(memprof::IndexedMemProfData Incoming,
309 function_ref<void(Error)> Warn) {
310 // Return immediately if everything is empty.
311 if (Incoming.Frames.empty() && Incoming.CallStacks.empty() &&
312 Incoming.Records.empty())
313 return true;
314
315 // Otherwise, every component must be non-empty.
316 assert(!Incoming.Frames.empty() && !Incoming.CallStacks.empty() &&
317 !Incoming.Records.empty());
318
319 if (MemProfData.Frames.empty())
320 MemProfData.Frames = std::move(Incoming.Frames);
321 else
322 for (const auto &[Id, F] : Incoming.Frames)
323 if (addMemProfFrame(Id, Frame: F, Warn))
324 return false;
325
326 if (MemProfData.CallStacks.empty())
327 MemProfData.CallStacks = std::move(Incoming.CallStacks);
328 else
329 for (const auto &[CSId, CS] : Incoming.CallStacks)
330 if (addMemProfCallStack(CSId, CallStack: CS, Warn))
331 return false;
332
333 // Add one record at a time if randomization is requested.
334 if (MemProfData.Records.empty() && !MemprofGenerateRandomHotness) {
335 // Need to manually add each record to the builder, which is otherwise done
336 // in addMemProfRecord.
337 for (const auto &[GUID, Record] : Incoming.Records)
338 MemProfSumBuilder.addRecord(Record);
339 MemProfData.Records = std::move(Incoming.Records);
340 } else {
341 for (const auto &[GUID, Record] : Incoming.Records)
342 addMemProfRecord(Id: GUID, Record);
343 }
344
345 return true;
346}
347
348void InstrProfWriter::addBinaryIds(ArrayRef<llvm::object::BuildID> BIs) {
349 llvm::append_range(C&: BinaryIds, R&: BIs);
350}
351
352void InstrProfWriter::addDataAccessProfData(
353 std::unique_ptr<memprof::DataAccessProfData> DataAccessProfDataIn) {
354 DataAccessProfileData = std::move(DataAccessProfDataIn);
355}
356
357void InstrProfWriter::addTemporalProfileTraces(
358 SmallVectorImpl<TemporalProfTraceTy> &SrcTraces, uint64_t SrcStreamSize) {
359 if (TemporalProfTraces.size() > TemporalProfTraceReservoirSize)
360 TemporalProfTraces.truncate(N: TemporalProfTraceReservoirSize);
361 for (auto &Trace : SrcTraces)
362 if (Trace.FunctionNameRefs.size() > MaxTemporalProfTraceLength)
363 Trace.FunctionNameRefs.resize(new_size: MaxTemporalProfTraceLength);
364 llvm::erase_if(C&: SrcTraces, P: [](auto &T) { return T.FunctionNameRefs.empty(); });
365 // If there are no source traces, it is probably because
366 // --temporal-profile-max-trace-length=0 was set to deliberately remove all
367 // traces. In that case, we do not want to increase the stream size
368 if (SrcTraces.empty())
369 return;
370 // Add traces until our reservoir is full or we run out of source traces
371 auto SrcTraceIt = SrcTraces.begin();
372 while (TemporalProfTraces.size() < TemporalProfTraceReservoirSize &&
373 SrcTraceIt < SrcTraces.end())
374 TemporalProfTraces.push_back(Elt: *SrcTraceIt++);
375 // Our reservoir is full, we need to sample the source stream
376 llvm::shuffle(first: SrcTraceIt, last: SrcTraces.end(), g&: RNG);
377 for (uint64_t I = TemporalProfTraces.size();
378 I < SrcStreamSize && SrcTraceIt < SrcTraces.end(); I++) {
379 std::uniform_int_distribution<uint64_t> Distribution(0, I);
380 uint64_t RandomIndex = Distribution(RNG);
381 if (RandomIndex < TemporalProfTraces.size())
382 TemporalProfTraces[RandomIndex] = *SrcTraceIt++;
383 }
384 TemporalProfTraceStreamSize += SrcStreamSize;
385}
386
387void InstrProfWriter::mergeRecordsFromWriter(InstrProfWriter &&IPW,
388 function_ref<void(Error)> Warn) {
389 for (auto &I : IPW.FunctionData)
390 for (auto &Func : I.getValue())
391 addRecord(Name: I.getKey(), Hash: Func.first, I: std::move(Func.second), Weight: 1, Warn);
392
393 BinaryIds.reserve(n: BinaryIds.size() + IPW.BinaryIds.size());
394 for (auto &I : IPW.BinaryIds)
395 addBinaryIds(BIs: I);
396
397 addTemporalProfileTraces(SrcTraces&: IPW.TemporalProfTraces,
398 SrcStreamSize: IPW.TemporalProfTraceStreamSize);
399
400 MemProfData.Frames.reserve(NumEntries: IPW.MemProfData.Frames.size());
401 for (auto &[FrameId, Frame] : IPW.MemProfData.Frames) {
402 // If we weren't able to add the frame mappings then it doesn't make sense
403 // to try to merge the records from this profile.
404 if (!addMemProfFrame(Id: FrameId, Frame, Warn))
405 return;
406 }
407
408 MemProfData.CallStacks.reserve(NumEntries: IPW.MemProfData.CallStacks.size());
409 for (auto &[CSId, CallStack] : IPW.MemProfData.CallStacks) {
410 if (!addMemProfCallStack(CSId, CallStack, Warn))
411 return;
412 }
413
414 MemProfData.Records.reserve(NumEntries: IPW.MemProfData.Records.size());
415 for (auto &[GUID, Record] : IPW.MemProfData.Records) {
416 addMemProfRecord(Id: GUID, Record);
417 }
418}
419
420bool InstrProfWriter::shouldEncodeData(const ProfilingData &PD) {
421 if (!Sparse)
422 return true;
423 for (const auto &Func : PD) {
424 const InstrProfRecord &IPR = Func.second;
425 if (llvm::any_of(Range: IPR.Counts, P: [](uint64_t Count) { return Count > 0; }))
426 return true;
427 if (llvm::any_of(Range: IPR.BitmapBytes, P: [](uint8_t Byte) { return Byte > 0; }))
428 return true;
429 }
430 return false;
431}
432
433static void setSummary(IndexedInstrProf::Summary *TheSummary,
434 ProfileSummary &PS) {
435 using namespace IndexedInstrProf;
436
437 const std::vector<ProfileSummaryEntry> &Res = PS.getDetailedSummary();
438 TheSummary->NumSummaryFields = Summary::NumKinds;
439 TheSummary->NumCutoffEntries = Res.size();
440 TheSummary->set(K: Summary::MaxFunctionCount, V: PS.getMaxFunctionCount());
441 TheSummary->set(K: Summary::MaxBlockCount, V: PS.getMaxCount());
442 TheSummary->set(K: Summary::MaxInternalBlockCount, V: PS.getMaxInternalCount());
443 TheSummary->set(K: Summary::TotalBlockCount, V: PS.getTotalCount());
444 TheSummary->set(K: Summary::TotalNumBlocks, V: PS.getNumCounts());
445 TheSummary->set(K: Summary::TotalNumFunctions, V: PS.getNumFunctions());
446 for (unsigned I = 0; I < Res.size(); I++)
447 TheSummary->setEntry(I, E: Res[I]);
448}
449
450uint64_t InstrProfWriter::writeHeader(const IndexedInstrProf::Header &Header,
451 const bool WritePrevVersion,
452 ProfOStream &OS) {
453 // Only write out the first four fields.
454 for (int I = 0; I < 4; I++)
455 OS.write(V: reinterpret_cast<const uint64_t *>(&Header)[I]);
456
457 // Remember the offset of the remaining fields to allow back patching later.
458 auto BackPatchStartOffset = OS.tell();
459
460 // Reserve the space for back patching later.
461 OS.write(V: 0); // HashOffset
462 OS.write(V: 0); // MemProfOffset
463 OS.write(V: 0); // BinaryIdOffset
464 OS.write(V: 0); // TemporalProfTracesOffset
465 if (!WritePrevVersion)
466 OS.write(V: 0); // VTableNamesOffset
467
468 return BackPatchStartOffset;
469}
470
471Error InstrProfWriter::writeBinaryIds(ProfOStream &OS) {
472 // BinaryIdSection has two parts:
473 // 1. uint64_t BinaryIdsSectionSize
474 // 2. list of binary ids that consist of:
475 // a. uint64_t BinaryIdLength
476 // b. uint8_t BinaryIdData
477 // c. uint8_t Padding (if necessary)
478 // Calculate size of binary section.
479 uint64_t BinaryIdsSectionSize = 0;
480
481 // Remove duplicate binary ids.
482 llvm::sort(C&: BinaryIds);
483 BinaryIds.erase(first: llvm::unique(R&: BinaryIds), last: BinaryIds.end());
484
485 for (const auto &BI : BinaryIds) {
486 // Increment by binary id length data type size.
487 BinaryIdsSectionSize += sizeof(uint64_t);
488 // Increment by binary id data length, aligned to 8 bytes.
489 BinaryIdsSectionSize += alignToPowerOf2(Value: BI.size(), Align: sizeof(uint64_t));
490 }
491 // Write binary ids section size.
492 OS.write(V: BinaryIdsSectionSize);
493
494 for (const auto &BI : BinaryIds) {
495 uint64_t BILen = BI.size();
496 // Write binary id length.
497 OS.write(V: BILen);
498 // Write binary id data.
499 for (unsigned K = 0; K < BILen; K++)
500 OS.writeByte(V: BI[K]);
501 // Write padding if necessary.
502 uint64_t PaddingSize = alignToPowerOf2(Value: BILen, Align: sizeof(uint64_t)) - BILen;
503 for (unsigned K = 0; K < PaddingSize; K++)
504 OS.writeByte(V: 0);
505 }
506
507 return Error::success();
508}
509
510Error InstrProfWriter::writeVTableNames(ProfOStream &OS) {
511 std::vector<std::string> VTableNameStrs;
512 for (StringRef VTableName : VTableNames.keys())
513 VTableNameStrs.push_back(x: VTableName.str());
514
515 std::string CompressedVTableNames;
516 if (!VTableNameStrs.empty())
517 if (Error E = collectGlobalObjectNameStrings(
518 NameStrs: VTableNameStrs, doCompression: compression::zlib::isAvailable(),
519 Result&: CompressedVTableNames))
520 return E;
521
522 const uint64_t CompressedStringLen = CompressedVTableNames.length();
523
524 // Record the length of compressed string.
525 OS.write(V: CompressedStringLen);
526
527 // Write the chars in compressed strings.
528 for (auto &c : CompressedVTableNames)
529 OS.writeByte(V: static_cast<uint8_t>(c));
530
531 // Pad up to a multiple of 8.
532 // InstrProfReader could read bytes according to 'CompressedStringLen'.
533 const uint64_t PaddedLength = alignTo(Value: CompressedStringLen, Align: 8);
534
535 for (uint64_t K = CompressedStringLen; K < PaddedLength; K++)
536 OS.writeByte(V: 0);
537
538 return Error::success();
539}
540
541Error InstrProfWriter::writeImpl(ProfOStream &OS) {
542 using namespace IndexedInstrProf;
543 using namespace support;
544
545 OnDiskChainedHashTableGenerator<InstrProfRecordWriterTrait> Generator;
546
547 InstrProfSummaryBuilder ISB(ProfileSummaryBuilder::DefaultCutoffs);
548 InfoObj->SummaryBuilder = &ISB;
549 InstrProfSummaryBuilder CSISB(ProfileSummaryBuilder::DefaultCutoffs);
550 InfoObj->CSSummaryBuilder = &CSISB;
551 InfoObj->WritePrevVersion = WritePrevVersion;
552
553 // Populate the hash table generator.
554 SmallVector<std::pair<StringRef, const ProfilingData *>> OrderedData;
555 for (const auto &I : FunctionData)
556 if (shouldEncodeData(PD: I.getValue()))
557 OrderedData.emplace_back(Args: (I.getKey()), Args: &I.getValue());
558 llvm::sort(C&: OrderedData, Comp: less_first());
559 for (const auto &I : OrderedData)
560 Generator.insert(Key: I.first, Data: I.second);
561
562 // Write the header.
563 IndexedInstrProf::Header Header;
564 Header.Version = WritePrevVersion
565 ? IndexedInstrProf::ProfVersion::Version11
566 : IndexedInstrProf::ProfVersion::CurrentVersion;
567 // The WritePrevVersion handling will either need to be removed or updated
568 // if the version is advanced beyond 12.
569 static_assert(IndexedInstrProf::ProfVersion::CurrentVersion ==
570 IndexedInstrProf::ProfVersion::Version14);
571 if (static_cast<bool>(ProfileKind & InstrProfKind::IRInstrumentation))
572 Header.Version |= VARIANT_MASK_IR_PROF;
573 if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive))
574 Header.Version |= VARIANT_MASK_CSIR_PROF;
575 if (static_cast<bool>(ProfileKind &
576 InstrProfKind::FunctionEntryInstrumentation))
577 Header.Version |= VARIANT_MASK_INSTR_ENTRY;
578 if (static_cast<bool>(ProfileKind &
579 InstrProfKind::LoopEntriesInstrumentation))
580 Header.Version |= VARIANT_MASK_INSTR_LOOP_ENTRIES;
581 if (static_cast<bool>(ProfileKind & InstrProfKind::SingleByteCoverage))
582 Header.Version |= VARIANT_MASK_BYTE_COVERAGE;
583 if (static_cast<bool>(ProfileKind & InstrProfKind::FunctionEntryOnly))
584 Header.Version |= VARIANT_MASK_FUNCTION_ENTRY_ONLY;
585 if (static_cast<bool>(ProfileKind & InstrProfKind::MemProf))
586 Header.Version |= VARIANT_MASK_MEMPROF;
587 if (static_cast<bool>(ProfileKind & InstrProfKind::TemporalProfile))
588 Header.Version |= VARIANT_MASK_TEMPORAL_PROF;
589
590 const uint64_t BackPatchStartOffset =
591 writeHeader(Header, WritePrevVersion, OS);
592
593 // Reserve space to write profile summary data.
594 uint32_t NumEntries = ProfileSummaryBuilder::DefaultCutoffs.size();
595 uint32_t SummarySize = Summary::getSize(NumSumFields: Summary::NumKinds, NumCutoffEntries: NumEntries);
596 // Remember the summary offset.
597 uint64_t SummaryOffset = OS.tell();
598 for (unsigned I = 0; I < SummarySize / sizeof(uint64_t); I++)
599 OS.write(V: 0);
600 uint64_t CSSummaryOffset = 0;
601 uint64_t CSSummarySize = 0;
602 if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive)) {
603 CSSummaryOffset = OS.tell();
604 CSSummarySize = SummarySize / sizeof(uint64_t);
605 for (unsigned I = 0; I < CSSummarySize; I++)
606 OS.write(V: 0);
607 }
608
609 // Write the hash table.
610 uint64_t HashTableStart = Generator.Emit(Out&: OS.OS, InfoObj&: *InfoObj);
611
612 // Write the MemProf profile data if we have it.
613 uint64_t MemProfSectionStart = 0;
614 if (static_cast<bool>(ProfileKind & InstrProfKind::MemProf)) {
615 MemProfSectionStart = OS.tell();
616
617 if (auto E = writeMemProf(
618 OS, MemProfData, MemProfVersionRequested, MemProfFullSchema,
619 DataAccessProfileData: std::move(DataAccessProfileData), MemProfSum: MemProfSumBuilder.getSummary()))
620 return E;
621 }
622
623 uint64_t BinaryIdSectionStart = OS.tell();
624 if (auto E = writeBinaryIds(OS))
625 return E;
626
627 uint64_t VTableNamesSectionStart = OS.tell();
628
629 if (!WritePrevVersion)
630 if (Error E = writeVTableNames(OS))
631 return E;
632
633 uint64_t TemporalProfTracesSectionStart = 0;
634 if (static_cast<bool>(ProfileKind & InstrProfKind::TemporalProfile)) {
635 TemporalProfTracesSectionStart = OS.tell();
636 OS.write(V: TemporalProfTraces.size());
637 OS.write(V: TemporalProfTraceStreamSize);
638 for (auto &Trace : TemporalProfTraces) {
639 OS.write(V: Trace.Weight);
640 OS.write(V: Trace.FunctionNameRefs.size());
641 for (auto &NameRef : Trace.FunctionNameRefs)
642 OS.write(V: NameRef);
643 }
644 }
645
646 // Allocate space for data to be serialized out.
647 std::unique_ptr<IndexedInstrProf::Summary> TheSummary =
648 IndexedInstrProf::allocSummary(TotalSize: SummarySize);
649 // Compute the Summary and copy the data to the data
650 // structure to be serialized out (to disk or buffer).
651 std::unique_ptr<ProfileSummary> PS = ISB.getSummary();
652 setSummary(TheSummary: TheSummary.get(), PS&: *PS);
653 InfoObj->SummaryBuilder = nullptr;
654
655 // For Context Sensitive summary.
656 std::unique_ptr<IndexedInstrProf::Summary> TheCSSummary = nullptr;
657 if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive)) {
658 TheCSSummary = IndexedInstrProf::allocSummary(TotalSize: SummarySize);
659 std::unique_ptr<ProfileSummary> CSPS = CSISB.getSummary();
660 setSummary(TheSummary: TheCSSummary.get(), PS&: *CSPS);
661 }
662 InfoObj->CSSummaryBuilder = nullptr;
663
664 SmallVector<uint64_t, 8> HeaderOffsets = {HashTableStart, MemProfSectionStart,
665 BinaryIdSectionStart,
666 TemporalProfTracesSectionStart};
667 if (!WritePrevVersion)
668 HeaderOffsets.push_back(Elt: VTableNamesSectionStart);
669
670 PatchItem PatchItems[] = {
671 // Patch the Header fields
672 {.Pos: BackPatchStartOffset, .D: HeaderOffsets},
673 // Patch the summary data.
674 {.Pos: SummaryOffset,
675 .D: ArrayRef<uint64_t>(reinterpret_cast<uint64_t *>(TheSummary.get()),
676 SummarySize / sizeof(uint64_t))},
677 {.Pos: CSSummaryOffset,
678 .D: ArrayRef<uint64_t>(reinterpret_cast<uint64_t *>(TheCSSummary.get()),
679 CSSummarySize)}};
680
681 OS.patch(P: PatchItems);
682
683 for (const auto &I : FunctionData)
684 for (const auto &F : I.getValue())
685 if (Error E = validateRecord(Func: F.second))
686 return E;
687
688 return Error::success();
689}
690
691Error InstrProfWriter::write(raw_fd_ostream &OS) {
692 // Write the hash table.
693 ProfOStream POS(OS);
694 return writeImpl(OS&: POS);
695}
696
697Error InstrProfWriter::write(raw_string_ostream &OS) {
698 ProfOStream POS(OS);
699 return writeImpl(OS&: POS);
700}
701
702std::unique_ptr<MemoryBuffer> InstrProfWriter::writeBuffer() {
703 std::string Data;
704 raw_string_ostream OS(Data);
705 // Write the hash table.
706 if (Error E = write(OS))
707 return nullptr;
708 // Return this in an aligned memory buffer.
709 return MemoryBuffer::getMemBufferCopy(InputData: Data);
710}
711
712static const char *ValueProfKindStr[] = {
713#define VALUE_PROF_KIND(Enumerator, Value, Descr) #Enumerator,
714#include "llvm/ProfileData/InstrProfData.inc"
715};
716
717Error InstrProfWriter::validateRecord(const InstrProfRecord &Func) {
718 for (uint32_t VK = 0; VK <= IPVK_Last; VK++) {
719 if (VK == IPVK_IndirectCallTarget || VK == IPVK_VTableTarget)
720 continue;
721 uint32_t NS = Func.getNumValueSites(ValueKind: VK);
722 for (uint32_t S = 0; S < NS; S++) {
723 DenseSet<uint64_t> SeenValues;
724 for (const auto &V : Func.getValueArrayForSite(ValueKind: VK, Site: S))
725 if (!SeenValues.insert(V: V.Value).second)
726 return make_error<InstrProfError>(Args: instrprof_error::invalid_prof);
727 }
728 }
729
730 return Error::success();
731}
732
733void InstrProfWriter::writeRecordInText(StringRef Name, uint64_t Hash,
734 const InstrProfRecord &Func,
735 InstrProfSymtab &Symtab,
736 raw_fd_ostream &OS) {
737 OS << Name << "\n";
738 OS << "# Func Hash:\n" << Hash << "\n";
739 OS << "# Num Counters:\n" << Func.Counts.size() << "\n";
740 OS << "# Counter Values:\n";
741 for (uint64_t Count : Func.Counts)
742 OS << Count << "\n";
743
744 if (Func.BitmapBytes.size() > 0) {
745 OS << "# Num Bitmap Bytes:\n$" << Func.BitmapBytes.size() << "\n";
746 OS << "# Bitmap Byte Values:\n";
747 for (uint8_t Byte : Func.BitmapBytes) {
748 OS << "0x";
749 OS.write_hex(N: Byte);
750 OS << "\n";
751 }
752 OS << "\n";
753 }
754
755 uint32_t NumValueKinds = Func.getNumValueKinds();
756 if (!NumValueKinds) {
757 OS << "\n";
758 return;
759 }
760
761 OS << "# Num Value Kinds:\n" << Func.getNumValueKinds() << "\n";
762 for (uint32_t VK = 0; VK < IPVK_Last + 1; VK++) {
763 uint32_t NS = Func.getNumValueSites(ValueKind: VK);
764 if (!NS)
765 continue;
766 OS << "# ValueKind = " << ValueProfKindStr[VK] << ":\n" << VK << "\n";
767 OS << "# NumValueSites:\n" << NS << "\n";
768 for (uint32_t S = 0; S < NS; S++) {
769 auto VD = Func.getValueArrayForSite(ValueKind: VK, Site: S);
770 OS << VD.size() << "\n";
771 for (const auto &V : VD) {
772 if (VK == IPVK_IndirectCallTarget || VK == IPVK_VTableTarget)
773 OS << Symtab.getFuncOrVarNameIfDefined(MD5Hash: V.Value) << ":" << V.Count
774 << "\n";
775 else
776 OS << V.Value << ":" << V.Count << "\n";
777 }
778 }
779 }
780
781 OS << "\n";
782}
783
784Error InstrProfWriter::writeText(raw_fd_ostream &OS) {
785 // Check CS first since it implies an IR level profile.
786 if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive))
787 OS << "# CSIR level Instrumentation Flag\n:csir\n";
788 else if (static_cast<bool>(ProfileKind & InstrProfKind::IRInstrumentation))
789 OS << "# IR level Instrumentation Flag\n:ir\n";
790
791 if (static_cast<bool>(ProfileKind &
792 InstrProfKind::FunctionEntryInstrumentation))
793 OS << "# Always instrument the function entry block\n:entry_first\n";
794 if (static_cast<bool>(ProfileKind &
795 InstrProfKind::LoopEntriesInstrumentation))
796 OS << "# Always instrument the loop entry "
797 "blocks\n:instrument_loop_entries\n";
798 if (static_cast<bool>(ProfileKind & InstrProfKind::SingleByteCoverage))
799 OS << "# Instrument block coverage\n:single_byte_coverage\n";
800 InstrProfSymtab Symtab;
801
802 using FuncPair = detail::DenseMapPair<uint64_t, InstrProfRecord>;
803 using RecordType = std::pair<StringRef, FuncPair>;
804 SmallVector<RecordType, 4> OrderedFuncData;
805
806 for (const auto &I : FunctionData) {
807 if (shouldEncodeData(PD: I.getValue())) {
808 if (Error E = Symtab.addFuncName(FuncName: I.getKey()))
809 return E;
810 for (const auto &Func : I.getValue())
811 OrderedFuncData.push_back(Elt: std::make_pair(x: I.getKey(), y: Func));
812 }
813 }
814
815 for (const auto &VTableName : VTableNames)
816 if (Error E = Symtab.addVTableName(VTableName: VTableName.getKey()))
817 return E;
818
819 if (static_cast<bool>(ProfileKind & InstrProfKind::TemporalProfile))
820 writeTextTemporalProfTraceData(OS, Symtab);
821
822 llvm::sort(C&: OrderedFuncData, Comp: [](const RecordType &A, const RecordType &B) {
823 return std::tie(args: A.first, args: A.second.first) <
824 std::tie(args: B.first, args: B.second.first);
825 });
826
827 for (const auto &record : OrderedFuncData) {
828 const StringRef &Name = record.first;
829 const FuncPair &Func = record.second;
830 writeRecordInText(Name, Hash: Func.first, Func: Func.second, Symtab, OS);
831 }
832
833 for (const auto &record : OrderedFuncData) {
834 const FuncPair &Func = record.second;
835 if (Error E = validateRecord(Func: Func.second))
836 return E;
837 }
838
839 return Error::success();
840}
841
842void InstrProfWriter::writeTextTemporalProfTraceData(raw_fd_ostream &OS,
843 InstrProfSymtab &Symtab) {
844 OS << ":temporal_prof_traces\n";
845 OS << "# Num Temporal Profile Traces:\n" << TemporalProfTraces.size() << "\n";
846 OS << "# Temporal Profile Trace Stream Size:\n"
847 << TemporalProfTraceStreamSize << "\n";
848 for (auto &Trace : TemporalProfTraces) {
849 OS << "# Weight:\n" << Trace.Weight << "\n";
850 for (auto &NameRef : Trace.FunctionNameRefs)
851 OS << Symtab.getFuncOrVarName(MD5Hash: NameRef) << ",";
852 OS << "\n";
853 }
854 OS << "\n";
855}
856