1//===- InstrProfWriter.cpp - Instrumented profiling writer ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains support for writing profiling data for clang's
10// instrumentation based PGO and coverage.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/ProfileData/InstrProfWriter.h"
15#include "llvm/ADT/STLExtras.h"
16#include "llvm/ADT/StringRef.h"
17#include "llvm/IR/ProfileSummary.h"
18#include "llvm/ProfileData/DataAccessProf.h"
19#include "llvm/ProfileData/IndexedMemProfData.h"
20#include "llvm/ProfileData/InstrProf.h"
21#include "llvm/ProfileData/ProfileCommon.h"
22#include "llvm/Support/Compression.h"
23#include "llvm/Support/EndianStream.h"
24#include "llvm/Support/Error.h"
25#include "llvm/Support/MemoryBuffer.h"
26#include "llvm/Support/OnDiskHashTable.h"
27#include "llvm/Support/raw_ostream.h"
28#include <cstdint>
29#include <ctime>
30#include <memory>
31#include <string>
32#include <tuple>
33#include <utility>
34#include <vector>
35
36using namespace llvm;
37
38namespace llvm {
39
40class InstrProfRecordWriterTrait {
41public:
42 using key_type = StringRef;
43 using key_type_ref = StringRef;
44
45 using data_type = const InstrProfWriter::ProfilingData *const;
46 using data_type_ref = const InstrProfWriter::ProfilingData *const;
47
48 using hash_value_type = uint64_t;
49 using offset_type = uint64_t;
50
51 llvm::endianness ValueProfDataEndianness = llvm::endianness::little;
52 InstrProfSummaryBuilder *SummaryBuilder;
53 InstrProfSummaryBuilder *CSSummaryBuilder;
54
55 InstrProfRecordWriterTrait() = default;
56
57 static hash_value_type ComputeHash(key_type_ref K) {
58 return IndexedInstrProf::ComputeHash(K);
59 }
60
61 static std::pair<offset_type, offset_type>
62 EmitKeyDataLength(raw_ostream &Out, key_type_ref K, data_type_ref V) {
63 using namespace support;
64
65 endian::Writer LE(Out, llvm::endianness::little);
66
67 offset_type N = K.size();
68 LE.write<offset_type>(Val: N);
69
70 offset_type M = 0;
71 for (const auto &ProfileData : *V) {
72 const InstrProfRecord &ProfRecord = ProfileData.second;
73 M += sizeof(uint64_t); // The function hash
74 M += sizeof(uint64_t); // The size of the Counts vector
75 M += ProfRecord.Counts.size() * sizeof(uint64_t);
76 M += sizeof(uint64_t); // The size of the Bitmap vector
77 M += ProfRecord.BitmapBytes.size() * sizeof(uint64_t);
78
79 // Value data
80 M += ValueProfData::getSize(Record: ProfileData.second);
81 }
82 LE.write<offset_type>(Val: M);
83
84 return std::make_pair(x&: N, y&: M);
85 }
86
87 void EmitKey(raw_ostream &Out, key_type_ref K, offset_type N) {
88 Out.write(Ptr: K.data(), Size: N);
89 }
90
91 void EmitData(raw_ostream &Out, key_type_ref, data_type_ref V, offset_type) {
92 using namespace support;
93
94 endian::Writer LE(Out, llvm::endianness::little);
95 for (const auto &ProfileData : *V) {
96 const InstrProfRecord &ProfRecord = ProfileData.second;
97 if (NamedInstrProfRecord::hasCSFlagInHash(FuncHash: ProfileData.first))
98 CSSummaryBuilder->addRecord(ProfRecord);
99 else
100 SummaryBuilder->addRecord(ProfRecord);
101
102 LE.write<uint64_t>(Val: ProfileData.first); // Function hash
103 LE.write<uint64_t>(Val: ProfRecord.Counts.size());
104 for (uint64_t I : ProfRecord.Counts)
105 LE.write<uint64_t>(Val: I);
106
107 LE.write<uint64_t>(Val: ProfRecord.BitmapBytes.size());
108 for (uint64_t I : ProfRecord.BitmapBytes)
109 LE.write<uint64_t>(Val: I);
110
111 // Write value data
112 std::unique_ptr<ValueProfData> VDataPtr =
113 ValueProfData::serializeFrom(Record: ProfileData.second);
114 uint32_t S = VDataPtr->getSize();
115 VDataPtr->swapBytesFromHost(Endianness: ValueProfDataEndianness);
116 Out.write(Ptr: (const char *)VDataPtr.get(), Size: S);
117 }
118 }
119};
120
121} // end namespace llvm
122
123InstrProfWriter::InstrProfWriter(
124 bool Sparse, uint64_t TemporalProfTraceReservoirSize,
125 uint64_t MaxTemporalProfTraceLength, bool WritePrevVersion,
126 memprof::IndexedVersion MemProfVersionRequested, bool MemProfFullSchema,
127 bool MemprofGenerateRandomHotness,
128 unsigned MemprofGenerateRandomHotnessSeed)
129 : Sparse(Sparse), MaxTemporalProfTraceLength(MaxTemporalProfTraceLength),
130 TemporalProfTraceReservoirSize(TemporalProfTraceReservoirSize),
131 InfoObj(new InstrProfRecordWriterTrait()),
132 WritePrevVersion(WritePrevVersion),
133 MemProfVersionRequested(MemProfVersionRequested),
134 MemProfFullSchema(MemProfFullSchema),
135 MemprofGenerateRandomHotness(MemprofGenerateRandomHotness) {
136 // Set up the random number seed if requested.
137 if (MemprofGenerateRandomHotness) {
138 unsigned seed = MemprofGenerateRandomHotnessSeed
139 ? MemprofGenerateRandomHotnessSeed
140 : std::time(timer: nullptr);
141 errs() << "random hotness seed = " << seed << "\n";
142 std::srand(seed: seed);
143 }
144}
145
146InstrProfWriter::~InstrProfWriter() { delete InfoObj; }
147
148// Internal interface for testing purpose only.
149void InstrProfWriter::setValueProfDataEndianness(llvm::endianness Endianness) {
150 InfoObj->ValueProfDataEndianness = Endianness;
151}
152
153void InstrProfWriter::setOutputSparse(bool Sparse) { this->Sparse = Sparse; }
154
155void InstrProfWriter::addRecord(NamedInstrProfRecord &&I, uint64_t Weight,
156 function_ref<void(Error)> Warn) {
157 auto Name = I.Name;
158 auto Hash = I.Hash;
159 addRecord(Name, Hash, I: std::move(I), Weight, Warn);
160}
161
162void InstrProfWriter::overlapRecord(NamedInstrProfRecord &&Other,
163 OverlapStats &Overlap,
164 OverlapStats &FuncLevelOverlap,
165 const OverlapFuncFilters &FuncFilter) {
166 auto Name = Other.Name;
167 auto Hash = Other.Hash;
168 Other.accumulateCounts(Sum&: FuncLevelOverlap.Test);
169 auto It = FunctionData.find(Key: Name);
170 if (It == FunctionData.end()) {
171 Overlap.addOneUnique(UniqueFunc: FuncLevelOverlap.Test);
172 return;
173 }
174 if (FuncLevelOverlap.Test.CountSum < 1.0f) {
175 Overlap.Overlap.NumEntries += 1;
176 return;
177 }
178 auto &ProfileDataMap = It->second;
179 auto [Where, NewFunc] = ProfileDataMap.try_emplace(Key: Hash);
180 if (NewFunc) {
181 Overlap.addOneMismatch(MismatchFunc: FuncLevelOverlap.Test);
182 return;
183 }
184 InstrProfRecord &Dest = Where->second;
185
186 uint64_t ValueCutoff = FuncFilter.ValueCutoff;
187 if (!FuncFilter.NameFilter.empty() && Name.contains(Other: FuncFilter.NameFilter))
188 ValueCutoff = 0;
189
190 Dest.overlap(Other, Overlap, FuncLevelOverlap, ValueCutoff);
191}
192
193void InstrProfWriter::addRecord(StringRef Name, uint64_t Hash,
194 InstrProfRecord &&I, uint64_t Weight,
195 function_ref<void(Error)> Warn) {
196 auto &ProfileDataMap = FunctionData[Name];
197
198 auto [Where, NewFunc] = ProfileDataMap.try_emplace(Key: Hash);
199 InstrProfRecord &Dest = Where->second;
200
201 auto MapWarn = [&](instrprof_error E) {
202 Warn(make_error<InstrProfError>(Args&: E));
203 };
204
205 if (NewFunc) {
206 // We've never seen a function with this name and hash, add it.
207 Dest = std::move(I);
208 if (Weight > 1)
209 Dest.scale(N: Weight, D: 1, Warn: MapWarn);
210 } else {
211 // We're updating a function we've seen before.
212 Dest.merge(Other&: I, Weight, Warn: MapWarn);
213 }
214
215 Dest.sortValueData();
216}
217
218void InstrProfWriter::addMemProfRecord(
219 const Function::GUID Id, const memprof::IndexedMemProfRecord &Record) {
220 auto NewRecord = Record;
221 // Provoke random hotness values if requested. We specify the lifetime access
222 // density and lifetime length that will result in a cold or not cold hotness.
223 // See the logic in getAllocType() in Analysis/MemoryProfileInfo.cpp.
224 if (MemprofGenerateRandomHotness) {
225 for (auto &Alloc : NewRecord.AllocSites) {
226 // To get a not cold context, set the lifetime access density to the
227 // maximum value and the lifetime to 0.
228 uint64_t NewTLAD = std::numeric_limits<uint64_t>::max();
229 uint64_t NewTL = 0;
230 bool IsCold = std::rand() % 2;
231 if (IsCold) {
232 // To get a cold context, set the lifetime access density to 0 and the
233 // lifetime to the maximum value.
234 NewTLAD = 0;
235 NewTL = std::numeric_limits<uint64_t>::max();
236 }
237 Alloc.Info.setTotalLifetimeAccessDensity(NewTLAD);
238 Alloc.Info.setTotalLifetime(NewTL);
239 }
240 }
241 MemProfSumBuilder.addRecord(NewRecord);
242 auto [Iter, Inserted] = MemProfData.Records.insert(KV: {Id, NewRecord});
243 // If we inserted a new record then we are done.
244 if (Inserted) {
245 return;
246 }
247 memprof::IndexedMemProfRecord &Existing = Iter->second;
248 Existing.merge(Other: NewRecord);
249}
250
251bool InstrProfWriter::addMemProfFrame(const memprof::FrameId Id,
252 const memprof::Frame &Frame,
253 function_ref<void(Error)> Warn) {
254 auto [Iter, Inserted] = MemProfData.Frames.insert(KV: {Id, Frame});
255 // If a mapping already exists for the current frame id and it does not
256 // match the new mapping provided then reset the existing contents and bail
257 // out. We don't support the merging of memprof data whose Frame -> Id
258 // mapping across profiles is inconsistent.
259 if (!Inserted && Iter->second != Frame) {
260 Warn(make_error<InstrProfError>(Args: instrprof_error::malformed,
261 Args: "frame to id mapping mismatch"));
262 return false;
263 }
264 return true;
265}
266
267bool InstrProfWriter::addMemProfCallStack(
268 const memprof::CallStackId CSId,
269 const llvm::SmallVector<memprof::FrameId> &CallStack,
270 function_ref<void(Error)> Warn) {
271 auto [Iter, Inserted] = MemProfData.CallStacks.insert(KV: {CSId, CallStack});
272 // If a mapping already exists for the current call stack id and it does not
273 // match the new mapping provided then reset the existing contents and bail
274 // out. We don't support the merging of memprof data whose CallStack -> Id
275 // mapping across profiles is inconsistent.
276 if (!Inserted && Iter->second != CallStack) {
277 Warn(make_error<InstrProfError>(Args: instrprof_error::malformed,
278 Args: "call stack to id mapping mismatch"));
279 return false;
280 }
281 return true;
282}
283
284bool InstrProfWriter::addMemProfData(memprof::IndexedMemProfData Incoming,
285 function_ref<void(Error)> Warn) {
286 // Return immediately if everything is empty.
287 if (Incoming.Frames.empty() && Incoming.CallStacks.empty() &&
288 Incoming.Records.empty())
289 return true;
290
291 // Otherwise, every component must be non-empty.
292 assert(!Incoming.Frames.empty() && !Incoming.CallStacks.empty() &&
293 !Incoming.Records.empty());
294
295 if (MemProfData.Frames.empty())
296 MemProfData.Frames = std::move(Incoming.Frames);
297 else
298 for (const auto &[Id, F] : Incoming.Frames)
299 if (addMemProfFrame(Id, Frame: F, Warn))
300 return false;
301
302 if (MemProfData.CallStacks.empty())
303 MemProfData.CallStacks = std::move(Incoming.CallStacks);
304 else
305 for (const auto &[CSId, CS] : Incoming.CallStacks)
306 if (addMemProfCallStack(CSId, CallStack: CS, Warn))
307 return false;
308
309 // Add one record at a time if randomization is requested.
310 if (MemProfData.Records.empty() && !MemprofGenerateRandomHotness) {
311 // Need to manually add each record to the builder, which is otherwise done
312 // in addMemProfRecord.
313 for (const auto &[GUID, Record] : Incoming.Records)
314 MemProfSumBuilder.addRecord(Record);
315 MemProfData.Records = std::move(Incoming.Records);
316 } else {
317 for (const auto &[GUID, Record] : Incoming.Records)
318 addMemProfRecord(Id: GUID, Record);
319 }
320
321 return true;
322}
323
324void InstrProfWriter::addBinaryIds(ArrayRef<llvm::object::BuildID> BIs) {
325 llvm::append_range(C&: BinaryIds, R&: BIs);
326}
327
328void InstrProfWriter::addDataAccessProfData(
329 std::unique_ptr<memprof::DataAccessProfData> DataAccessProfDataIn) {
330 DataAccessProfileData = std::move(DataAccessProfDataIn);
331}
332
333void InstrProfWriter::addTemporalProfileTraces(
334 SmallVectorImpl<TemporalProfTraceTy> &SrcTraces, uint64_t SrcStreamSize) {
335 if (TemporalProfTraces.size() > TemporalProfTraceReservoirSize)
336 TemporalProfTraces.truncate(N: TemporalProfTraceReservoirSize);
337 for (auto &Trace : SrcTraces)
338 if (Trace.FunctionNameRefs.size() > MaxTemporalProfTraceLength)
339 Trace.FunctionNameRefs.resize(new_size: MaxTemporalProfTraceLength);
340 llvm::erase_if(C&: SrcTraces, P: [](auto &T) { return T.FunctionNameRefs.empty(); });
341 // If there are no source traces, it is probably because
342 // --temporal-profile-max-trace-length=0 was set to deliberately remove all
343 // traces. In that case, we do not want to increase the stream size
344 if (SrcTraces.empty())
345 return;
346 // Add traces until our reservoir is full or we run out of source traces
347 auto SrcTraceIt = SrcTraces.begin();
348 while (TemporalProfTraces.size() < TemporalProfTraceReservoirSize &&
349 SrcTraceIt < SrcTraces.end())
350 TemporalProfTraces.push_back(Elt: *SrcTraceIt++);
351 // Our reservoir is full, we need to sample the source stream
352 llvm::shuffle(first: SrcTraceIt, last: SrcTraces.end(), g&: RNG);
353 for (uint64_t I = TemporalProfTraces.size();
354 I < SrcStreamSize && SrcTraceIt < SrcTraces.end(); I++) {
355 std::uniform_int_distribution<uint64_t> Distribution(0, I);
356 uint64_t RandomIndex = Distribution(RNG);
357 if (RandomIndex < TemporalProfTraces.size())
358 TemporalProfTraces[RandomIndex] = *SrcTraceIt++;
359 }
360 TemporalProfTraceStreamSize += SrcStreamSize;
361}
362
363void InstrProfWriter::mergeRecordsFromWriter(InstrProfWriter &&IPW,
364 function_ref<void(Error)> Warn) {
365 for (auto &I : IPW.FunctionData)
366 for (auto &Func : I.getValue())
367 addRecord(Name: I.getKey(), Hash: Func.first, I: std::move(Func.second), Weight: 1, Warn);
368
369 BinaryIds.reserve(n: BinaryIds.size() + IPW.BinaryIds.size());
370 for (auto &I : IPW.BinaryIds)
371 addBinaryIds(BIs: I);
372
373 addTemporalProfileTraces(SrcTraces&: IPW.TemporalProfTraces,
374 SrcStreamSize: IPW.TemporalProfTraceStreamSize);
375
376 MemProfData.Frames.reserve(NumEntries: IPW.MemProfData.Frames.size());
377 for (auto &[FrameId, Frame] : IPW.MemProfData.Frames) {
378 // If we weren't able to add the frame mappings then it doesn't make sense
379 // to try to merge the records from this profile.
380 if (!addMemProfFrame(Id: FrameId, Frame, Warn))
381 return;
382 }
383
384 MemProfData.CallStacks.reserve(NumEntries: IPW.MemProfData.CallStacks.size());
385 for (auto &[CSId, CallStack] : IPW.MemProfData.CallStacks) {
386 if (!addMemProfCallStack(CSId, CallStack, Warn))
387 return;
388 }
389
390 MemProfData.Records.reserve(NumEntries: IPW.MemProfData.Records.size());
391 for (auto &[GUID, Record] : IPW.MemProfData.Records) {
392 addMemProfRecord(Id: GUID, Record);
393 }
394}
395
396bool InstrProfWriter::shouldEncodeData(const ProfilingData &PD) {
397 if (!Sparse)
398 return true;
399 for (const auto &Func : PD) {
400 const InstrProfRecord &IPR = Func.second;
401 if (llvm::any_of(Range: IPR.Counts, P: [](uint64_t Count) { return Count > 0; }))
402 return true;
403 if (llvm::any_of(Range: IPR.BitmapBytes, P: [](uint8_t Byte) { return Byte > 0; }))
404 return true;
405 }
406 return false;
407}
408
409static void setSummary(IndexedInstrProf::Summary *TheSummary,
410 ProfileSummary &PS) {
411 using namespace IndexedInstrProf;
412
413 const std::vector<ProfileSummaryEntry> &Res = PS.getDetailedSummary();
414 TheSummary->NumSummaryFields = Summary::NumKinds;
415 TheSummary->NumCutoffEntries = Res.size();
416 TheSummary->set(K: Summary::MaxFunctionCount, V: PS.getMaxFunctionCount());
417 TheSummary->set(K: Summary::MaxBlockCount, V: PS.getMaxCount());
418 TheSummary->set(K: Summary::MaxInternalBlockCount, V: PS.getMaxInternalCount());
419 TheSummary->set(K: Summary::TotalBlockCount, V: PS.getTotalCount());
420 TheSummary->set(K: Summary::TotalNumBlocks, V: PS.getNumCounts());
421 TheSummary->set(K: Summary::TotalNumFunctions, V: PS.getNumFunctions());
422 for (unsigned I = 0; I < Res.size(); I++)
423 TheSummary->setEntry(I, E: Res[I]);
424}
425
426uint64_t InstrProfWriter::writeHeader(const IndexedInstrProf::Header &Header,
427 const bool WritePrevVersion,
428 ProfOStream &OS) {
429 // Only write out the first four fields.
430 for (int I = 0; I < 4; I++)
431 OS.write(V: reinterpret_cast<const uint64_t *>(&Header)[I]);
432
433 // Remember the offset of the remaining fields to allow back patching later.
434 auto BackPatchStartOffset = OS.tell();
435
436 // Reserve the space for back patching later.
437 OS.write(V: 0); // HashOffset
438 OS.write(V: 0); // MemProfOffset
439 OS.write(V: 0); // BinaryIdOffset
440 OS.write(V: 0); // TemporalProfTracesOffset
441 if (!WritePrevVersion)
442 OS.write(V: 0); // VTableNamesOffset
443
444 return BackPatchStartOffset;
445}
446
447Error InstrProfWriter::writeBinaryIds(ProfOStream &OS) {
448 // BinaryIdSection has two parts:
449 // 1. uint64_t BinaryIdsSectionSize
450 // 2. list of binary ids that consist of:
451 // a. uint64_t BinaryIdLength
452 // b. uint8_t BinaryIdData
453 // c. uint8_t Padding (if necessary)
454 // Calculate size of binary section.
455 uint64_t BinaryIdsSectionSize = 0;
456
457 // Remove duplicate binary ids.
458 llvm::sort(C&: BinaryIds);
459 BinaryIds.erase(first: llvm::unique(R&: BinaryIds), last: BinaryIds.end());
460
461 for (const auto &BI : BinaryIds) {
462 // Increment by binary id length data type size.
463 BinaryIdsSectionSize += sizeof(uint64_t);
464 // Increment by binary id data length, aligned to 8 bytes.
465 BinaryIdsSectionSize += alignToPowerOf2(Value: BI.size(), Align: sizeof(uint64_t));
466 }
467 // Write binary ids section size.
468 OS.write(V: BinaryIdsSectionSize);
469
470 for (const auto &BI : BinaryIds) {
471 uint64_t BILen = BI.size();
472 // Write binary id length.
473 OS.write(V: BILen);
474 // Write binary id data.
475 for (unsigned K = 0; K < BILen; K++)
476 OS.writeByte(V: BI[K]);
477 // Write padding if necessary.
478 uint64_t PaddingSize = alignToPowerOf2(Value: BILen, Align: sizeof(uint64_t)) - BILen;
479 for (unsigned K = 0; K < PaddingSize; K++)
480 OS.writeByte(V: 0);
481 }
482
483 return Error::success();
484}
485
486Error InstrProfWriter::writeVTableNames(ProfOStream &OS) {
487 std::vector<std::string> VTableNameStrs;
488 for (StringRef VTableName : VTableNames.keys())
489 VTableNameStrs.push_back(x: VTableName.str());
490
491 std::string CompressedVTableNames;
492 if (!VTableNameStrs.empty())
493 if (Error E = collectGlobalObjectNameStrings(
494 NameStrs: VTableNameStrs, doCompression: compression::zlib::isAvailable(),
495 Result&: CompressedVTableNames))
496 return E;
497
498 const uint64_t CompressedStringLen = CompressedVTableNames.length();
499
500 // Record the length of compressed string.
501 OS.write(V: CompressedStringLen);
502
503 // Write the chars in compressed strings.
504 for (auto &c : CompressedVTableNames)
505 OS.writeByte(V: static_cast<uint8_t>(c));
506
507 // Pad up to a multiple of 8.
508 // InstrProfReader could read bytes according to 'CompressedStringLen'.
509 const uint64_t PaddedLength = alignTo(Value: CompressedStringLen, Align: 8);
510
511 for (uint64_t K = CompressedStringLen; K < PaddedLength; K++)
512 OS.writeByte(V: 0);
513
514 return Error::success();
515}
516
517Error InstrProfWriter::writeImpl(ProfOStream &OS) {
518 using namespace IndexedInstrProf;
519 using namespace support;
520
521 OnDiskChainedHashTableGenerator<InstrProfRecordWriterTrait> Generator;
522
523 InstrProfSummaryBuilder ISB(ProfileSummaryBuilder::DefaultCutoffs);
524 InfoObj->SummaryBuilder = &ISB;
525 InstrProfSummaryBuilder CSISB(ProfileSummaryBuilder::DefaultCutoffs);
526 InfoObj->CSSummaryBuilder = &CSISB;
527
528 // Populate the hash table generator.
529 SmallVector<std::pair<StringRef, const ProfilingData *>> OrderedData;
530 for (const auto &I : FunctionData)
531 if (shouldEncodeData(PD: I.getValue()))
532 OrderedData.emplace_back(Args: (I.getKey()), Args: &I.getValue());
533 llvm::sort(C&: OrderedData, Comp: less_first());
534 for (const auto &I : OrderedData)
535 Generator.insert(Key: I.first, Data: I.second);
536
537 // Write the header.
538 IndexedInstrProf::Header Header;
539 Header.Version = WritePrevVersion
540 ? IndexedInstrProf::ProfVersion::Version11
541 : IndexedInstrProf::ProfVersion::CurrentVersion;
542 // The WritePrevVersion handling will either need to be removed or updated
543 // if the version is advanced beyond 12.
544 static_assert(IndexedInstrProf::ProfVersion::CurrentVersion ==
545 IndexedInstrProf::ProfVersion::Version13);
546 if (static_cast<bool>(ProfileKind & InstrProfKind::IRInstrumentation))
547 Header.Version |= VARIANT_MASK_IR_PROF;
548 if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive))
549 Header.Version |= VARIANT_MASK_CSIR_PROF;
550 if (static_cast<bool>(ProfileKind &
551 InstrProfKind::FunctionEntryInstrumentation))
552 Header.Version |= VARIANT_MASK_INSTR_ENTRY;
553 if (static_cast<bool>(ProfileKind &
554 InstrProfKind::LoopEntriesInstrumentation))
555 Header.Version |= VARIANT_MASK_INSTR_LOOP_ENTRIES;
556 if (static_cast<bool>(ProfileKind & InstrProfKind::SingleByteCoverage))
557 Header.Version |= VARIANT_MASK_BYTE_COVERAGE;
558 if (static_cast<bool>(ProfileKind & InstrProfKind::FunctionEntryOnly))
559 Header.Version |= VARIANT_MASK_FUNCTION_ENTRY_ONLY;
560 if (static_cast<bool>(ProfileKind & InstrProfKind::MemProf))
561 Header.Version |= VARIANT_MASK_MEMPROF;
562 if (static_cast<bool>(ProfileKind & InstrProfKind::TemporalProfile))
563 Header.Version |= VARIANT_MASK_TEMPORAL_PROF;
564
565 const uint64_t BackPatchStartOffset =
566 writeHeader(Header, WritePrevVersion, OS);
567
568 // Reserve space to write profile summary data.
569 uint32_t NumEntries = ProfileSummaryBuilder::DefaultCutoffs.size();
570 uint32_t SummarySize = Summary::getSize(NumSumFields: Summary::NumKinds, NumCutoffEntries: NumEntries);
571 // Remember the summary offset.
572 uint64_t SummaryOffset = OS.tell();
573 for (unsigned I = 0; I < SummarySize / sizeof(uint64_t); I++)
574 OS.write(V: 0);
575 uint64_t CSSummaryOffset = 0;
576 uint64_t CSSummarySize = 0;
577 if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive)) {
578 CSSummaryOffset = OS.tell();
579 CSSummarySize = SummarySize / sizeof(uint64_t);
580 for (unsigned I = 0; I < CSSummarySize; I++)
581 OS.write(V: 0);
582 }
583
584 // Write the hash table.
585 uint64_t HashTableStart = Generator.Emit(Out&: OS.OS, InfoObj&: *InfoObj);
586
587 // Write the MemProf profile data if we have it.
588 uint64_t MemProfSectionStart = 0;
589 if (static_cast<bool>(ProfileKind & InstrProfKind::MemProf)) {
590 MemProfSectionStart = OS.tell();
591
592 if (auto E = writeMemProf(
593 OS, MemProfData, MemProfVersionRequested, MemProfFullSchema,
594 DataAccessProfileData: std::move(DataAccessProfileData), MemProfSum: MemProfSumBuilder.getSummary()))
595 return E;
596 }
597
598 uint64_t BinaryIdSectionStart = OS.tell();
599 if (auto E = writeBinaryIds(OS))
600 return E;
601
602 uint64_t VTableNamesSectionStart = OS.tell();
603
604 if (!WritePrevVersion)
605 if (Error E = writeVTableNames(OS))
606 return E;
607
608 uint64_t TemporalProfTracesSectionStart = 0;
609 if (static_cast<bool>(ProfileKind & InstrProfKind::TemporalProfile)) {
610 TemporalProfTracesSectionStart = OS.tell();
611 OS.write(V: TemporalProfTraces.size());
612 OS.write(V: TemporalProfTraceStreamSize);
613 for (auto &Trace : TemporalProfTraces) {
614 OS.write(V: Trace.Weight);
615 OS.write(V: Trace.FunctionNameRefs.size());
616 for (auto &NameRef : Trace.FunctionNameRefs)
617 OS.write(V: NameRef);
618 }
619 }
620
621 // Allocate space for data to be serialized out.
622 std::unique_ptr<IndexedInstrProf::Summary> TheSummary =
623 IndexedInstrProf::allocSummary(TotalSize: SummarySize);
624 // Compute the Summary and copy the data to the data
625 // structure to be serialized out (to disk or buffer).
626 std::unique_ptr<ProfileSummary> PS = ISB.getSummary();
627 setSummary(TheSummary: TheSummary.get(), PS&: *PS);
628 InfoObj->SummaryBuilder = nullptr;
629
630 // For Context Sensitive summary.
631 std::unique_ptr<IndexedInstrProf::Summary> TheCSSummary = nullptr;
632 if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive)) {
633 TheCSSummary = IndexedInstrProf::allocSummary(TotalSize: SummarySize);
634 std::unique_ptr<ProfileSummary> CSPS = CSISB.getSummary();
635 setSummary(TheSummary: TheCSSummary.get(), PS&: *CSPS);
636 }
637 InfoObj->CSSummaryBuilder = nullptr;
638
639 SmallVector<uint64_t, 8> HeaderOffsets = {HashTableStart, MemProfSectionStart,
640 BinaryIdSectionStart,
641 TemporalProfTracesSectionStart};
642 if (!WritePrevVersion)
643 HeaderOffsets.push_back(Elt: VTableNamesSectionStart);
644
645 PatchItem PatchItems[] = {
646 // Patch the Header fields
647 {.Pos: BackPatchStartOffset, .D: HeaderOffsets},
648 // Patch the summary data.
649 {.Pos: SummaryOffset,
650 .D: ArrayRef<uint64_t>(reinterpret_cast<uint64_t *>(TheSummary.get()),
651 SummarySize / sizeof(uint64_t))},
652 {.Pos: CSSummaryOffset,
653 .D: ArrayRef<uint64_t>(reinterpret_cast<uint64_t *>(TheCSSummary.get()),
654 CSSummarySize)}};
655
656 OS.patch(P: PatchItems);
657
658 for (const auto &I : FunctionData)
659 for (const auto &F : I.getValue())
660 if (Error E = validateRecord(Func: F.second))
661 return E;
662
663 return Error::success();
664}
665
666Error InstrProfWriter::write(raw_fd_ostream &OS) {
667 // Write the hash table.
668 ProfOStream POS(OS);
669 return writeImpl(OS&: POS);
670}
671
672Error InstrProfWriter::write(raw_string_ostream &OS) {
673 ProfOStream POS(OS);
674 return writeImpl(OS&: POS);
675}
676
677std::unique_ptr<MemoryBuffer> InstrProfWriter::writeBuffer() {
678 std::string Data;
679 raw_string_ostream OS(Data);
680 // Write the hash table.
681 if (Error E = write(OS))
682 return nullptr;
683 // Return this in an aligned memory buffer.
684 return MemoryBuffer::getMemBufferCopy(InputData: Data);
685}
686
687static const char *ValueProfKindStr[] = {
688#define VALUE_PROF_KIND(Enumerator, Value, Descr) #Enumerator,
689#include "llvm/ProfileData/InstrProfData.inc"
690};
691
692Error InstrProfWriter::validateRecord(const InstrProfRecord &Func) {
693 for (uint32_t VK = 0; VK <= IPVK_Last; VK++) {
694 if (VK == IPVK_IndirectCallTarget || VK == IPVK_VTableTarget)
695 continue;
696 uint32_t NS = Func.getNumValueSites(ValueKind: VK);
697 for (uint32_t S = 0; S < NS; S++) {
698 DenseSet<uint64_t> SeenValues;
699 for (const auto &V : Func.getValueArrayForSite(ValueKind: VK, Site: S))
700 if (!SeenValues.insert(V: V.Value).second)
701 return make_error<InstrProfError>(Args: instrprof_error::invalid_prof);
702 }
703 }
704
705 return Error::success();
706}
707
708void InstrProfWriter::writeRecordInText(StringRef Name, uint64_t Hash,
709 const InstrProfRecord &Func,
710 InstrProfSymtab &Symtab,
711 raw_fd_ostream &OS) {
712 OS << Name << "\n";
713 OS << "# Func Hash:\n" << Hash << "\n";
714 OS << "# Num Counters:\n" << Func.Counts.size() << "\n";
715 OS << "# Counter Values:\n";
716 for (uint64_t Count : Func.Counts)
717 OS << Count << "\n";
718
719 if (Func.BitmapBytes.size() > 0) {
720 OS << "# Num Bitmap Bytes:\n$" << Func.BitmapBytes.size() << "\n";
721 OS << "# Bitmap Byte Values:\n";
722 for (uint8_t Byte : Func.BitmapBytes) {
723 OS << "0x";
724 OS.write_hex(N: Byte);
725 OS << "\n";
726 }
727 OS << "\n";
728 }
729
730 uint32_t NumValueKinds = Func.getNumValueKinds();
731 if (!NumValueKinds) {
732 OS << "\n";
733 return;
734 }
735
736 OS << "# Num Value Kinds:\n" << Func.getNumValueKinds() << "\n";
737 for (uint32_t VK = 0; VK < IPVK_Last + 1; VK++) {
738 uint32_t NS = Func.getNumValueSites(ValueKind: VK);
739 if (!NS)
740 continue;
741 OS << "# ValueKind = " << ValueProfKindStr[VK] << ":\n" << VK << "\n";
742 OS << "# NumValueSites:\n" << NS << "\n";
743 for (uint32_t S = 0; S < NS; S++) {
744 auto VD = Func.getValueArrayForSite(ValueKind: VK, Site: S);
745 OS << VD.size() << "\n";
746 for (const auto &V : VD) {
747 if (VK == IPVK_IndirectCallTarget || VK == IPVK_VTableTarget)
748 OS << Symtab.getFuncOrVarNameIfDefined(MD5Hash: V.Value) << ":" << V.Count
749 << "\n";
750 else
751 OS << V.Value << ":" << V.Count << "\n";
752 }
753 }
754 }
755
756 OS << "\n";
757}
758
759Error InstrProfWriter::writeText(raw_fd_ostream &OS) {
760 // Check CS first since it implies an IR level profile.
761 if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive))
762 OS << "# CSIR level Instrumentation Flag\n:csir\n";
763 else if (static_cast<bool>(ProfileKind & InstrProfKind::IRInstrumentation))
764 OS << "# IR level Instrumentation Flag\n:ir\n";
765
766 if (static_cast<bool>(ProfileKind &
767 InstrProfKind::FunctionEntryInstrumentation))
768 OS << "# Always instrument the function entry block\n:entry_first\n";
769 if (static_cast<bool>(ProfileKind &
770 InstrProfKind::LoopEntriesInstrumentation))
771 OS << "# Always instrument the loop entry "
772 "blocks\n:instrument_loop_entries\n";
773 if (static_cast<bool>(ProfileKind & InstrProfKind::SingleByteCoverage))
774 OS << "# Instrument block coverage\n:single_byte_coverage\n";
775 InstrProfSymtab Symtab;
776
777 using FuncPair = detail::DenseMapPair<uint64_t, InstrProfRecord>;
778 using RecordType = std::pair<StringRef, FuncPair>;
779 SmallVector<RecordType, 4> OrderedFuncData;
780
781 for (const auto &I : FunctionData) {
782 if (shouldEncodeData(PD: I.getValue())) {
783 if (Error E = Symtab.addFuncName(FuncName: I.getKey()))
784 return E;
785 for (const auto &Func : I.getValue())
786 OrderedFuncData.push_back(Elt: std::make_pair(x: I.getKey(), y: Func));
787 }
788 }
789
790 for (const auto &VTableName : VTableNames)
791 if (Error E = Symtab.addVTableName(VTableName: VTableName.getKey()))
792 return E;
793
794 if (static_cast<bool>(ProfileKind & InstrProfKind::TemporalProfile))
795 writeTextTemporalProfTraceData(OS, Symtab);
796
797 llvm::sort(C&: OrderedFuncData, Comp: [](const RecordType &A, const RecordType &B) {
798 return std::tie(args: A.first, args: A.second.first) <
799 std::tie(args: B.first, args: B.second.first);
800 });
801
802 for (const auto &record : OrderedFuncData) {
803 const StringRef &Name = record.first;
804 const FuncPair &Func = record.second;
805 writeRecordInText(Name, Hash: Func.first, Func: Func.second, Symtab, OS);
806 }
807
808 for (const auto &record : OrderedFuncData) {
809 const FuncPair &Func = record.second;
810 if (Error E = validateRecord(Func: Func.second))
811 return E;
812 }
813
814 return Error::success();
815}
816
817void InstrProfWriter::writeTextTemporalProfTraceData(raw_fd_ostream &OS,
818 InstrProfSymtab &Symtab) {
819 OS << ":temporal_prof_traces\n";
820 OS << "# Num Temporal Profile Traces:\n" << TemporalProfTraces.size() << "\n";
821 OS << "# Temporal Profile Trace Stream Size:\n"
822 << TemporalProfTraceStreamSize << "\n";
823 for (auto &Trace : TemporalProfTraces) {
824 OS << "# Weight:\n" << Trace.Weight << "\n";
825 for (auto &NameRef : Trace.FunctionNameRefs)
826 OS << Symtab.getFuncOrVarName(MD5Hash: NameRef) << ",";
827 OS << "\n";
828 }
829 OS << "\n";
830}
831