1//===- InstrProfWriter.cpp - Instrumented profiling writer ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains support for writing profiling data for clang's
10// instrumentation based PGO and coverage.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/ProfileData/InstrProfWriter.h"
15#include "llvm/ADT/STLExtras.h"
16#include "llvm/ADT/SetVector.h"
17#include "llvm/ADT/StringRef.h"
18#include "llvm/IR/ProfileSummary.h"
19#include "llvm/ProfileData/DataAccessProf.h"
20#include "llvm/ProfileData/IndexedMemProfData.h"
21#include "llvm/ProfileData/InstrProf.h"
22#include "llvm/ProfileData/ProfileCommon.h"
23#include "llvm/Support/Compression.h"
24#include "llvm/Support/EndianStream.h"
25#include "llvm/Support/Error.h"
26#include "llvm/Support/MemoryBuffer.h"
27#include "llvm/Support/OnDiskHashTable.h"
28#include "llvm/Support/raw_ostream.h"
29#include <cstdint>
30#include <ctime>
31#include <memory>
32#include <string>
33#include <tuple>
34#include <utility>
35#include <vector>
36
37using namespace llvm;
38
39namespace llvm {
40
41class InstrProfRecordWriterTrait {
42public:
43 using key_type = StringRef;
44 using key_type_ref = StringRef;
45
46 using data_type = const InstrProfWriter::ProfilingData *const;
47 using data_type_ref = const InstrProfWriter::ProfilingData *const;
48
49 using hash_value_type = uint64_t;
50 using offset_type = uint64_t;
51
52 llvm::endianness ValueProfDataEndianness = llvm::endianness::little;
53 InstrProfSummaryBuilder *SummaryBuilder;
54 InstrProfSummaryBuilder *CSSummaryBuilder;
55
56 InstrProfRecordWriterTrait() = default;
57
58 static hash_value_type ComputeHash(key_type_ref K) {
59 return IndexedInstrProf::ComputeHash(K);
60 }
61
62 static std::pair<offset_type, offset_type>
63 EmitKeyDataLength(raw_ostream &Out, key_type_ref K, data_type_ref V) {
64 using namespace support;
65
66 endian::Writer LE(Out, llvm::endianness::little);
67
68 offset_type N = K.size();
69 LE.write<offset_type>(Val: N);
70
71 offset_type M = 0;
72 for (const auto &ProfileData : *V) {
73 const InstrProfRecord &ProfRecord = ProfileData.second;
74 M += sizeof(uint64_t); // The function hash
75 M += sizeof(uint64_t); // The size of the Counts vector
76 M += ProfRecord.Counts.size() * sizeof(uint64_t);
77 M += sizeof(uint64_t); // The size of the Bitmap vector
78 M += ProfRecord.BitmapBytes.size() * sizeof(uint64_t);
79
80 // Value data
81 M += ValueProfData::getSize(Record: ProfileData.second);
82 }
83 LE.write<offset_type>(Val: M);
84
85 return std::make_pair(x&: N, y&: M);
86 }
87
88 void EmitKey(raw_ostream &Out, key_type_ref K, offset_type N) {
89 Out.write(Ptr: K.data(), Size: N);
90 }
91
92 void EmitData(raw_ostream &Out, key_type_ref, data_type_ref V, offset_type) {
93 using namespace support;
94
95 endian::Writer LE(Out, llvm::endianness::little);
96 for (const auto &ProfileData : *V) {
97 const InstrProfRecord &ProfRecord = ProfileData.second;
98 if (NamedInstrProfRecord::hasCSFlagInHash(FuncHash: ProfileData.first))
99 CSSummaryBuilder->addRecord(ProfRecord);
100 else
101 SummaryBuilder->addRecord(ProfRecord);
102
103 LE.write<uint64_t>(Val: ProfileData.first); // Function hash
104 LE.write<uint64_t>(Val: ProfRecord.Counts.size());
105 for (uint64_t I : ProfRecord.Counts)
106 LE.write<uint64_t>(Val: I);
107
108 LE.write<uint64_t>(Val: ProfRecord.BitmapBytes.size());
109 for (uint64_t I : ProfRecord.BitmapBytes)
110 LE.write<uint64_t>(Val: I);
111
112 // Write value data
113 std::unique_ptr<ValueProfData> VDataPtr =
114 ValueProfData::serializeFrom(Record: ProfileData.second);
115 uint32_t S = VDataPtr->getSize();
116 VDataPtr->swapBytesFromHost(Endianness: ValueProfDataEndianness);
117 Out.write(Ptr: (const char *)VDataPtr.get(), Size: S);
118 }
119 }
120};
121
122} // end namespace llvm
123
124InstrProfWriter::InstrProfWriter(
125 bool Sparse, uint64_t TemporalProfTraceReservoirSize,
126 uint64_t MaxTemporalProfTraceLength, bool WritePrevVersion,
127 memprof::IndexedVersion MemProfVersionRequested, bool MemProfFullSchema,
128 bool MemprofGenerateRandomHotness,
129 unsigned MemprofGenerateRandomHotnessSeed)
130 : Sparse(Sparse), MaxTemporalProfTraceLength(MaxTemporalProfTraceLength),
131 TemporalProfTraceReservoirSize(TemporalProfTraceReservoirSize),
132 InfoObj(new InstrProfRecordWriterTrait()),
133 WritePrevVersion(WritePrevVersion),
134 MemProfVersionRequested(MemProfVersionRequested),
135 MemProfFullSchema(MemProfFullSchema),
136 MemprofGenerateRandomHotness(MemprofGenerateRandomHotness) {
137 // Set up the random number seed if requested.
138 if (MemprofGenerateRandomHotness) {
139 unsigned seed = MemprofGenerateRandomHotnessSeed
140 ? MemprofGenerateRandomHotnessSeed
141 : std::time(timer: nullptr);
142 errs() << "random hotness seed = " << seed << "\n";
143 std::srand(seed: seed);
144 }
145}
146
147InstrProfWriter::~InstrProfWriter() { delete InfoObj; }
148
149// Internal interface for testing purpose only.
150void InstrProfWriter::setValueProfDataEndianness(llvm::endianness Endianness) {
151 InfoObj->ValueProfDataEndianness = Endianness;
152}
153
154void InstrProfWriter::setOutputSparse(bool Sparse) { this->Sparse = Sparse; }
155
156void InstrProfWriter::addRecord(NamedInstrProfRecord &&I, uint64_t Weight,
157 function_ref<void(Error)> Warn) {
158 auto Name = I.Name;
159 auto Hash = I.Hash;
160 addRecord(Name, Hash, I: std::move(I), Weight, Warn);
161}
162
163void InstrProfWriter::overlapRecord(NamedInstrProfRecord &&Other,
164 OverlapStats &Overlap,
165 OverlapStats &FuncLevelOverlap,
166 const OverlapFuncFilters &FuncFilter) {
167 auto Name = Other.Name;
168 auto Hash = Other.Hash;
169 Other.accumulateCounts(Sum&: FuncLevelOverlap.Test);
170 auto It = FunctionData.find(Key: Name);
171 if (It == FunctionData.end()) {
172 Overlap.addOneUnique(UniqueFunc: FuncLevelOverlap.Test);
173 return;
174 }
175 if (FuncLevelOverlap.Test.CountSum < 1.0f) {
176 Overlap.Overlap.NumEntries += 1;
177 return;
178 }
179 auto &ProfileDataMap = It->second;
180 auto [Where, NewFunc] = ProfileDataMap.try_emplace(Key: Hash);
181 if (NewFunc) {
182 Overlap.addOneMismatch(MismatchFunc: FuncLevelOverlap.Test);
183 return;
184 }
185 InstrProfRecord &Dest = Where->second;
186
187 uint64_t ValueCutoff = FuncFilter.ValueCutoff;
188 if (!FuncFilter.NameFilter.empty() && Name.contains(Other: FuncFilter.NameFilter))
189 ValueCutoff = 0;
190
191 Dest.overlap(Other, Overlap, FuncLevelOverlap, ValueCutoff);
192}
193
194void InstrProfWriter::addRecord(StringRef Name, uint64_t Hash,
195 InstrProfRecord &&I, uint64_t Weight,
196 function_ref<void(Error)> Warn) {
197 auto &ProfileDataMap = FunctionData[Name];
198
199 auto [Where, NewFunc] = ProfileDataMap.try_emplace(Key: Hash);
200 InstrProfRecord &Dest = Where->second;
201
202 auto MapWarn = [&](instrprof_error E) {
203 Warn(make_error<InstrProfError>(Args&: E));
204 };
205
206 if (NewFunc) {
207 // We've never seen a function with this name and hash, add it.
208 Dest = std::move(I);
209 if (Weight > 1)
210 Dest.scale(N: Weight, D: 1, Warn: MapWarn);
211 } else {
212 // We're updating a function we've seen before.
213 Dest.merge(Other&: I, Weight, Warn: MapWarn);
214 }
215
216 Dest.sortValueData();
217}
218
219void InstrProfWriter::addMemProfRecord(
220 const Function::GUID Id, const memprof::IndexedMemProfRecord &Record) {
221 auto NewRecord = Record;
222 // Provoke random hotness values if requested. We specify the lifetime access
223 // density and lifetime length that will result in a cold or not cold hotness.
224 // See the logic in getAllocType() in Analysis/MemoryProfileInfo.cpp.
225 if (MemprofGenerateRandomHotness) {
226 for (auto &Alloc : NewRecord.AllocSites) {
227 // To get a not cold context, set the lifetime access density to the
228 // maximum value and the lifetime to 0.
229 uint64_t NewTLAD = std::numeric_limits<uint64_t>::max();
230 uint64_t NewTL = 0;
231 bool IsCold = std::rand() % 2;
232 if (IsCold) {
233 // To get a cold context, set the lifetime access density to 0 and the
234 // lifetime to the maximum value.
235 NewTLAD = 0;
236 NewTL = std::numeric_limits<uint64_t>::max();
237 }
238 Alloc.Info.setTotalLifetimeAccessDensity(NewTLAD);
239 Alloc.Info.setTotalLifetime(NewTL);
240 }
241 }
242 MemProfSumBuilder.addRecord(NewRecord);
243 auto [Iter, Inserted] = MemProfData.Records.insert(KV: {Id, NewRecord});
244 // If we inserted a new record then we are done.
245 if (Inserted) {
246 return;
247 }
248 memprof::IndexedMemProfRecord &Existing = Iter->second;
249 Existing.merge(Other: NewRecord);
250}
251
252bool InstrProfWriter::addMemProfFrame(const memprof::FrameId Id,
253 const memprof::Frame &Frame,
254 function_ref<void(Error)> Warn) {
255 auto [Iter, Inserted] = MemProfData.Frames.insert(KV: {Id, Frame});
256 // If a mapping already exists for the current frame id and it does not
257 // match the new mapping provided then reset the existing contents and bail
258 // out. We don't support the merging of memprof data whose Frame -> Id
259 // mapping across profiles is inconsistent.
260 if (!Inserted && Iter->second != Frame) {
261 Warn(make_error<InstrProfError>(Args: instrprof_error::malformed,
262 Args: "frame to id mapping mismatch"));
263 return false;
264 }
265 return true;
266}
267
268bool InstrProfWriter::addMemProfCallStack(
269 const memprof::CallStackId CSId,
270 const llvm::SmallVector<memprof::FrameId> &CallStack,
271 function_ref<void(Error)> Warn) {
272 auto [Iter, Inserted] = MemProfData.CallStacks.insert(KV: {CSId, CallStack});
273 // If a mapping already exists for the current call stack id and it does not
274 // match the new mapping provided then reset the existing contents and bail
275 // out. We don't support the merging of memprof data whose CallStack -> Id
276 // mapping across profiles is inconsistent.
277 if (!Inserted && Iter->second != CallStack) {
278 Warn(make_error<InstrProfError>(Args: instrprof_error::malformed,
279 Args: "call stack to id mapping mismatch"));
280 return false;
281 }
282 return true;
283}
284
285bool InstrProfWriter::addMemProfData(memprof::IndexedMemProfData Incoming,
286 function_ref<void(Error)> Warn) {
287 // Return immediately if everything is empty.
288 if (Incoming.Frames.empty() && Incoming.CallStacks.empty() &&
289 Incoming.Records.empty())
290 return true;
291
292 // Otherwise, every component must be non-empty.
293 assert(!Incoming.Frames.empty() && !Incoming.CallStacks.empty() &&
294 !Incoming.Records.empty());
295
296 if (MemProfData.Frames.empty())
297 MemProfData.Frames = std::move(Incoming.Frames);
298 else
299 for (const auto &[Id, F] : Incoming.Frames)
300 if (addMemProfFrame(Id, Frame: F, Warn))
301 return false;
302
303 if (MemProfData.CallStacks.empty())
304 MemProfData.CallStacks = std::move(Incoming.CallStacks);
305 else
306 for (const auto &[CSId, CS] : Incoming.CallStacks)
307 if (addMemProfCallStack(CSId, CallStack: CS, Warn))
308 return false;
309
310 // Add one record at a time if randomization is requested.
311 if (MemProfData.Records.empty() && !MemprofGenerateRandomHotness) {
312 // Need to manually add each record to the builder, which is otherwise done
313 // in addMemProfRecord.
314 for (const auto &[GUID, Record] : Incoming.Records)
315 MemProfSumBuilder.addRecord(Record);
316 MemProfData.Records = std::move(Incoming.Records);
317 } else {
318 for (const auto &[GUID, Record] : Incoming.Records)
319 addMemProfRecord(Id: GUID, Record);
320 }
321
322 return true;
323}
324
325void InstrProfWriter::addBinaryIds(ArrayRef<llvm::object::BuildID> BIs) {
326 llvm::append_range(C&: BinaryIds, R&: BIs);
327}
328
329void InstrProfWriter::addDataAccessProfData(
330 std::unique_ptr<memprof::DataAccessProfData> DataAccessProfDataIn) {
331 DataAccessProfileData = std::move(DataAccessProfDataIn);
332}
333
334void InstrProfWriter::addTemporalProfileTrace(TemporalProfTraceTy Trace) {
335 assert(Trace.FunctionNameRefs.size() <= MaxTemporalProfTraceLength);
336 assert(!Trace.FunctionNameRefs.empty());
337 if (TemporalProfTraceStreamSize < TemporalProfTraceReservoirSize) {
338 // Simply append the trace if we have not yet hit our reservoir size limit.
339 TemporalProfTraces.push_back(Elt: std::move(Trace));
340 } else {
341 // Otherwise, replace a random trace in the stream.
342 std::uniform_int_distribution<uint64_t> Distribution(
343 0, TemporalProfTraceStreamSize);
344 uint64_t RandomIndex = Distribution(RNG);
345 if (RandomIndex < TemporalProfTraces.size())
346 TemporalProfTraces[RandomIndex] = std::move(Trace);
347 }
348 ++TemporalProfTraceStreamSize;
349}
350
351void InstrProfWriter::addTemporalProfileTraces(
352 SmallVectorImpl<TemporalProfTraceTy> &SrcTraces, uint64_t SrcStreamSize) {
353 for (auto &Trace : SrcTraces)
354 if (Trace.FunctionNameRefs.size() > MaxTemporalProfTraceLength)
355 Trace.FunctionNameRefs.resize(new_size: MaxTemporalProfTraceLength);
356 llvm::erase_if(C&: SrcTraces, P: [](auto &T) { return T.FunctionNameRefs.empty(); });
357 // Assume that the source has the same reservoir size as the destination to
358 // avoid needing to record it in the indexed profile format.
359 bool IsDestSampled =
360 (TemporalProfTraceStreamSize > TemporalProfTraceReservoirSize);
361 bool IsSrcSampled = (SrcStreamSize > TemporalProfTraceReservoirSize);
362 if (!IsDestSampled && IsSrcSampled) {
363 // If one of the traces are sampled, ensure that it belongs to Dest.
364 std::swap(LHS&: TemporalProfTraces, RHS&: SrcTraces);
365 std::swap(a&: TemporalProfTraceStreamSize, b&: SrcStreamSize);
366 std::swap(a&: IsDestSampled, b&: IsSrcSampled);
367 }
368 if (!IsSrcSampled) {
369 // If the source stream is not sampled, we add each source trace normally.
370 for (auto &Trace : SrcTraces)
371 addTemporalProfileTrace(Trace: std::move(Trace));
372 return;
373 }
374 // Otherwise, we find the traces that would have been removed if we added
375 // the whole source stream.
376 SmallSetVector<uint64_t, 8> IndicesToReplace;
377 for (uint64_t I = 0; I < SrcStreamSize; I++) {
378 std::uniform_int_distribution<uint64_t> Distribution(
379 0, TemporalProfTraceStreamSize);
380 uint64_t RandomIndex = Distribution(RNG);
381 if (RandomIndex < TemporalProfTraces.size())
382 IndicesToReplace.insert(X: RandomIndex);
383 ++TemporalProfTraceStreamSize;
384 }
385 // Then we insert a random sample of the source traces.
386 llvm::shuffle(first: SrcTraces.begin(), last: SrcTraces.end(), g&: RNG);
387 for (const auto &[Index, Trace] : llvm::zip(t&: IndicesToReplace, u&: SrcTraces))
388 TemporalProfTraces[Index] = std::move(Trace);
389}
390
391void InstrProfWriter::mergeRecordsFromWriter(InstrProfWriter &&IPW,
392 function_ref<void(Error)> Warn) {
393 for (auto &I : IPW.FunctionData)
394 for (auto &Func : I.getValue())
395 addRecord(Name: I.getKey(), Hash: Func.first, I: std::move(Func.second), Weight: 1, Warn);
396
397 BinaryIds.reserve(n: BinaryIds.size() + IPW.BinaryIds.size());
398 for (auto &I : IPW.BinaryIds)
399 addBinaryIds(BIs: I);
400
401 addTemporalProfileTraces(SrcTraces&: IPW.TemporalProfTraces,
402 SrcStreamSize: IPW.TemporalProfTraceStreamSize);
403
404 MemProfData.Frames.reserve(NumEntries: IPW.MemProfData.Frames.size());
405 for (auto &[FrameId, Frame] : IPW.MemProfData.Frames) {
406 // If we weren't able to add the frame mappings then it doesn't make sense
407 // to try to merge the records from this profile.
408 if (!addMemProfFrame(Id: FrameId, Frame, Warn))
409 return;
410 }
411
412 MemProfData.CallStacks.reserve(NumEntries: IPW.MemProfData.CallStacks.size());
413 for (auto &[CSId, CallStack] : IPW.MemProfData.CallStacks) {
414 if (!addMemProfCallStack(CSId, CallStack, Warn))
415 return;
416 }
417
418 MemProfData.Records.reserve(NumEntries: IPW.MemProfData.Records.size());
419 for (auto &[GUID, Record] : IPW.MemProfData.Records) {
420 addMemProfRecord(Id: GUID, Record);
421 }
422}
423
424bool InstrProfWriter::shouldEncodeData(const ProfilingData &PD) {
425 if (!Sparse)
426 return true;
427 for (const auto &Func : PD) {
428 const InstrProfRecord &IPR = Func.second;
429 if (llvm::any_of(Range: IPR.Counts, P: [](uint64_t Count) { return Count > 0; }))
430 return true;
431 if (llvm::any_of(Range: IPR.BitmapBytes, P: [](uint8_t Byte) { return Byte > 0; }))
432 return true;
433 }
434 return false;
435}
436
437static void setSummary(IndexedInstrProf::Summary *TheSummary,
438 ProfileSummary &PS) {
439 using namespace IndexedInstrProf;
440
441 const std::vector<ProfileSummaryEntry> &Res = PS.getDetailedSummary();
442 TheSummary->NumSummaryFields = Summary::NumKinds;
443 TheSummary->NumCutoffEntries = Res.size();
444 TheSummary->set(K: Summary::MaxFunctionCount, V: PS.getMaxFunctionCount());
445 TheSummary->set(K: Summary::MaxBlockCount, V: PS.getMaxCount());
446 TheSummary->set(K: Summary::MaxInternalBlockCount, V: PS.getMaxInternalCount());
447 TheSummary->set(K: Summary::TotalBlockCount, V: PS.getTotalCount());
448 TheSummary->set(K: Summary::TotalNumBlocks, V: PS.getNumCounts());
449 TheSummary->set(K: Summary::TotalNumFunctions, V: PS.getNumFunctions());
450 for (unsigned I = 0; I < Res.size(); I++)
451 TheSummary->setEntry(I, E: Res[I]);
452}
453
454uint64_t InstrProfWriter::writeHeader(const IndexedInstrProf::Header &Header,
455 const bool WritePrevVersion,
456 ProfOStream &OS) {
457 // Only write out the first four fields.
458 for (int I = 0; I < 4; I++)
459 OS.write(V: reinterpret_cast<const uint64_t *>(&Header)[I]);
460
461 // Remember the offset of the remaining fields to allow back patching later.
462 auto BackPatchStartOffset = OS.tell();
463
464 // Reserve the space for back patching later.
465 OS.write(V: 0); // HashOffset
466 OS.write(V: 0); // MemProfOffset
467 OS.write(V: 0); // BinaryIdOffset
468 OS.write(V: 0); // TemporalProfTracesOffset
469 if (!WritePrevVersion)
470 OS.write(V: 0); // VTableNamesOffset
471
472 return BackPatchStartOffset;
473}
474
475Error InstrProfWriter::writeBinaryIds(ProfOStream &OS) {
476 // BinaryIdSection has two parts:
477 // 1. uint64_t BinaryIdsSectionSize
478 // 2. list of binary ids that consist of:
479 // a. uint64_t BinaryIdLength
480 // b. uint8_t BinaryIdData
481 // c. uint8_t Padding (if necessary)
482 // Calculate size of binary section.
483 uint64_t BinaryIdsSectionSize = 0;
484
485 // Remove duplicate binary ids.
486 llvm::sort(C&: BinaryIds);
487 BinaryIds.erase(first: llvm::unique(R&: BinaryIds), last: BinaryIds.end());
488
489 for (const auto &BI : BinaryIds) {
490 // Increment by binary id length data type size.
491 BinaryIdsSectionSize += sizeof(uint64_t);
492 // Increment by binary id data length, aligned to 8 bytes.
493 BinaryIdsSectionSize += alignToPowerOf2(Value: BI.size(), Align: sizeof(uint64_t));
494 }
495 // Write binary ids section size.
496 OS.write(V: BinaryIdsSectionSize);
497
498 for (const auto &BI : BinaryIds) {
499 uint64_t BILen = BI.size();
500 // Write binary id length.
501 OS.write(V: BILen);
502 // Write binary id data.
503 for (unsigned K = 0; K < BILen; K++)
504 OS.writeByte(V: BI[K]);
505 // Write padding if necessary.
506 uint64_t PaddingSize = alignToPowerOf2(Value: BILen, Align: sizeof(uint64_t)) - BILen;
507 for (unsigned K = 0; K < PaddingSize; K++)
508 OS.writeByte(V: 0);
509 }
510
511 return Error::success();
512}
513
514Error InstrProfWriter::writeVTableNames(ProfOStream &OS) {
515 std::vector<std::string> VTableNameStrs;
516 for (StringRef VTableName : VTableNames.keys())
517 VTableNameStrs.push_back(x: VTableName.str());
518
519 std::string CompressedVTableNames;
520 if (!VTableNameStrs.empty())
521 if (Error E = collectGlobalObjectNameStrings(
522 NameStrs: VTableNameStrs, doCompression: compression::zlib::isAvailable(),
523 Result&: CompressedVTableNames))
524 return E;
525
526 const uint64_t CompressedStringLen = CompressedVTableNames.length();
527
528 // Record the length of compressed string.
529 OS.write(V: CompressedStringLen);
530
531 // Write the chars in compressed strings.
532 for (auto &c : CompressedVTableNames)
533 OS.writeByte(V: static_cast<uint8_t>(c));
534
535 // Pad up to a multiple of 8.
536 // InstrProfReader could read bytes according to 'CompressedStringLen'.
537 const uint64_t PaddedLength = alignTo(Value: CompressedStringLen, Align: 8);
538
539 for (uint64_t K = CompressedStringLen; K < PaddedLength; K++)
540 OS.writeByte(V: 0);
541
542 return Error::success();
543}
544
545Error InstrProfWriter::writeImpl(ProfOStream &OS) {
546 using namespace IndexedInstrProf;
547 using namespace support;
548
549 OnDiskChainedHashTableGenerator<InstrProfRecordWriterTrait> Generator;
550
551 InstrProfSummaryBuilder ISB(ProfileSummaryBuilder::DefaultCutoffs);
552 InfoObj->SummaryBuilder = &ISB;
553 InstrProfSummaryBuilder CSISB(ProfileSummaryBuilder::DefaultCutoffs);
554 InfoObj->CSSummaryBuilder = &CSISB;
555
556 // Populate the hash table generator.
557 SmallVector<std::pair<StringRef, const ProfilingData *>> OrderedData;
558 for (const auto &I : FunctionData)
559 if (shouldEncodeData(PD: I.getValue()))
560 OrderedData.emplace_back(Args: (I.getKey()), Args: &I.getValue());
561 llvm::sort(C&: OrderedData, Comp: less_first());
562 for (const auto &I : OrderedData)
563 Generator.insert(Key: I.first, Data: I.second);
564
565 // Write the header.
566 IndexedInstrProf::Header Header;
567 Header.Version = WritePrevVersion
568 ? IndexedInstrProf::ProfVersion::Version11
569 : IndexedInstrProf::ProfVersion::CurrentVersion;
570 // The WritePrevVersion handling will either need to be removed or updated
571 // if the version is advanced beyond 12.
572 static_assert(IndexedInstrProf::ProfVersion::CurrentVersion ==
573 IndexedInstrProf::ProfVersion::Version12);
574 if (static_cast<bool>(ProfileKind & InstrProfKind::IRInstrumentation))
575 Header.Version |= VARIANT_MASK_IR_PROF;
576 if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive))
577 Header.Version |= VARIANT_MASK_CSIR_PROF;
578 if (static_cast<bool>(ProfileKind &
579 InstrProfKind::FunctionEntryInstrumentation))
580 Header.Version |= VARIANT_MASK_INSTR_ENTRY;
581 if (static_cast<bool>(ProfileKind &
582 InstrProfKind::LoopEntriesInstrumentation))
583 Header.Version |= VARIANT_MASK_INSTR_LOOP_ENTRIES;
584 if (static_cast<bool>(ProfileKind & InstrProfKind::SingleByteCoverage))
585 Header.Version |= VARIANT_MASK_BYTE_COVERAGE;
586 if (static_cast<bool>(ProfileKind & InstrProfKind::FunctionEntryOnly))
587 Header.Version |= VARIANT_MASK_FUNCTION_ENTRY_ONLY;
588 if (static_cast<bool>(ProfileKind & InstrProfKind::MemProf))
589 Header.Version |= VARIANT_MASK_MEMPROF;
590 if (static_cast<bool>(ProfileKind & InstrProfKind::TemporalProfile))
591 Header.Version |= VARIANT_MASK_TEMPORAL_PROF;
592
593 const uint64_t BackPatchStartOffset =
594 writeHeader(Header, WritePrevVersion, OS);
595
596 // Reserve space to write profile summary data.
597 uint32_t NumEntries = ProfileSummaryBuilder::DefaultCutoffs.size();
598 uint32_t SummarySize = Summary::getSize(NumSumFields: Summary::NumKinds, NumCutoffEntries: NumEntries);
599 // Remember the summary offset.
600 uint64_t SummaryOffset = OS.tell();
601 for (unsigned I = 0; I < SummarySize / sizeof(uint64_t); I++)
602 OS.write(V: 0);
603 uint64_t CSSummaryOffset = 0;
604 uint64_t CSSummarySize = 0;
605 if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive)) {
606 CSSummaryOffset = OS.tell();
607 CSSummarySize = SummarySize / sizeof(uint64_t);
608 for (unsigned I = 0; I < CSSummarySize; I++)
609 OS.write(V: 0);
610 }
611
612 // Write the hash table.
613 uint64_t HashTableStart = Generator.Emit(Out&: OS.OS, InfoObj&: *InfoObj);
614
615 // Write the MemProf profile data if we have it.
616 uint64_t MemProfSectionStart = 0;
617 if (static_cast<bool>(ProfileKind & InstrProfKind::MemProf)) {
618 MemProfSectionStart = OS.tell();
619
620 if (auto E = writeMemProf(
621 OS, MemProfData, MemProfVersionRequested, MemProfFullSchema,
622 DataAccessProfileData: std::move(DataAccessProfileData), MemProfSum: MemProfSumBuilder.getSummary()))
623 return E;
624 }
625
626 uint64_t BinaryIdSectionStart = OS.tell();
627 if (auto E = writeBinaryIds(OS))
628 return E;
629
630 uint64_t VTableNamesSectionStart = OS.tell();
631
632 if (!WritePrevVersion)
633 if (Error E = writeVTableNames(OS))
634 return E;
635
636 uint64_t TemporalProfTracesSectionStart = 0;
637 if (static_cast<bool>(ProfileKind & InstrProfKind::TemporalProfile)) {
638 TemporalProfTracesSectionStart = OS.tell();
639 OS.write(V: TemporalProfTraces.size());
640 OS.write(V: TemporalProfTraceStreamSize);
641 for (auto &Trace : TemporalProfTraces) {
642 OS.write(V: Trace.Weight);
643 OS.write(V: Trace.FunctionNameRefs.size());
644 for (auto &NameRef : Trace.FunctionNameRefs)
645 OS.write(V: NameRef);
646 }
647 }
648
649 // Allocate space for data to be serialized out.
650 std::unique_ptr<IndexedInstrProf::Summary> TheSummary =
651 IndexedInstrProf::allocSummary(TotalSize: SummarySize);
652 // Compute the Summary and copy the data to the data
653 // structure to be serialized out (to disk or buffer).
654 std::unique_ptr<ProfileSummary> PS = ISB.getSummary();
655 setSummary(TheSummary: TheSummary.get(), PS&: *PS);
656 InfoObj->SummaryBuilder = nullptr;
657
658 // For Context Sensitive summary.
659 std::unique_ptr<IndexedInstrProf::Summary> TheCSSummary = nullptr;
660 if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive)) {
661 TheCSSummary = IndexedInstrProf::allocSummary(TotalSize: SummarySize);
662 std::unique_ptr<ProfileSummary> CSPS = CSISB.getSummary();
663 setSummary(TheSummary: TheCSSummary.get(), PS&: *CSPS);
664 }
665 InfoObj->CSSummaryBuilder = nullptr;
666
667 SmallVector<uint64_t, 8> HeaderOffsets = {HashTableStart, MemProfSectionStart,
668 BinaryIdSectionStart,
669 TemporalProfTracesSectionStart};
670 if (!WritePrevVersion)
671 HeaderOffsets.push_back(Elt: VTableNamesSectionStart);
672
673 PatchItem PatchItems[] = {
674 // Patch the Header fields
675 {.Pos: BackPatchStartOffset, .D: HeaderOffsets},
676 // Patch the summary data.
677 {.Pos: SummaryOffset,
678 .D: ArrayRef<uint64_t>(reinterpret_cast<uint64_t *>(TheSummary.get()),
679 SummarySize / sizeof(uint64_t))},
680 {.Pos: CSSummaryOffset,
681 .D: ArrayRef<uint64_t>(reinterpret_cast<uint64_t *>(TheCSSummary.get()),
682 CSSummarySize)}};
683
684 OS.patch(P: PatchItems);
685
686 for (const auto &I : FunctionData)
687 for (const auto &F : I.getValue())
688 if (Error E = validateRecord(Func: F.second))
689 return E;
690
691 return Error::success();
692}
693
694Error InstrProfWriter::write(raw_fd_ostream &OS) {
695 // Write the hash table.
696 ProfOStream POS(OS);
697 return writeImpl(OS&: POS);
698}
699
700Error InstrProfWriter::write(raw_string_ostream &OS) {
701 ProfOStream POS(OS);
702 return writeImpl(OS&: POS);
703}
704
705std::unique_ptr<MemoryBuffer> InstrProfWriter::writeBuffer() {
706 std::string Data;
707 raw_string_ostream OS(Data);
708 // Write the hash table.
709 if (Error E = write(OS))
710 return nullptr;
711 // Return this in an aligned memory buffer.
712 return MemoryBuffer::getMemBufferCopy(InputData: Data);
713}
714
715static const char *ValueProfKindStr[] = {
716#define VALUE_PROF_KIND(Enumerator, Value, Descr) #Enumerator,
717#include "llvm/ProfileData/InstrProfData.inc"
718};
719
720Error InstrProfWriter::validateRecord(const InstrProfRecord &Func) {
721 for (uint32_t VK = 0; VK <= IPVK_Last; VK++) {
722 if (VK == IPVK_IndirectCallTarget || VK == IPVK_VTableTarget)
723 continue;
724 uint32_t NS = Func.getNumValueSites(ValueKind: VK);
725 for (uint32_t S = 0; S < NS; S++) {
726 DenseSet<uint64_t> SeenValues;
727 for (const auto &V : Func.getValueArrayForSite(ValueKind: VK, Site: S))
728 if (!SeenValues.insert(V: V.Value).second)
729 return make_error<InstrProfError>(Args: instrprof_error::invalid_prof);
730 }
731 }
732
733 return Error::success();
734}
735
736void InstrProfWriter::writeRecordInText(StringRef Name, uint64_t Hash,
737 const InstrProfRecord &Func,
738 InstrProfSymtab &Symtab,
739 raw_fd_ostream &OS) {
740 OS << Name << "\n";
741 OS << "# Func Hash:\n" << Hash << "\n";
742 OS << "# Num Counters:\n" << Func.Counts.size() << "\n";
743 OS << "# Counter Values:\n";
744 for (uint64_t Count : Func.Counts)
745 OS << Count << "\n";
746
747 if (Func.BitmapBytes.size() > 0) {
748 OS << "# Num Bitmap Bytes:\n$" << Func.BitmapBytes.size() << "\n";
749 OS << "# Bitmap Byte Values:\n";
750 for (uint8_t Byte : Func.BitmapBytes) {
751 OS << "0x";
752 OS.write_hex(N: Byte);
753 OS << "\n";
754 }
755 OS << "\n";
756 }
757
758 uint32_t NumValueKinds = Func.getNumValueKinds();
759 if (!NumValueKinds) {
760 OS << "\n";
761 return;
762 }
763
764 OS << "# Num Value Kinds:\n" << Func.getNumValueKinds() << "\n";
765 for (uint32_t VK = 0; VK < IPVK_Last + 1; VK++) {
766 uint32_t NS = Func.getNumValueSites(ValueKind: VK);
767 if (!NS)
768 continue;
769 OS << "# ValueKind = " << ValueProfKindStr[VK] << ":\n" << VK << "\n";
770 OS << "# NumValueSites:\n" << NS << "\n";
771 for (uint32_t S = 0; S < NS; S++) {
772 auto VD = Func.getValueArrayForSite(ValueKind: VK, Site: S);
773 OS << VD.size() << "\n";
774 for (const auto &V : VD) {
775 if (VK == IPVK_IndirectCallTarget || VK == IPVK_VTableTarget)
776 OS << Symtab.getFuncOrVarNameIfDefined(MD5Hash: V.Value) << ":" << V.Count
777 << "\n";
778 else
779 OS << V.Value << ":" << V.Count << "\n";
780 }
781 }
782 }
783
784 OS << "\n";
785}
786
787Error InstrProfWriter::writeText(raw_fd_ostream &OS) {
788 // Check CS first since it implies an IR level profile.
789 if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive))
790 OS << "# CSIR level Instrumentation Flag\n:csir\n";
791 else if (static_cast<bool>(ProfileKind & InstrProfKind::IRInstrumentation))
792 OS << "# IR level Instrumentation Flag\n:ir\n";
793
794 if (static_cast<bool>(ProfileKind &
795 InstrProfKind::FunctionEntryInstrumentation))
796 OS << "# Always instrument the function entry block\n:entry_first\n";
797 if (static_cast<bool>(ProfileKind &
798 InstrProfKind::LoopEntriesInstrumentation))
799 OS << "# Always instrument the loop entry "
800 "blocks\n:instrument_loop_entries\n";
801 if (static_cast<bool>(ProfileKind & InstrProfKind::SingleByteCoverage))
802 OS << "# Instrument block coverage\n:single_byte_coverage\n";
803 InstrProfSymtab Symtab;
804
805 using FuncPair = detail::DenseMapPair<uint64_t, InstrProfRecord>;
806 using RecordType = std::pair<StringRef, FuncPair>;
807 SmallVector<RecordType, 4> OrderedFuncData;
808
809 for (const auto &I : FunctionData) {
810 if (shouldEncodeData(PD: I.getValue())) {
811 if (Error E = Symtab.addFuncName(FuncName: I.getKey()))
812 return E;
813 for (const auto &Func : I.getValue())
814 OrderedFuncData.push_back(Elt: std::make_pair(x: I.getKey(), y: Func));
815 }
816 }
817
818 for (const auto &VTableName : VTableNames)
819 if (Error E = Symtab.addVTableName(VTableName: VTableName.getKey()))
820 return E;
821
822 if (static_cast<bool>(ProfileKind & InstrProfKind::TemporalProfile))
823 writeTextTemporalProfTraceData(OS, Symtab);
824
825 llvm::sort(C&: OrderedFuncData, Comp: [](const RecordType &A, const RecordType &B) {
826 return std::tie(args: A.first, args: A.second.first) <
827 std::tie(args: B.first, args: B.second.first);
828 });
829
830 for (const auto &record : OrderedFuncData) {
831 const StringRef &Name = record.first;
832 const FuncPair &Func = record.second;
833 writeRecordInText(Name, Hash: Func.first, Func: Func.second, Symtab, OS);
834 }
835
836 for (const auto &record : OrderedFuncData) {
837 const FuncPair &Func = record.second;
838 if (Error E = validateRecord(Func: Func.second))
839 return E;
840 }
841
842 return Error::success();
843}
844
845void InstrProfWriter::writeTextTemporalProfTraceData(raw_fd_ostream &OS,
846 InstrProfSymtab &Symtab) {
847 OS << ":temporal_prof_traces\n";
848 OS << "# Num Temporal Profile Traces:\n" << TemporalProfTraces.size() << "\n";
849 OS << "# Temporal Profile Trace Stream Size:\n"
850 << TemporalProfTraceStreamSize << "\n";
851 for (auto &Trace : TemporalProfTraces) {
852 OS << "# Weight:\n" << Trace.Weight << "\n";
853 for (auto &NameRef : Trace.FunctionNameRefs)
854 OS << Symtab.getFuncOrVarName(MD5Hash: NameRef) << ",";
855 OS << "\n";
856 }
857 OS << "\n";
858}
859