1//===- InstrProfWriter.cpp - Instrumented profiling writer ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains support for writing profiling data for clang's
10// instrumentation based PGO and coverage.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/ProfileData/InstrProfWriter.h"
15#include "llvm/ADT/STLExtras.h"
16#include "llvm/ADT/StringRef.h"
17#include "llvm/IR/ProfileSummary.h"
18#include "llvm/ProfileData/DataAccessProf.h"
19#include "llvm/ProfileData/IndexedMemProfData.h"
20#include "llvm/ProfileData/InstrProf.h"
21#include "llvm/ProfileData/ProfileCommon.h"
22#include "llvm/Support/Compression.h"
23#include "llvm/Support/EndianStream.h"
24#include "llvm/Support/Error.h"
25#include "llvm/Support/MemoryBuffer.h"
26#include "llvm/Support/OnDiskHashTable.h"
27#include "llvm/Support/raw_ostream.h"
28#include <cstdint>
29#include <ctime>
30#include <memory>
31#include <string>
32#include <tuple>
33#include <utility>
34#include <vector>
35
36using namespace llvm;
37
38namespace llvm {
39
40class InstrProfRecordWriterTrait {
41public:
42 using key_type = StringRef;
43 using key_type_ref = StringRef;
44
45 using data_type = const InstrProfWriter::ProfilingData *const;
46 using data_type_ref = const InstrProfWriter::ProfilingData *const;
47
48 using hash_value_type = uint64_t;
49 using offset_type = uint64_t;
50
51 llvm::endianness ValueProfDataEndianness = llvm::endianness::little;
52 InstrProfSummaryBuilder *SummaryBuilder;
53 InstrProfSummaryBuilder *CSSummaryBuilder;
54 bool WritePrevVersion = false;
55
56 InstrProfRecordWriterTrait() = default;
57
58 static hash_value_type ComputeHash(key_type_ref K) {
59 return IndexedInstrProf::ComputeHash(K);
60 }
61
62 std::pair<offset_type, offset_type>
63 EmitKeyDataLength(raw_ostream &Out, key_type_ref K, data_type_ref V) {
64 using namespace support;
65
66 endian::Writer LE(Out, llvm::endianness::little);
67
68 offset_type N = K.size();
69 LE.write<offset_type>(Val: N);
70
71 offset_type M = 0;
72 for (const auto &ProfileData : *V) {
73 const InstrProfRecord &ProfRecord = ProfileData.second;
74 M += sizeof(uint64_t); // The function hash
75 M += sizeof(uint64_t); // The size of the Counts vector
76 M += ProfRecord.Counts.size() * sizeof(uint64_t);
77 M += sizeof(uint64_t); // The size of the Bitmap vector
78 if (WritePrevVersion) {
79 // Compatibility mode: each bitmap byte is stored as a uint64_t.
80 M += ProfRecord.BitmapBytes.size() * sizeof(uint64_t);
81 } else {
82 // Version 14+: bitmap bytes as uint8_t with padding, plus
83 // uniformity bits.
84 M += alignTo(Value: ProfRecord.BitmapBytes.size(), Align: sizeof(uint64_t));
85 M += sizeof(uint64_t); // The size of the UniformityBits vector
86 M += alignTo(Value: ProfRecord.UniformityBits.size(), Align: sizeof(uint64_t));
87 }
88
89 // Value data
90 M += ValueProfData::getSize(Record: ProfileData.second);
91 }
92 LE.write<offset_type>(Val: M);
93
94 return std::make_pair(x&: N, y&: M);
95 }
96
97 void EmitKey(raw_ostream &Out, key_type_ref K, offset_type N) {
98 Out.write(Ptr: K.data(), Size: N);
99 }
100
101 void EmitData(raw_ostream &Out, key_type_ref K, data_type_ref V,
102 offset_type) {
103 using namespace support;
104
105 endian::Writer LE(Out, llvm::endianness::little);
106 for (const auto &ProfileData : *V) {
107 const InstrProfRecord &ProfRecord = ProfileData.second;
108 if (NamedInstrProfRecord::hasCSFlagInHash(FuncHash: ProfileData.first))
109 CSSummaryBuilder->addRecord(ProfRecord);
110 else
111 SummaryBuilder->addRecord(ProfRecord);
112
113 LE.write<uint64_t>(Val: ProfileData.first); // Function hash
114 LE.write<uint64_t>(Val: ProfRecord.Counts.size());
115 for (uint64_t I : ProfRecord.Counts)
116 LE.write<uint64_t>(Val: I);
117
118 LE.write<uint64_t>(Val: ProfRecord.BitmapBytes.size());
119 if (WritePrevVersion) {
120 // Compatibility mode: each bitmap byte is stored as a uint64_t.
121 for (uint8_t I : ProfRecord.BitmapBytes)
122 LE.write<uint64_t>(Val: I);
123 } else {
124 // Version 14+: bitmap bytes as uint8_t with padding.
125 for (uint8_t I : ProfRecord.BitmapBytes)
126 LE.write<uint8_t>(Val: I);
127 for (size_t I = ProfRecord.BitmapBytes.size();
128 I < alignTo(Value: ProfRecord.BitmapBytes.size(), Align: sizeof(uint64_t)); ++I)
129 LE.write<uint8_t>(Val: 0);
130
131 // Write uniformity bits (AMDGPU offload profiling).
132 LE.write<uint64_t>(Val: ProfRecord.UniformityBits.size());
133 for (uint8_t I : ProfRecord.UniformityBits)
134 LE.write<uint8_t>(Val: I);
135 for (size_t I = ProfRecord.UniformityBits.size();
136 I < alignTo(Value: ProfRecord.UniformityBits.size(), Align: sizeof(uint64_t));
137 ++I)
138 LE.write<uint8_t>(Val: 0);
139 }
140
141 // Write value data
142 std::unique_ptr<ValueProfData> VDataPtr =
143 ValueProfData::serializeFrom(Record: ProfileData.second);
144 uint32_t S = VDataPtr->getSize();
145 VDataPtr->swapBytesFromHost(Endianness: ValueProfDataEndianness);
146 Out.write(Ptr: (const char *)VDataPtr.get(), Size: S);
147 }
148 }
149};
150
151} // end namespace llvm
152
153InstrProfWriter::InstrProfWriter(
154 bool Sparse, uint64_t TemporalProfTraceReservoirSize,
155 uint64_t MaxTemporalProfTraceLength, bool WritePrevVersion,
156 memprof::IndexedVersion MemProfVersionRequested, bool MemProfFullSchema,
157 bool MemprofGenerateRandomHotness,
158 unsigned MemprofGenerateRandomHotnessSeed)
159 : Sparse(Sparse), MaxTemporalProfTraceLength(MaxTemporalProfTraceLength),
160 TemporalProfTraceReservoirSize(TemporalProfTraceReservoirSize),
161 InfoObj(new InstrProfRecordWriterTrait()),
162 WritePrevVersion(WritePrevVersion),
163 MemProfVersionRequested(MemProfVersionRequested),
164 MemProfFullSchema(MemProfFullSchema),
165 MemprofGenerateRandomHotness(MemprofGenerateRandomHotness) {
166 // Set up the random number seed if requested.
167 if (MemprofGenerateRandomHotness) {
168 unsigned seed = MemprofGenerateRandomHotnessSeed
169 ? MemprofGenerateRandomHotnessSeed
170 : std::time(timer: nullptr);
171 errs() << "random hotness seed = " << seed << "\n";
172 std::srand(seed: seed);
173 }
174}
175
176InstrProfWriter::~InstrProfWriter() { delete InfoObj; }
177
178// Internal interface for testing purpose only.
179void InstrProfWriter::setValueProfDataEndianness(llvm::endianness Endianness) {
180 InfoObj->ValueProfDataEndianness = Endianness;
181}
182
183void InstrProfWriter::setOutputSparse(bool Sparse) { this->Sparse = Sparse; }
184
185void InstrProfWriter::addRecord(NamedInstrProfRecord &&I, uint64_t Weight,
186 function_ref<void(Error)> Warn) {
187 auto Name = I.Name;
188 auto Hash = I.Hash;
189 addRecord(Name, Hash, I: std::move(I), Weight, Warn);
190}
191
192void InstrProfWriter::overlapRecord(NamedInstrProfRecord &&Other,
193 OverlapStats &Overlap,
194 OverlapStats &FuncLevelOverlap,
195 const OverlapFuncFilters &FuncFilter) {
196 auto Name = Other.Name;
197 auto Hash = Other.Hash;
198 Other.accumulateCounts(Sum&: FuncLevelOverlap.Test);
199 auto It = FunctionData.find(Key: Name);
200 if (It == FunctionData.end()) {
201 Overlap.addOneUnique(UniqueFunc: FuncLevelOverlap.Test);
202 return;
203 }
204 if (FuncLevelOverlap.Test.CountSum < 1.0f) {
205 Overlap.Overlap.NumEntries += 1;
206 return;
207 }
208 auto &ProfileDataMap = It->second;
209 auto [Where, NewFunc] = ProfileDataMap.try_emplace(Key: Hash);
210 if (NewFunc) {
211 Overlap.addOneMismatch(MismatchFunc: FuncLevelOverlap.Test);
212 return;
213 }
214 InstrProfRecord &Dest = Where->second;
215
216 uint64_t ValueCutoff = FuncFilter.ValueCutoff;
217 if (!FuncFilter.NameFilter.empty() && Name.contains(Other: FuncFilter.NameFilter))
218 ValueCutoff = 0;
219
220 Dest.overlap(Other, Overlap, FuncLevelOverlap, ValueCutoff);
221}
222
223void InstrProfWriter::addRecord(StringRef Name, uint64_t Hash,
224 InstrProfRecord &&I, uint64_t Weight,
225 function_ref<void(Error)> Warn) {
226 I.computeBlockUniformity();
227
228 auto &ProfileDataMap = FunctionData[Name];
229
230 auto [Where, NewFunc] = ProfileDataMap.try_emplace(Key: Hash);
231 InstrProfRecord &Dest = Where->second;
232
233 auto MapWarn = [&](instrprof_error E) {
234 Warn(make_error<InstrProfError>(Args&: E));
235 };
236
237 if (NewFunc) {
238 // We've never seen a function with this name and hash, add it.
239 Dest = std::move(I);
240 if (Weight > 1)
241 Dest.scale(N: Weight, D: 1, Warn: MapWarn);
242 } else {
243 // We're updating a function we've seen before.
244 Dest.merge(Other&: I, Weight, Warn: MapWarn);
245 }
246
247 Dest.sortValueData();
248}
249
250void InstrProfWriter::addMemProfRecord(
251 const Function::GUID Id, const memprof::IndexedMemProfRecord &Record) {
252 auto NewRecord = Record;
253 // Provoke random hotness values if requested. We specify the lifetime access
254 // density and lifetime length that will result in a cold or not cold hotness.
255 // See the logic in getAllocType() in Analysis/MemoryProfileInfo.cpp.
256 if (MemprofGenerateRandomHotness) {
257 for (auto &Alloc : NewRecord.AllocSites) {
258 // To get a not cold context, set the lifetime access density to the
259 // maximum value and the lifetime to 0.
260 uint64_t NewTLAD = std::numeric_limits<uint64_t>::max();
261 uint64_t NewTL = 0;
262 bool IsCold = std::rand() % 2;
263 if (IsCold) {
264 // To get a cold context, set the lifetime access density to 0 and the
265 // lifetime to the maximum value.
266 NewTLAD = 0;
267 NewTL = std::numeric_limits<uint64_t>::max();
268 }
269 Alloc.Info.setTotalLifetimeAccessDensity(NewTLAD);
270 Alloc.Info.setTotalLifetime(NewTL);
271 }
272 }
273 MemProfSumBuilder.addRecord(NewRecord);
274 auto [Iter, Inserted] = MemProfData.Records.insert(KV: {Id, NewRecord});
275 // If we inserted a new record then we are done.
276 if (Inserted) {
277 return;
278 }
279 memprof::IndexedMemProfRecord &Existing = Iter->second;
280 Existing.merge(Other: NewRecord);
281}
282
283bool InstrProfWriter::addMemProfFrame(const memprof::FrameId Id,
284 const memprof::Frame &Frame,
285 function_ref<void(Error)> Warn) {
286 auto [Iter, Inserted] = MemProfData.Frames.insert(KV: {Id, Frame});
287 // If a mapping already exists for the current frame id and it does not
288 // match the new mapping provided then reset the existing contents and bail
289 // out. We don't support the merging of memprof data whose Frame -> Id
290 // mapping across profiles is inconsistent.
291 if (!Inserted && Iter->second != Frame) {
292 Warn(make_error<InstrProfError>(Args: instrprof_error::malformed,
293 Args: "frame to id mapping mismatch"));
294 return false;
295 }
296 return true;
297}
298
299bool InstrProfWriter::addMemProfCallStack(
300 const memprof::CallStackId CSId,
301 const llvm::SmallVector<memprof::FrameId> &CallStack,
302 function_ref<void(Error)> Warn) {
303 auto [Iter, Inserted] = MemProfData.CallStacks.insert(KV: {CSId, CallStack});
304 // If a mapping already exists for the current call stack id and it does not
305 // match the new mapping provided then reset the existing contents and bail
306 // out. We don't support the merging of memprof data whose CallStack -> Id
307 // mapping across profiles is inconsistent.
308 if (!Inserted && Iter->second != CallStack) {
309 Warn(make_error<InstrProfError>(Args: instrprof_error::malformed,
310 Args: "call stack to id mapping mismatch"));
311 return false;
312 }
313 return true;
314}
315
316bool InstrProfWriter::addMemProfData(memprof::IndexedMemProfData Incoming,
317 function_ref<void(Error)> Warn) {
318 // Return immediately if everything is empty.
319 if (Incoming.Frames.empty() && Incoming.CallStacks.empty() &&
320 Incoming.Records.empty())
321 return true;
322
323 // Otherwise, every component must be non-empty.
324 assert(!Incoming.Frames.empty() && !Incoming.CallStacks.empty() &&
325 !Incoming.Records.empty());
326
327 if (MemProfData.Frames.empty())
328 MemProfData.Frames = std::move(Incoming.Frames);
329 else
330 for (const auto &[Id, F] : Incoming.Frames)
331 if (addMemProfFrame(Id, Frame: F, Warn))
332 return false;
333
334 if (MemProfData.CallStacks.empty())
335 MemProfData.CallStacks = std::move(Incoming.CallStacks);
336 else
337 for (const auto &[CSId, CS] : Incoming.CallStacks)
338 if (addMemProfCallStack(CSId, CallStack: CS, Warn))
339 return false;
340
341 // Add one record at a time if randomization is requested.
342 if (MemProfData.Records.empty() && !MemprofGenerateRandomHotness) {
343 // Need to manually add each record to the builder, which is otherwise done
344 // in addMemProfRecord.
345 for (const auto &[GUID, Record] : Incoming.Records)
346 MemProfSumBuilder.addRecord(Record);
347 MemProfData.Records = std::move(Incoming.Records);
348 } else {
349 for (const auto &[GUID, Record] : Incoming.Records)
350 addMemProfRecord(Id: GUID, Record);
351 }
352
353 return true;
354}
355
356void InstrProfWriter::addBinaryIds(ArrayRef<llvm::object::BuildID> BIs) {
357 llvm::append_range(C&: BinaryIds, R&: BIs);
358}
359
360void InstrProfWriter::addDataAccessProfData(
361 std::unique_ptr<memprof::DataAccessProfData> DataAccessProfDataIn) {
362 DataAccessProfileData = std::move(DataAccessProfDataIn);
363}
364
365void InstrProfWriter::addTemporalProfileTraces(
366 SmallVectorImpl<TemporalProfTraceTy> &SrcTraces, uint64_t SrcStreamSize) {
367 if (TemporalProfTraces.size() > TemporalProfTraceReservoirSize)
368 TemporalProfTraces.truncate(N: TemporalProfTraceReservoirSize);
369 for (auto &Trace : SrcTraces)
370 if (Trace.FunctionNameRefs.size() > MaxTemporalProfTraceLength)
371 Trace.FunctionNameRefs.resize(new_size: MaxTemporalProfTraceLength);
372 llvm::erase_if(C&: SrcTraces, P: [](auto &T) { return T.FunctionNameRefs.empty(); });
373 // If there are no source traces, it is probably because
374 // --temporal-profile-max-trace-length=0 was set to deliberately remove all
375 // traces. In that case, we do not want to increase the stream size
376 if (SrcTraces.empty())
377 return;
378 // Add traces until our reservoir is full or we run out of source traces
379 auto SrcTraceIt = SrcTraces.begin();
380 while (TemporalProfTraces.size() < TemporalProfTraceReservoirSize &&
381 SrcTraceIt < SrcTraces.end())
382 TemporalProfTraces.push_back(Elt: *SrcTraceIt++);
383 // Our reservoir is full, we need to sample the source stream
384 llvm::shuffle(first: SrcTraceIt, last: SrcTraces.end(), g&: RNG);
385 for (uint64_t I = TemporalProfTraces.size();
386 I < SrcStreamSize && SrcTraceIt < SrcTraces.end(); I++) {
387 std::uniform_int_distribution<uint64_t> Distribution(0, I);
388 uint64_t RandomIndex = Distribution(RNG);
389 if (RandomIndex < TemporalProfTraces.size())
390 TemporalProfTraces[RandomIndex] = *SrcTraceIt++;
391 }
392 TemporalProfTraceStreamSize += SrcStreamSize;
393}
394
395void InstrProfWriter::mergeRecordsFromWriter(InstrProfWriter &&IPW,
396 function_ref<void(Error)> Warn) {
397 for (auto &I : IPW.FunctionData)
398 for (auto &Func : I.getValue())
399 addRecord(Name: I.getKey(), Hash: Func.first, I: std::move(Func.second), Weight: 1, Warn);
400
401 BinaryIds.reserve(n: BinaryIds.size() + IPW.BinaryIds.size());
402 for (auto &I : IPW.BinaryIds)
403 addBinaryIds(BIs: I);
404
405 addTemporalProfileTraces(SrcTraces&: IPW.TemporalProfTraces,
406 SrcStreamSize: IPW.TemporalProfTraceStreamSize);
407
408 MemProfData.Frames.reserve(NumEntries: IPW.MemProfData.Frames.size());
409 for (auto &[FrameId, Frame] : IPW.MemProfData.Frames) {
410 // If we weren't able to add the frame mappings then it doesn't make sense
411 // to try to merge the records from this profile.
412 if (!addMemProfFrame(Id: FrameId, Frame, Warn))
413 return;
414 }
415
416 MemProfData.CallStacks.reserve(NumEntries: IPW.MemProfData.CallStacks.size());
417 for (auto &[CSId, CallStack] : IPW.MemProfData.CallStacks) {
418 if (!addMemProfCallStack(CSId, CallStack, Warn))
419 return;
420 }
421
422 MemProfData.Records.reserve(NumEntries: IPW.MemProfData.Records.size());
423 for (auto &[GUID, Record] : IPW.MemProfData.Records) {
424 addMemProfRecord(Id: GUID, Record);
425 }
426}
427
428bool InstrProfWriter::shouldEncodeData(const ProfilingData &PD) {
429 if (!Sparse)
430 return true;
431 for (const auto &Func : PD) {
432 const InstrProfRecord &IPR = Func.second;
433 if (llvm::any_of(Range: IPR.Counts, P: [](uint64_t Count) { return Count > 0; }))
434 return true;
435 if (llvm::any_of(Range: IPR.BitmapBytes, P: [](uint8_t Byte) { return Byte > 0; }))
436 return true;
437 }
438 return false;
439}
440
441static void setSummary(IndexedInstrProf::Summary *TheSummary,
442 ProfileSummary &PS) {
443 using namespace IndexedInstrProf;
444
445 const std::vector<ProfileSummaryEntry> &Res = PS.getDetailedSummary();
446 TheSummary->NumSummaryFields = Summary::NumKinds;
447 TheSummary->NumCutoffEntries = Res.size();
448 TheSummary->set(K: Summary::MaxFunctionCount, V: PS.getMaxFunctionCount());
449 TheSummary->set(K: Summary::MaxBlockCount, V: PS.getMaxCount());
450 TheSummary->set(K: Summary::MaxInternalBlockCount, V: PS.getMaxInternalCount());
451 TheSummary->set(K: Summary::TotalBlockCount, V: PS.getTotalCount());
452 TheSummary->set(K: Summary::TotalNumBlocks, V: PS.getNumCounts());
453 TheSummary->set(K: Summary::TotalNumFunctions, V: PS.getNumFunctions());
454 for (unsigned I = 0; I < Res.size(); I++)
455 TheSummary->setEntry(I, E: Res[I]);
456}
457
458uint64_t InstrProfWriter::writeHeader(const IndexedInstrProf::Header &Header,
459 const bool WritePrevVersion,
460 ProfOStream &OS) {
461 // Only write out the first four fields.
462 for (int I = 0; I < 4; I++)
463 OS.write(V: reinterpret_cast<const uint64_t *>(&Header)[I]);
464
465 // Remember the offset of the remaining fields to allow back patching later.
466 auto BackPatchStartOffset = OS.tell();
467
468 // Reserve the space for back patching later.
469 OS.write(V: 0); // HashOffset
470 OS.write(V: 0); // MemProfOffset
471 OS.write(V: 0); // BinaryIdOffset
472 OS.write(V: 0); // TemporalProfTracesOffset
473 if (!WritePrevVersion)
474 OS.write(V: 0); // VTableNamesOffset
475
476 return BackPatchStartOffset;
477}
478
479Error InstrProfWriter::writeBinaryIds(ProfOStream &OS) {
480 // BinaryIdSection has two parts:
481 // 1. uint64_t BinaryIdsSectionSize
482 // 2. list of binary ids that consist of:
483 // a. uint64_t BinaryIdLength
484 // b. uint8_t BinaryIdData
485 // c. uint8_t Padding (if necessary)
486 // Calculate size of binary section.
487 uint64_t BinaryIdsSectionSize = 0;
488
489 // Remove duplicate binary ids.
490 llvm::sort(C&: BinaryIds);
491 BinaryIds.erase(first: llvm::unique(R&: BinaryIds), last: BinaryIds.end());
492
493 for (const auto &BI : BinaryIds) {
494 // Increment by binary id length data type size.
495 BinaryIdsSectionSize += sizeof(uint64_t);
496 // Increment by binary id data length, aligned to 8 bytes.
497 BinaryIdsSectionSize += alignToPowerOf2(Value: BI.size(), Align: sizeof(uint64_t));
498 }
499 // Write binary ids section size.
500 OS.write(V: BinaryIdsSectionSize);
501
502 for (const auto &BI : BinaryIds) {
503 uint64_t BILen = BI.size();
504 // Write binary id length.
505 OS.write(V: BILen);
506 // Write binary id data.
507 for (unsigned K = 0; K < BILen; K++)
508 OS.writeByte(V: BI[K]);
509 // Write padding if necessary.
510 uint64_t PaddingSize = alignToPowerOf2(Value: BILen, Align: sizeof(uint64_t)) - BILen;
511 for (unsigned K = 0; K < PaddingSize; K++)
512 OS.writeByte(V: 0);
513 }
514
515 return Error::success();
516}
517
518Error InstrProfWriter::writeVTableNames(ProfOStream &OS) {
519 std::vector<std::string> VTableNameStrs;
520 for (StringRef VTableName : VTableNames.keys())
521 VTableNameStrs.push_back(x: VTableName.str());
522
523 std::string CompressedVTableNames;
524 if (!VTableNameStrs.empty())
525 if (Error E = collectGlobalObjectNameStrings(
526 NameStrs: VTableNameStrs, doCompression: compression::zlib::isAvailable(),
527 Result&: CompressedVTableNames))
528 return E;
529
530 const uint64_t CompressedStringLen = CompressedVTableNames.length();
531
532 // Record the length of compressed string.
533 OS.write(V: CompressedStringLen);
534
535 // Write the chars in compressed strings.
536 for (auto &c : CompressedVTableNames)
537 OS.writeByte(V: static_cast<uint8_t>(c));
538
539 // Pad up to a multiple of 8.
540 // InstrProfReader could read bytes according to 'CompressedStringLen'.
541 const uint64_t PaddedLength = alignTo(Value: CompressedStringLen, Align: 8);
542
543 for (uint64_t K = CompressedStringLen; K < PaddedLength; K++)
544 OS.writeByte(V: 0);
545
546 return Error::success();
547}
548
549Error InstrProfWriter::writeImpl(ProfOStream &OS) {
550 using namespace IndexedInstrProf;
551 using namespace support;
552
553 OnDiskChainedHashTableGenerator<InstrProfRecordWriterTrait> Generator;
554
555 InstrProfSummaryBuilder ISB(ProfileSummaryBuilder::DefaultCutoffs);
556 InfoObj->SummaryBuilder = &ISB;
557 InstrProfSummaryBuilder CSISB(ProfileSummaryBuilder::DefaultCutoffs);
558 InfoObj->CSSummaryBuilder = &CSISB;
559 InfoObj->WritePrevVersion = WritePrevVersion;
560
561 // Populate the hash table generator.
562 SmallVector<std::pair<StringRef, const ProfilingData *>> OrderedData;
563 for (const auto &I : FunctionData)
564 if (shouldEncodeData(PD: I.getValue()))
565 OrderedData.emplace_back(Args: (I.getKey()), Args: &I.getValue());
566 llvm::sort(C&: OrderedData, Comp: less_first());
567 for (const auto &I : OrderedData)
568 Generator.insert(Key: I.first, Data: I.second);
569
570 // Write the header.
571 IndexedInstrProf::Header Header;
572 Header.Version = WritePrevVersion
573 ? IndexedInstrProf::ProfVersion::Version11
574 : IndexedInstrProf::ProfVersion::CurrentVersion;
575 // The WritePrevVersion handling will either need to be removed or updated
576 // if the version is advanced beyond 12.
577 static_assert(IndexedInstrProf::ProfVersion::CurrentVersion ==
578 IndexedInstrProf::ProfVersion::Version14);
579 if (static_cast<bool>(ProfileKind & InstrProfKind::IRInstrumentation))
580 Header.Version |= VARIANT_MASK_IR_PROF;
581 if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive))
582 Header.Version |= VARIANT_MASK_CSIR_PROF;
583 if (static_cast<bool>(ProfileKind &
584 InstrProfKind::FunctionEntryInstrumentation))
585 Header.Version |= VARIANT_MASK_INSTR_ENTRY;
586 if (static_cast<bool>(ProfileKind &
587 InstrProfKind::LoopEntriesInstrumentation))
588 Header.Version |= VARIANT_MASK_INSTR_LOOP_ENTRIES;
589 if (static_cast<bool>(ProfileKind & InstrProfKind::SingleByteCoverage))
590 Header.Version |= VARIANT_MASK_BYTE_COVERAGE;
591 if (static_cast<bool>(ProfileKind & InstrProfKind::FunctionEntryOnly))
592 Header.Version |= VARIANT_MASK_FUNCTION_ENTRY_ONLY;
593 if (static_cast<bool>(ProfileKind & InstrProfKind::MemProf))
594 Header.Version |= VARIANT_MASK_MEMPROF;
595 if (static_cast<bool>(ProfileKind & InstrProfKind::TemporalProfile))
596 Header.Version |= VARIANT_MASK_TEMPORAL_PROF;
597
598 const uint64_t BackPatchStartOffset =
599 writeHeader(Header, WritePrevVersion, OS);
600
601 // Reserve space to write profile summary data.
602 uint32_t NumEntries = ProfileSummaryBuilder::DefaultCutoffs.size();
603 uint32_t SummarySize = Summary::getSize(NumSumFields: Summary::NumKinds, NumCutoffEntries: NumEntries);
604 // Remember the summary offset.
605 uint64_t SummaryOffset = OS.tell();
606 for (unsigned I = 0; I < SummarySize / sizeof(uint64_t); I++)
607 OS.write(V: 0);
608 uint64_t CSSummaryOffset = 0;
609 uint64_t CSSummarySize = 0;
610 if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive)) {
611 CSSummaryOffset = OS.tell();
612 CSSummarySize = SummarySize / sizeof(uint64_t);
613 for (unsigned I = 0; I < CSSummarySize; I++)
614 OS.write(V: 0);
615 }
616
617 // Write the hash table.
618 uint64_t HashTableStart = Generator.Emit(Out&: OS.OS, InfoObj&: *InfoObj);
619
620 // Write the MemProf profile data if we have it.
621 uint64_t MemProfSectionStart = 0;
622 if (static_cast<bool>(ProfileKind & InstrProfKind::MemProf)) {
623 MemProfSectionStart = OS.tell();
624
625 if (auto E = writeMemProf(
626 OS, MemProfData, MemProfVersionRequested, MemProfFullSchema,
627 DataAccessProfileData: std::move(DataAccessProfileData), MemProfSum: MemProfSumBuilder.getSummary()))
628 return E;
629 }
630
631 uint64_t BinaryIdSectionStart = OS.tell();
632 if (auto E = writeBinaryIds(OS))
633 return E;
634
635 uint64_t VTableNamesSectionStart = OS.tell();
636
637 if (!WritePrevVersion)
638 if (Error E = writeVTableNames(OS))
639 return E;
640
641 uint64_t TemporalProfTracesSectionStart = 0;
642 if (static_cast<bool>(ProfileKind & InstrProfKind::TemporalProfile)) {
643 TemporalProfTracesSectionStart = OS.tell();
644 OS.write(V: TemporalProfTraces.size());
645 OS.write(V: TemporalProfTraceStreamSize);
646 for (auto &Trace : TemporalProfTraces) {
647 OS.write(V: Trace.Weight);
648 OS.write(V: Trace.FunctionNameRefs.size());
649 for (auto &NameRef : Trace.FunctionNameRefs)
650 OS.write(V: NameRef);
651 }
652 }
653
654 // Allocate space for data to be serialized out.
655 std::unique_ptr<IndexedInstrProf::Summary> TheSummary =
656 IndexedInstrProf::allocSummary(TotalSize: SummarySize);
657 // Compute the Summary and copy the data to the data
658 // structure to be serialized out (to disk or buffer).
659 std::unique_ptr<ProfileSummary> PS = ISB.getSummary();
660 setSummary(TheSummary: TheSummary.get(), PS&: *PS);
661 InfoObj->SummaryBuilder = nullptr;
662
663 // For Context Sensitive summary.
664 std::unique_ptr<IndexedInstrProf::Summary> TheCSSummary = nullptr;
665 if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive)) {
666 TheCSSummary = IndexedInstrProf::allocSummary(TotalSize: SummarySize);
667 std::unique_ptr<ProfileSummary> CSPS = CSISB.getSummary();
668 setSummary(TheSummary: TheCSSummary.get(), PS&: *CSPS);
669 }
670 InfoObj->CSSummaryBuilder = nullptr;
671
672 SmallVector<uint64_t, 8> HeaderOffsets = {HashTableStart, MemProfSectionStart,
673 BinaryIdSectionStart,
674 TemporalProfTracesSectionStart};
675 if (!WritePrevVersion)
676 HeaderOffsets.push_back(Elt: VTableNamesSectionStart);
677
678 PatchItem PatchItems[] = {
679 // Patch the Header fields
680 {.Pos: BackPatchStartOffset, .D: HeaderOffsets},
681 // Patch the summary data.
682 {.Pos: SummaryOffset,
683 .D: ArrayRef<uint64_t>(reinterpret_cast<uint64_t *>(TheSummary.get()),
684 SummarySize / sizeof(uint64_t))},
685 {.Pos: CSSummaryOffset,
686 .D: ArrayRef<uint64_t>(reinterpret_cast<uint64_t *>(TheCSSummary.get()),
687 CSSummarySize)}};
688
689 OS.patch(P: PatchItems);
690
691 for (const auto &I : FunctionData)
692 for (const auto &F : I.getValue())
693 if (Error E = validateRecord(Func: F.second))
694 return E;
695
696 return Error::success();
697}
698
699Error InstrProfWriter::write(raw_fd_ostream &OS) {
700 // Write the hash table.
701 ProfOStream POS(OS);
702 return writeImpl(OS&: POS);
703}
704
705Error InstrProfWriter::write(raw_string_ostream &OS) {
706 ProfOStream POS(OS);
707 return writeImpl(OS&: POS);
708}
709
710std::unique_ptr<MemoryBuffer> InstrProfWriter::writeBuffer() {
711 std::string Data;
712 raw_string_ostream OS(Data);
713 // Write the hash table.
714 if (Error E = write(OS))
715 return nullptr;
716 // Return this in an aligned memory buffer.
717 return MemoryBuffer::getMemBufferCopy(InputData: Data);
718}
719
720static const char *ValueProfKindStr[] = {
721#define VALUE_PROF_KIND(Enumerator, Value, Descr) #Enumerator,
722#include "llvm/ProfileData/InstrProfData.inc"
723};
724
725Error InstrProfWriter::validateRecord(const InstrProfRecord &Func) {
726 for (uint32_t VK = 0; VK <= IPVK_Last; VK++) {
727 if (VK == IPVK_IndirectCallTarget || VK == IPVK_VTableTarget)
728 continue;
729 uint32_t NS = Func.getNumValueSites(ValueKind: VK);
730 for (uint32_t S = 0; S < NS; S++) {
731 DenseSet<uint64_t> SeenValues;
732 for (const auto &V : Func.getValueArrayForSite(ValueKind: VK, Site: S))
733 if (!SeenValues.insert(V: V.Value).second)
734 return make_error<InstrProfError>(Args: instrprof_error::invalid_prof);
735 }
736 }
737
738 return Error::success();
739}
740
741void InstrProfWriter::writeRecordInText(StringRef Name, uint64_t Hash,
742 const InstrProfRecord &Func,
743 InstrProfSymtab &Symtab,
744 raw_fd_ostream &OS) {
745 OS << Name << "\n";
746 OS << "# Func Hash:\n" << Hash << "\n";
747 OS << "# Num Counters:\n" << Func.Counts.size() << "\n";
748 OS << "# Counter Values:\n";
749 for (uint64_t Count : Func.Counts)
750 OS << Count << "\n";
751
752 if (Func.BitmapBytes.size() > 0) {
753 OS << "# Num Bitmap Bytes:\n$" << Func.BitmapBytes.size() << "\n";
754 OS << "# Bitmap Byte Values:\n";
755 for (uint8_t Byte : Func.BitmapBytes) {
756 OS << "0x";
757 OS.write_hex(N: Byte);
758 OS << "\n";
759 }
760 OS << "\n";
761 }
762
763 uint32_t NumValueKinds = Func.getNumValueKinds();
764 if (!NumValueKinds) {
765 OS << "\n";
766 return;
767 }
768
769 OS << "# Num Value Kinds:\n" << Func.getNumValueKinds() << "\n";
770 for (uint32_t VK = 0; VK < IPVK_Last + 1; VK++) {
771 uint32_t NS = Func.getNumValueSites(ValueKind: VK);
772 if (!NS)
773 continue;
774 OS << "# ValueKind = " << ValueProfKindStr[VK] << ":\n" << VK << "\n";
775 OS << "# NumValueSites:\n" << NS << "\n";
776 for (uint32_t S = 0; S < NS; S++) {
777 auto VD = Func.getValueArrayForSite(ValueKind: VK, Site: S);
778 OS << VD.size() << "\n";
779 for (const auto &V : VD) {
780 if (VK == IPVK_IndirectCallTarget || VK == IPVK_VTableTarget)
781 OS << Symtab.getFuncOrVarNameIfDefined(MD5Hash: V.Value) << ":" << V.Count
782 << "\n";
783 else
784 OS << V.Value << ":" << V.Count << "\n";
785 }
786 }
787 }
788
789 OS << "\n";
790}
791
792Error InstrProfWriter::writeText(raw_fd_ostream &OS) {
793 // Check CS first since it implies an IR level profile.
794 if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive))
795 OS << "# CSIR level Instrumentation Flag\n:csir\n";
796 else if (static_cast<bool>(ProfileKind & InstrProfKind::IRInstrumentation))
797 OS << "# IR level Instrumentation Flag\n:ir\n";
798
799 if (static_cast<bool>(ProfileKind &
800 InstrProfKind::FunctionEntryInstrumentation))
801 OS << "# Always instrument the function entry block\n:entry_first\n";
802 if (static_cast<bool>(ProfileKind &
803 InstrProfKind::LoopEntriesInstrumentation))
804 OS << "# Always instrument the loop entry "
805 "blocks\n:instrument_loop_entries\n";
806 if (static_cast<bool>(ProfileKind & InstrProfKind::SingleByteCoverage))
807 OS << "# Instrument block coverage\n:single_byte_coverage\n";
808 InstrProfSymtab Symtab;
809
810 using FuncPair = detail::DenseMapPair<uint64_t, InstrProfRecord>;
811 using RecordType = std::pair<StringRef, FuncPair>;
812 SmallVector<RecordType, 4> OrderedFuncData;
813
814 for (const auto &I : FunctionData) {
815 if (shouldEncodeData(PD: I.getValue())) {
816 if (Error E = Symtab.addFuncName(FuncName: I.getKey()))
817 return E;
818 for (const auto &Func : I.getValue())
819 OrderedFuncData.push_back(Elt: std::make_pair(x: I.getKey(), y: Func));
820 }
821 }
822
823 for (const auto &VTableName : VTableNames)
824 if (Error E = Symtab.addVTableName(VTableName: VTableName.getKey()))
825 return E;
826
827 if (static_cast<bool>(ProfileKind & InstrProfKind::TemporalProfile))
828 writeTextTemporalProfTraceData(OS, Symtab);
829
830 llvm::sort(C&: OrderedFuncData, Comp: [](const RecordType &A, const RecordType &B) {
831 return std::tie(args: A.first, args: A.second.first) <
832 std::tie(args: B.first, args: B.second.first);
833 });
834
835 for (const auto &record : OrderedFuncData) {
836 const StringRef &Name = record.first;
837 const FuncPair &Func = record.second;
838 writeRecordInText(Name, Hash: Func.first, Func: Func.second, Symtab, OS);
839 }
840
841 for (const auto &record : OrderedFuncData) {
842 const FuncPair &Func = record.second;
843 if (Error E = validateRecord(Func: Func.second))
844 return E;
845 }
846
847 return Error::success();
848}
849
850void InstrProfWriter::writeTextTemporalProfTraceData(raw_fd_ostream &OS,
851 InstrProfSymtab &Symtab) {
852 OS << ":temporal_prof_traces\n";
853 OS << "# Num Temporal Profile Traces:\n" << TemporalProfTraces.size() << "\n";
854 OS << "# Temporal Profile Trace Stream Size:\n"
855 << TemporalProfTraceStreamSize << "\n";
856 for (auto &Trace : TemporalProfTraces) {
857 OS << "# Weight:\n" << Trace.Weight << "\n";
858 for (auto &NameRef : Trace.FunctionNameRefs)
859 OS << Symtab.getFuncOrVarName(MD5Hash: NameRef) << ",";
860 OS << "\n";
861 }
862 OS << "\n";
863}
864