1//===- PGOCtxProfWriter.cpp - Contextual Instrumentation profile writer ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Write a contextual profile to bitstream.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/ProfileData/PGOCtxProfWriter.h"
14#include "llvm/Bitstream/BitCodeEnums.h"
15#include "llvm/ProfileData/CtxInstrContextNode.h"
16#include "llvm/Support/CommandLine.h"
17#include "llvm/Support/Error.h"
18#include "llvm/Support/YAMLTraits.h"
19#include "llvm/Support/raw_ostream.h"
20
21using namespace llvm;
22using namespace llvm::ctx_profile;
23
24static cl::opt<bool>
25 IncludeEmptyOpt("ctx-prof-include-empty", cl::init(Val: false),
26 cl::desc("Also write profiles with all-zero counters. "
27 "Intended for testing/debugging."));
28
29PGOCtxProfileWriter::PGOCtxProfileWriter(
30 raw_ostream &Out, std::optional<unsigned> VersionOverride,
31 bool IncludeEmpty)
32 : Writer(Out, 0),
33 IncludeEmpty(IncludeEmptyOpt.getNumOccurrences() > 0 ? IncludeEmptyOpt
34 : IncludeEmpty) {
35 static_assert(ContainerMagic.size() == 4);
36 Out.write(Ptr: ContainerMagic.data(), Size: ContainerMagic.size());
37 Writer.EnterBlockInfoBlock();
38 {
39 auto DescribeBlock = [&](unsigned ID, StringRef Name) {
40 Writer.EmitRecord(Code: bitc::BLOCKINFO_CODE_SETBID,
41 Vals: SmallVector<unsigned, 1>{ID});
42 Writer.EmitRecord(Code: bitc::BLOCKINFO_CODE_BLOCKNAME,
43 Vals: llvm::arrayRefFromStringRef(Input: Name));
44 };
45 SmallVector<uint64_t, 16> Data;
46 auto DescribeRecord = [&](unsigned RecordID, StringRef Name) {
47 Data.clear();
48 Data.push_back(Elt: RecordID);
49 llvm::append_range(C&: Data, R&: Name);
50 Writer.EmitRecord(Code: bitc::BLOCKINFO_CODE_SETRECORDNAME, Vals: Data);
51 };
52 DescribeBlock(PGOCtxProfileBlockIDs::ProfileMetadataBlockID, "Metadata");
53 DescribeRecord(PGOCtxProfileRecords::Version, "Version");
54 DescribeBlock(PGOCtxProfileBlockIDs::ContextsSectionBlockID, "Contexts");
55 DescribeBlock(PGOCtxProfileBlockIDs::ContextRootBlockID, "Root");
56 DescribeRecord(PGOCtxProfileRecords::Guid, "GUID");
57 DescribeRecord(PGOCtxProfileRecords::TotalRootEntryCount,
58 "TotalRootEntryCount");
59 DescribeRecord(PGOCtxProfileRecords::Counters, "Counters");
60 DescribeBlock(PGOCtxProfileBlockIDs::UnhandledBlockID, "Unhandled");
61 DescribeBlock(PGOCtxProfileBlockIDs::ContextNodeBlockID, "Context");
62 DescribeRecord(PGOCtxProfileRecords::Guid, "GUID");
63 DescribeRecord(PGOCtxProfileRecords::CallsiteIndex, "CalleeIndex");
64 DescribeRecord(PGOCtxProfileRecords::Counters, "Counters");
65 DescribeBlock(PGOCtxProfileBlockIDs::FlatProfilesSectionBlockID,
66 "FlatProfiles");
67 DescribeBlock(PGOCtxProfileBlockIDs::FlatProfileBlockID, "Flat");
68 DescribeRecord(PGOCtxProfileRecords::Guid, "GUID");
69 DescribeRecord(PGOCtxProfileRecords::Counters, "Counters");
70 }
71 Writer.ExitBlock();
72 Writer.EnterSubblock(BlockID: PGOCtxProfileBlockIDs::ProfileMetadataBlockID, CodeLen);
73 const auto Version = VersionOverride.value_or(u: CurrentVersion);
74 Writer.EmitRecord(Code: PGOCtxProfileRecords::Version,
75 Vals: SmallVector<unsigned, 1>({Version}));
76}
77
78void PGOCtxProfileWriter::writeCounters(ArrayRef<uint64_t> Counters) {
79 Writer.EmitCode(Val: bitc::UNABBREV_RECORD);
80 Writer.EmitVBR(Val: PGOCtxProfileRecords::Counters, NumBits: VBREncodingBits);
81 Writer.EmitVBR(Val: Counters.size(), NumBits: VBREncodingBits);
82 for (uint64_t C : Counters)
83 Writer.EmitVBR64(Val: C, NumBits: VBREncodingBits);
84}
85
86void PGOCtxProfileWriter::writeGuid(ctx_profile::GUID Guid) {
87 Writer.EmitRecord(Code: PGOCtxProfileRecords::Guid, Vals: SmallVector<uint64_t, 1>{Guid});
88}
89
90void PGOCtxProfileWriter::writeCallsiteIndex(uint32_t CallsiteIndex) {
91 Writer.EmitRecord(Code: PGOCtxProfileRecords::CallsiteIndex,
92 Vals: SmallVector<uint64_t, 1>{CallsiteIndex});
93}
94
95void PGOCtxProfileWriter::writeRootEntryCount(uint64_t TotalRootEntryCount) {
96 Writer.EmitRecord(Code: PGOCtxProfileRecords::TotalRootEntryCount,
97 Vals: SmallVector<uint64_t, 1>{TotalRootEntryCount});
98}
99
100// recursively write all the subcontexts. We do need to traverse depth first to
101// model the context->subcontext implicitly, and since this captures call
102// stacks, we don't really need to be worried about stack overflow and we can
103// keep the implementation simple.
104void PGOCtxProfileWriter::writeNode(uint32_t CallsiteIndex,
105 const ContextNode &Node) {
106 // A node with no counters is an error. We don't expect this to happen from
107 // the runtime, rather, this is interesting for testing the reader.
108 if (!IncludeEmpty && (Node.counters_size() > 0 && Node.entrycount() == 0))
109 return;
110 Writer.EnterSubblock(BlockID: PGOCtxProfileBlockIDs::ContextNodeBlockID, CodeLen);
111 writeGuid(Guid: Node.guid());
112 writeCallsiteIndex(CallsiteIndex);
113 writeCounters(Counters: {Node.counters(), Node.counters_size()});
114 writeSubcontexts(Node);
115 Writer.ExitBlock();
116}
117
118void PGOCtxProfileWriter::writeSubcontexts(const ContextNode &Node) {
119 for (uint32_t I = 0U; I < Node.callsites_size(); ++I)
120 for (const auto *Subcontext = Node.subContexts()[I]; Subcontext;
121 Subcontext = Subcontext->next())
122 writeNode(CallsiteIndex: I, Node: *Subcontext);
123}
124
125void PGOCtxProfileWriter::startContextSection() {
126 Writer.EnterSubblock(BlockID: PGOCtxProfileBlockIDs::ContextsSectionBlockID, CodeLen);
127}
128
129void PGOCtxProfileWriter::startFlatSection() {
130 Writer.EnterSubblock(BlockID: PGOCtxProfileBlockIDs::FlatProfilesSectionBlockID,
131 CodeLen);
132}
133
134void PGOCtxProfileWriter::endContextSection() { Writer.ExitBlock(); }
135void PGOCtxProfileWriter::endFlatSection() { Writer.ExitBlock(); }
136
137void PGOCtxProfileWriter::writeContextual(const ContextNode &RootNode,
138 const ContextNode *Unhandled,
139 uint64_t TotalRootEntryCount) {
140 if (!IncludeEmpty && (!TotalRootEntryCount || (RootNode.counters_size() > 0 &&
141 RootNode.entrycount() == 0)))
142 return;
143 Writer.EnterSubblock(BlockID: PGOCtxProfileBlockIDs::ContextRootBlockID, CodeLen);
144 writeGuid(Guid: RootNode.guid());
145 writeRootEntryCount(TotalRootEntryCount);
146 writeCounters(Counters: {RootNode.counters(), RootNode.counters_size()});
147
148 Writer.EnterSubblock(BlockID: PGOCtxProfileBlockIDs::UnhandledBlockID, CodeLen);
149 for (const auto *P = Unhandled; P; P = P->next())
150 writeFlat(Guid: P->guid(), Buffer: P->counters(), BufferSize: P->counters_size());
151 Writer.ExitBlock();
152
153 writeSubcontexts(Node: RootNode);
154 Writer.ExitBlock();
155}
156
157void PGOCtxProfileWriter::writeFlat(ctx_profile::GUID Guid,
158 const uint64_t *Buffer, size_t Size) {
159 Writer.EnterSubblock(BlockID: PGOCtxProfileBlockIDs::FlatProfileBlockID, CodeLen);
160 writeGuid(Guid);
161 writeCounters(Counters: {Buffer, Size});
162 Writer.ExitBlock();
163}
164
165namespace {
166
167/// Representation of the context node suitable for yaml serialization /
168/// deserialization.
169using SerializableFlatProfileRepresentation =
170 std::pair<ctx_profile::GUID, std::vector<uint64_t>>;
171
172struct SerializableCtxRepresentation {
173 ctx_profile::GUID Guid = 0;
174 std::vector<uint64_t> Counters;
175 std::vector<std::vector<SerializableCtxRepresentation>> Callsites;
176};
177
178struct SerializableRootRepresentation : public SerializableCtxRepresentation {
179 uint64_t TotalRootEntryCount = 0;
180 std::vector<SerializableFlatProfileRepresentation> Unhandled;
181};
182
183struct SerializableProfileRepresentation {
184 std::vector<SerializableRootRepresentation> Contexts;
185 std::vector<SerializableFlatProfileRepresentation> FlatProfiles;
186};
187
188ctx_profile::ContextNode *
189createNode(std::vector<std::unique_ptr<char[]>> &Nodes,
190 const std::vector<SerializableCtxRepresentation> &DCList);
191
192// Convert a DeserializableCtx into a ContextNode, potentially linking it to
193// its sibling (e.g. callee at same callsite) "Next".
194ctx_profile::ContextNode *
195createNode(std::vector<std::unique_ptr<char[]>> &Nodes,
196 const SerializableCtxRepresentation &DC,
197 ctx_profile::ContextNode *Next = nullptr) {
198 auto AllocSize = ctx_profile::ContextNode::getAllocSize(NumCounters: DC.Counters.size(),
199 NumCallsites: DC.Callsites.size());
200 auto *Mem = Nodes.emplace_back(args: std::make_unique<char[]>(num: AllocSize)).get();
201 std::memset(s: Mem, c: 0, n: AllocSize);
202 auto *Ret = new (Mem) ctx_profile::ContextNode(DC.Guid, DC.Counters.size(),
203 DC.Callsites.size(), Next);
204 std::memcpy(dest: Ret->counters(), src: DC.Counters.data(),
205 n: sizeof(uint64_t) * DC.Counters.size());
206 for (const auto &[I, DCList] : llvm::enumerate(First: DC.Callsites))
207 Ret->subContexts()[I] = createNode(Nodes, DCList);
208 return Ret;
209}
210
211// Convert a list of SerializableCtxRepresentation into a linked list of
212// ContextNodes.
213ctx_profile::ContextNode *
214createNode(std::vector<std::unique_ptr<char[]>> &Nodes,
215 const std::vector<SerializableCtxRepresentation> &DCList) {
216 ctx_profile::ContextNode *List = nullptr;
217 for (const auto &DC : DCList)
218 List = createNode(Nodes, DC, Next: List);
219 return List;
220}
221} // namespace
222
223LLVM_YAML_IS_SEQUENCE_VECTOR(SerializableCtxRepresentation)
224LLVM_YAML_IS_SEQUENCE_VECTOR(std::vector<SerializableCtxRepresentation>)
225LLVM_YAML_IS_SEQUENCE_VECTOR(SerializableRootRepresentation)
226LLVM_YAML_IS_SEQUENCE_VECTOR(SerializableFlatProfileRepresentation)
227template <> struct yaml::MappingTraits<SerializableCtxRepresentation> {
228 static void mapping(yaml::IO &IO, SerializableCtxRepresentation &SCR) {
229 IO.mapRequired(Key: "Guid", Val&: SCR.Guid);
230 IO.mapRequired(Key: "Counters", Val&: SCR.Counters);
231 IO.mapOptional(Key: "Callsites", Val&: SCR.Callsites);
232 }
233};
234
235template <> struct yaml::MappingTraits<SerializableRootRepresentation> {
236 static void mapping(yaml::IO &IO, SerializableRootRepresentation &R) {
237 yaml::MappingTraits<SerializableCtxRepresentation>::mapping(IO, SCR&: R);
238 IO.mapRequired(Key: "TotalRootEntryCount", Val&: R.TotalRootEntryCount);
239 IO.mapOptional(Key: "Unhandled", Val&: R.Unhandled);
240 }
241};
242
243template <> struct yaml::MappingTraits<SerializableProfileRepresentation> {
244 static void mapping(yaml::IO &IO, SerializableProfileRepresentation &SPR) {
245 IO.mapOptional(Key: "Contexts", Val&: SPR.Contexts);
246 IO.mapOptional(Key: "FlatProfiles", Val&: SPR.FlatProfiles);
247 }
248};
249
250template <> struct yaml::MappingTraits<SerializableFlatProfileRepresentation> {
251 static void mapping(yaml::IO &IO,
252 SerializableFlatProfileRepresentation &SFPR) {
253 IO.mapRequired(Key: "Guid", Val&: SFPR.first);
254 IO.mapRequired(Key: "Counters", Val&: SFPR.second);
255 }
256};
257
258Error llvm::createCtxProfFromYAML(StringRef Profile, raw_ostream &Out) {
259 yaml::Input In(Profile);
260 SerializableProfileRepresentation SPR;
261 In >> SPR;
262 if (In.error())
263 return createStringError(EC: In.error(), S: "incorrect yaml content");
264 std::vector<std::unique_ptr<char[]>> Nodes;
265 std::error_code EC;
266 if (EC)
267 return createStringError(EC, S: "failed to open output");
268 PGOCtxProfileWriter Writer(Out);
269
270 if (!SPR.Contexts.empty()) {
271 Writer.startContextSection();
272 for (const auto &DC : SPR.Contexts) {
273 auto *TopList = createNode(Nodes, DC);
274 if (!TopList)
275 return createStringError(
276 Fmt: "Unexpected error converting internal structure to ctx profile");
277
278 ctx_profile::ContextNode *FirstUnhandled = nullptr;
279 for (const auto &U : DC.Unhandled) {
280 SerializableCtxRepresentation Unhandled;
281 Unhandled.Guid = U.first;
282 Unhandled.Counters = U.second;
283 FirstUnhandled = createNode(Nodes, DC: Unhandled, Next: FirstUnhandled);
284 }
285 Writer.writeContextual(RootNode: *TopList, Unhandled: FirstUnhandled, TotalRootEntryCount: DC.TotalRootEntryCount);
286 }
287 Writer.endContextSection();
288 }
289 if (!SPR.FlatProfiles.empty()) {
290 Writer.startFlatSection();
291 for (const auto &[Guid, Counters] : SPR.FlatProfiles)
292 Writer.writeFlat(Guid, Buffer: Counters.data(), Size: Counters.size());
293 Writer.endFlatSection();
294 }
295 if (EC)
296 return createStringError(EC, S: "failed to write output");
297 return Error::success();
298}
299