1//===- PGOCtxProfReader.cpp - Contextual Instrumentation profile reader ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Read a contextual profile into a datastructure suitable for maintenance
10// throughout IPO
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/ProfileData/PGOCtxProfReader.h"
15#include "llvm/Bitstream/BitCodeEnums.h"
16#include "llvm/Bitstream/BitstreamReader.h"
17#include "llvm/ProfileData/InstrProf.h"
18#include "llvm/ProfileData/PGOCtxProfWriter.h"
19#include "llvm/Support/Error.h"
20#include "llvm/Support/ErrorHandling.h"
21#include "llvm/Support/YAMLTraits.h"
22#include <utility>
23
24using namespace llvm;
25
26// FIXME(#92054) - these Error handling macros are (re-)invented in a few
27// places.
28#define EXPECT_OR_RET(LHS, RHS) \
29 auto LHS = RHS; \
30 if (!LHS) \
31 return LHS.takeError();
32
33#define RET_ON_ERR(EXPR) \
34 if (auto Err = (EXPR)) \
35 return Err;
36
37Expected<PGOCtxProfContext &>
38PGOCtxProfContext::getOrEmplace(uint32_t Index, GlobalValue::GUID G,
39 SmallVectorImpl<uint64_t> &&Counters) {
40 auto [Iter, Inserted] =
41 Callsites[Index].insert(x: {G, PGOCtxProfContext(G, std::move(Counters))});
42 if (!Inserted)
43 return make_error<InstrProfError>(Args: instrprof_error::invalid_prof,
44 Args: "Duplicate GUID for same callsite.");
45 return Iter->second;
46}
47
48Expected<BitstreamEntry> PGOCtxProfileReader::advance() {
49 return Cursor.advance(Flags: BitstreamCursor::AF_DontAutoprocessAbbrevs);
50}
51
52Error PGOCtxProfileReader::wrongValue(const Twine &Msg) {
53 return make_error<InstrProfError>(Args: instrprof_error::invalid_prof, Args: Msg);
54}
55
56Error PGOCtxProfileReader::unsupported(const Twine &Msg) {
57 return make_error<InstrProfError>(Args: instrprof_error::unsupported_version, Args: Msg);
58}
59
60bool PGOCtxProfileReader::tryGetNextKnownBlockID(PGOCtxProfileBlockIDs &ID) {
61 auto Blk = advance();
62 if (!Blk) {
63 consumeError(Err: Blk.takeError());
64 return false;
65 }
66 if (Blk->Kind != BitstreamEntry::SubBlock)
67 return false;
68 if (PGOCtxProfileBlockIDs::FIRST_VALID > Blk->ID ||
69 PGOCtxProfileBlockIDs::LAST_VALID < Blk->ID)
70 return false;
71 ID = static_cast<PGOCtxProfileBlockIDs>(Blk->ID);
72 return true;
73}
74
75bool PGOCtxProfileReader::canEnterBlockWithID(PGOCtxProfileBlockIDs ID) {
76 PGOCtxProfileBlockIDs Test = {};
77 return tryGetNextKnownBlockID(ID&: Test) && Test == ID;
78}
79
80Error PGOCtxProfileReader::enterBlockWithID(PGOCtxProfileBlockIDs ID) {
81 RET_ON_ERR(Cursor.EnterSubBlock(ID));
82 return Error::success();
83}
84
85// Note: we use PGOCtxProfContext for flat profiles also, as the latter are
86// structurally similar. Alternative modeling here seems a bit overkill at the
87// moment.
88Expected<std::pair<std::optional<uint32_t>, PGOCtxProfContext>>
89PGOCtxProfileReader::readProfile(PGOCtxProfileBlockIDs Kind) {
90 assert((Kind == PGOCtxProfileBlockIDs::ContextRootBlockID ||
91 Kind == PGOCtxProfileBlockIDs::ContextNodeBlockID ||
92 Kind == PGOCtxProfileBlockIDs::FlatProfileBlockID) &&
93 "Unexpected profile kind");
94 RET_ON_ERR(enterBlockWithID(Kind));
95
96 std::optional<ctx_profile::GUID> Guid;
97 std::optional<SmallVector<uint64_t, 16>> Counters;
98 std::optional<uint32_t> CallsiteIndex;
99 std::optional<uint64_t> TotalEntryCount;
100 std::optional<CtxProfFlatProfile> Unhandled;
101 SmallVector<uint64_t, 1> RecordValues;
102
103 const bool ExpectIndex = Kind == PGOCtxProfileBlockIDs::ContextNodeBlockID;
104 const bool IsRoot = Kind == PGOCtxProfileBlockIDs::ContextRootBlockID;
105 // We don't prescribe the order in which the records come in, and we are ok
106 // if other unsupported records appear. We seek in the current subblock until
107 // we get all we know.
108 auto GotAllWeNeed = [&]() {
109 return Guid.has_value() && Counters.has_value() &&
110 (!ExpectIndex || CallsiteIndex.has_value()) &&
111 (!IsRoot || TotalEntryCount.has_value()) &&
112 (!IsRoot || Unhandled.has_value());
113 };
114
115 while (!GotAllWeNeed()) {
116 RecordValues.clear();
117 EXPECT_OR_RET(Entry, advance());
118 if (Entry->Kind != BitstreamEntry::Record) {
119 if (IsRoot && Entry->Kind == BitstreamEntry::SubBlock &&
120 Entry->ID == PGOCtxProfileBlockIDs::UnhandledBlockID) {
121 RET_ON_ERR(enterBlockWithID(PGOCtxProfileBlockIDs::UnhandledBlockID));
122 Unhandled = CtxProfFlatProfile();
123 RET_ON_ERR(loadFlatProfileList(*Unhandled));
124 continue;
125 }
126 return wrongValue(
127 Msg: "Expected records before encountering more subcontexts");
128 }
129 EXPECT_OR_RET(ReadRecord,
130 Cursor.readRecord(bitc::UNABBREV_RECORD, RecordValues));
131 switch (*ReadRecord) {
132 case PGOCtxProfileRecords::Guid:
133 if (RecordValues.size() != 1)
134 return wrongValue(Msg: "The GUID record should have exactly one value");
135 Guid = RecordValues[0];
136 break;
137 case PGOCtxProfileRecords::Counters:
138 Counters = std::move(RecordValues);
139 if (Counters->empty())
140 return wrongValue(Msg: "Empty counters. At least the entry counter (one "
141 "value) was expected");
142 break;
143 case PGOCtxProfileRecords::CallsiteIndex:
144 if (!ExpectIndex)
145 return wrongValue(Msg: "The root context should not have a callee index");
146 if (RecordValues.size() != 1)
147 return wrongValue(Msg: "The callee index should have exactly one value");
148 CallsiteIndex = RecordValues[0];
149 break;
150 case PGOCtxProfileRecords::TotalRootEntryCount:
151 if (!IsRoot)
152 return wrongValue(Msg: "Non-root has a total entry count record");
153 if (RecordValues.size() != 1)
154 return wrongValue(
155 Msg: "The root total entry count record should have exactly one value");
156 TotalEntryCount = RecordValues[0];
157 break;
158 default:
159 // OK if we see records we do not understand, like records (profile
160 // components) introduced later.
161 break;
162 }
163 }
164
165 PGOCtxProfContext Ret(*Guid, std::move(*Counters), TotalEntryCount,
166 std::move(Unhandled));
167
168 while (canEnterBlockWithID(ID: PGOCtxProfileBlockIDs::ContextNodeBlockID)) {
169 EXPECT_OR_RET(SC, readProfile(PGOCtxProfileBlockIDs::ContextNodeBlockID));
170 auto &Targets = Ret.callsites()[*SC->first];
171 auto [_, Inserted] =
172 Targets.insert(x: {SC->second.guid(), std::move(SC->second)});
173 if (!Inserted)
174 return wrongValue(
175 Msg: "Unexpected duplicate target (callee) at the same callsite.");
176 }
177 return std::make_pair(x&: CallsiteIndex, y: std::move(Ret));
178}
179
180Error PGOCtxProfileReader::readMetadata() {
181 if (Magic.size() < PGOCtxProfileWriter::ContainerMagic.size() ||
182 Magic != PGOCtxProfileWriter::ContainerMagic)
183 return make_error<InstrProfError>(Args: instrprof_error::invalid_prof,
184 Args: "Invalid magic");
185
186 BitstreamEntry Entry;
187 RET_ON_ERR(Cursor.advance().moveInto(Entry));
188 if (Entry.Kind != BitstreamEntry::SubBlock ||
189 Entry.ID != bitc::BLOCKINFO_BLOCK_ID)
190 return unsupported(Msg: "Expected Block ID");
191 // We don't need the blockinfo to read the rest, it's metadata usable for e.g.
192 // llvm-bcanalyzer.
193 RET_ON_ERR(Cursor.SkipBlock());
194
195 EXPECT_OR_RET(Blk, advance());
196 if (Blk->Kind != BitstreamEntry::SubBlock)
197 return unsupported(Msg: "Expected Version record");
198 RET_ON_ERR(
199 Cursor.EnterSubBlock(PGOCtxProfileBlockIDs::ProfileMetadataBlockID));
200 EXPECT_OR_RET(MData, advance());
201 if (MData->Kind != BitstreamEntry::Record)
202 return unsupported(Msg: "Expected Version record");
203
204 SmallVector<uint64_t, 1> Ver;
205 EXPECT_OR_RET(Code, Cursor.readRecord(bitc::UNABBREV_RECORD, Ver));
206 if (*Code != PGOCtxProfileRecords::Version)
207 return unsupported(Msg: "Expected Version record");
208 if (Ver.size() != 1 || Ver[0] > PGOCtxProfileWriter::CurrentVersion)
209 return unsupported(Msg: "Version " + Twine(*Code) +
210 " is higher than supported version " +
211 Twine(PGOCtxProfileWriter::CurrentVersion));
212 return Error::success();
213}
214
215Error PGOCtxProfileReader::loadContexts(CtxProfContextualProfiles &P) {
216 RET_ON_ERR(enterBlockWithID(PGOCtxProfileBlockIDs::ContextsSectionBlockID));
217 while (canEnterBlockWithID(ID: PGOCtxProfileBlockIDs::ContextRootBlockID)) {
218 EXPECT_OR_RET(E, readProfile(PGOCtxProfileBlockIDs::ContextRootBlockID));
219 auto Key = E->second.guid();
220 if (!P.insert(x: {Key, std::move(E->second)}).second)
221 return wrongValue(Msg: "Duplicate roots");
222 }
223 return Error::success();
224}
225
226Error PGOCtxProfileReader::loadFlatProfileList(CtxProfFlatProfile &P) {
227 while (canEnterBlockWithID(ID: PGOCtxProfileBlockIDs::FlatProfileBlockID)) {
228 EXPECT_OR_RET(E, readProfile(PGOCtxProfileBlockIDs::FlatProfileBlockID));
229 auto Guid = E->second.guid();
230 if (!P.insert(x: {Guid, std::move(E->second.counters())}).second)
231 return wrongValue(Msg: "Duplicate flat profile entries");
232 }
233 return Error::success();
234}
235
236Error PGOCtxProfileReader::loadFlatProfiles(CtxProfFlatProfile &P) {
237 RET_ON_ERR(
238 enterBlockWithID(PGOCtxProfileBlockIDs::FlatProfilesSectionBlockID));
239 return loadFlatProfileList(P);
240}
241
242Expected<PGOCtxProfile> PGOCtxProfileReader::loadProfiles() {
243 RET_ON_ERR(readMetadata());
244 PGOCtxProfile Ret;
245 PGOCtxProfileBlockIDs Test = {};
246 for (auto I = 0; I < 2; ++I) {
247 if (!tryGetNextKnownBlockID(ID&: Test))
248 break;
249 if (Test == PGOCtxProfileBlockIDs::ContextsSectionBlockID) {
250 RET_ON_ERR(loadContexts(Ret.Contexts));
251 } else if (Test == PGOCtxProfileBlockIDs::FlatProfilesSectionBlockID) {
252 RET_ON_ERR(loadFlatProfiles(Ret.FlatProfiles));
253 } else {
254 return wrongValue(Msg: "Unexpected section");
255 }
256 }
257
258 return std::move(Ret);
259}
260
261namespace {
262// We want to pass `const` values PGOCtxProfContext references to the yaml
263// converter, and the regular yaml mapping APIs are designed to handle both
264// serialization and deserialization, which prevents using const for
265// serialization. Using an intermediate datastructure is overkill, both
266// space-wise and design complexity-wise. Instead, we use the lower-level APIs.
267void toYaml(yaml::Output &Out, const PGOCtxProfContext &Ctx);
268
269void toYaml(yaml::Output &Out,
270 const PGOCtxProfContext::CallTargetMapTy &CallTargets) {
271 Out.beginSequence();
272 size_t Index = 0;
273 void *SaveData = nullptr;
274 for (const auto &[_, Ctx] : CallTargets) {
275 Out.preflightElement(Index++, SaveData);
276 toYaml(Out, Ctx);
277 Out.postflightElement(nullptr);
278 }
279 Out.endSequence();
280}
281
282void toYaml(yaml::Output &Out,
283 const PGOCtxProfContext::CallsiteMapTy &Callsites) {
284 auto AllCS = ::llvm::make_first_range(c: Callsites);
285 auto MaxIt = ::llvm::max_element(Range&: AllCS);
286 assert(MaxIt != AllCS.end() && "We should have a max value because the "
287 "callsites collection is not empty.");
288 void *SaveData = nullptr;
289 Out.beginSequence();
290 for (auto I = 0U; I <= *MaxIt; ++I) {
291 Out.preflightElement(I, SaveData);
292 auto It = Callsites.find(x: I);
293 if (It == Callsites.end()) {
294 // This will produce a `[ ]` sequence, which is what we want here.
295 Out.beginFlowSequence();
296 Out.endFlowSequence();
297 } else {
298 toYaml(Out, CallTargets: It->second);
299 }
300 Out.postflightElement(nullptr);
301 }
302 Out.endSequence();
303}
304
305void toYaml(yaml::Output &Out, const CtxProfFlatProfile &Flat);
306
307void toYaml(yaml::Output &Out, GlobalValue::GUID Guid,
308 const SmallVectorImpl<uint64_t> &Counters,
309 const PGOCtxProfContext::CallsiteMapTy &Callsites,
310 std::optional<uint64_t> TotalRootEntryCount = std::nullopt,
311 CtxProfFlatProfile Unhandled = {}) {
312 yaml::EmptyContext Empty;
313 Out.beginMapping();
314 void *SaveInfo = nullptr;
315 bool UseDefault = false;
316 {
317 Out.preflightKey(key: "Guid", /*Required=*/true, /*SameAsDefault=*/false,
318 UseDefault, SaveInfo);
319 yaml::yamlize(io&: Out, Val&: Guid, true, Ctx&: Empty);
320 Out.postflightKey(nullptr);
321 }
322 if (TotalRootEntryCount) {
323 Out.preflightKey(key: "TotalRootEntryCount", true, false, UseDefault, SaveInfo);
324 yaml::yamlize(io&: Out, Val&: *TotalRootEntryCount, true, Ctx&: Empty);
325 Out.postflightKey(nullptr);
326 }
327 {
328 Out.preflightKey(key: "Counters", true, false, UseDefault, SaveInfo);
329 Out.beginFlowSequence();
330 for (size_t I = 0U, E = Counters.size(); I < E; ++I) {
331 Out.preflightFlowElement(I, SaveInfo);
332 uint64_t V = Counters[I];
333 yaml::yamlize(io&: Out, Val&: V, true, Ctx&: Empty);
334 Out.postflightFlowElement(SaveInfo);
335 }
336 Out.endFlowSequence();
337 Out.postflightKey(nullptr);
338 }
339
340 if (!Unhandled.empty()) {
341 assert(TotalRootEntryCount.has_value());
342 Out.preflightKey(key: "Unhandled", false, false, UseDefault, SaveInfo);
343 toYaml(Out, Flat: Unhandled);
344 Out.postflightKey(nullptr);
345 }
346
347 if (!Callsites.empty()) {
348 Out.preflightKey(key: "Callsites", true, false, UseDefault, SaveInfo);
349 toYaml(Out, Callsites);
350 Out.postflightKey(nullptr);
351 }
352 Out.endMapping();
353}
354
355void toYaml(yaml::Output &Out, const CtxProfFlatProfile &Flat) {
356 void *SaveInfo = nullptr;
357 Out.beginSequence();
358 size_t ElemID = 0;
359 for (const auto &[Guid, Counters] : Flat) {
360 Out.preflightElement(ElemID++, SaveInfo);
361 toYaml(Out, Guid, Counters, Callsites: {});
362 Out.postflightElement(nullptr);
363 }
364 Out.endSequence();
365}
366
367void toYaml(yaml::Output &Out, const PGOCtxProfContext &Ctx) {
368 if (Ctx.isRoot())
369 toYaml(Out, Guid: Ctx.guid(), Counters: Ctx.counters(), Callsites: Ctx.callsites(),
370 TotalRootEntryCount: Ctx.getTotalRootEntryCount(), Unhandled: Ctx.getUnhandled());
371 else
372 toYaml(Out, Guid: Ctx.guid(), Counters: Ctx.counters(), Callsites: Ctx.callsites());
373}
374
375} // namespace
376
377void llvm::convertCtxProfToYaml(raw_ostream &OS, const PGOCtxProfile &Profile) {
378 yaml::Output Out(OS);
379 void *SaveInfo = nullptr;
380 bool UseDefault = false;
381 Out.beginMapping();
382 if (!Profile.Contexts.empty()) {
383 Out.preflightKey(key: "Contexts", false, false, UseDefault, SaveInfo);
384 toYaml(Out, CallTargets: Profile.Contexts);
385 Out.postflightKey(nullptr);
386 }
387 if (!Profile.FlatProfiles.empty()) {
388 Out.preflightKey(key: "FlatProfiles", false, false, UseDefault, SaveInfo);
389 toYaml(Out, Flat: Profile.FlatProfiles);
390 Out.postflightKey(nullptr);
391 }
392 Out.endMapping();
393}
394