1//===-- InstrProfCorrelator.cpp -------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ProfileData/InstrProfCorrelator.h"
10#include "llvm/DebugInfo/DIContext.h"
11#include "llvm/DebugInfo/DWARF/DWARFContext.h"
12#include "llvm/DebugInfo/DWARF/DWARFDie.h"
13#include "llvm/DebugInfo/DWARF/DWARFFormValue.h"
14#include "llvm/DebugInfo/DWARF/DWARFLocationExpression.h"
15#include "llvm/DebugInfo/DWARF/DWARFUnit.h"
16#include "llvm/DebugInfo/DWARF/LowLevel/DWARFExpression.h"
17#include "llvm/Object/MachO.h"
18#include "llvm/Support/Debug.h"
19#include "llvm/Support/Format.h"
20#include "llvm/Support/WithColor.h"
21#include <optional>
22
23#define DEBUG_TYPE "correlator"
24
25using namespace llvm;
26
27/// Get profile section.
28static Expected<object::SectionRef>
29getInstrProfSection(const object::ObjectFile &Obj, InstrProfSectKind IPSK) {
30 // On COFF, the getInstrProfSectionName returns the section names may followed
31 // by "$M". The linker removes the dollar and everything after it in the final
32 // binary. Do the same to match.
33 Triple::ObjectFormatType ObjFormat = Obj.getTripleObjectFormat();
34 auto StripSuffix = [ObjFormat](StringRef N) {
35 return ObjFormat == Triple::COFF ? N.split(Separator: '$').first : N;
36 };
37 std::string ExpectedSectionName =
38 getInstrProfSectionName(IPSK, OF: ObjFormat,
39 /*AddSegmentInfo=*/false);
40 ExpectedSectionName = StripSuffix(ExpectedSectionName);
41 for (auto &Section : Obj.sections()) {
42 if (auto SectionName = Section.getName())
43 if (*SectionName == ExpectedSectionName)
44 return Section;
45 }
46 return make_error<InstrProfError>(
47 Args: instrprof_error::unable_to_correlate_profile,
48 Args: "could not find section (" + Twine(ExpectedSectionName) + ")");
49}
50
51const char *InstrProfCorrelator::FunctionNameAttributeName = "Function Name";
52const char *InstrProfCorrelator::CFGHashAttributeName = "CFG Hash";
53const char *InstrProfCorrelator::NumCountersAttributeName = "Num Counters";
54const char *InstrProfCorrelator::NumBitmapBitsAttributeName = "Num BitmapBits";
55
56llvm::Expected<std::unique_ptr<InstrProfCorrelator::Context>>
57InstrProfCorrelator::Context::get(std::unique_ptr<MemoryBuffer> Buffer,
58 object::ObjectFile &Obj,
59 ProfCorrelatorKind FileKind) {
60 auto C = std::make_unique<Context>();
61 auto CountersSection = getInstrProfSection(Obj, IPSK: IPSK_cnts);
62 if (auto Err = CountersSection.takeError())
63 return std::move(Err);
64 Triple::ObjectFormatType ObjFormat = Obj.getTripleObjectFormat();
65 if (FileKind == InstrProfCorrelator::BINARY) {
66 auto DataSection = getInstrProfSection(Obj, IPSK: IPSK_covdata);
67 if (auto Err = DataSection.takeError())
68 return std::move(Err);
69 auto DataOrErr = DataSection->getContents();
70 if (!DataOrErr)
71 return DataOrErr.takeError();
72 auto NameSection = getInstrProfSection(Obj, IPSK: IPSK_covname);
73 if (auto Err = NameSection.takeError())
74 return std::move(Err);
75 auto NameOrErr = NameSection->getContents();
76 if (!NameOrErr)
77 return NameOrErr.takeError();
78 C->DataStart = DataOrErr->data();
79 C->DataEnd = DataOrErr->data() + DataOrErr->size();
80 C->NameStart = NameOrErr->data();
81 C->NameSize = NameOrErr->size();
82
83 if (ObjFormat == Triple::MachO) {
84 std::string FullSectionName =
85 getInstrProfSectionName(IPSK: IPSK_covdata, OF: ObjFormat);
86 SmallVector<StringRef, 3> SegmentAndSection;
87 StringRef(FullSectionName).split(A&: SegmentAndSection, Separator: ',', MaxSplit: 2);
88 auto *MachO = static_cast<object::MachOObjectFile *>(&Obj);
89 Error Err = Error::success();
90 for (const object::MachOChainedFixupEntry &Entry :
91 MachO->fixupTable(Err)) {
92 if (Entry.isRebase() && Entry.segmentName() == SegmentAndSection[0] &&
93 Entry.sectionName() == SegmentAndSection[1]) {
94 C->MachOFixups[Entry.address() - DataSection->getAddress()] =
95 Entry.pointerValue();
96 }
97 }
98 if (Err)
99 return std::move(Err);
100 }
101 }
102 C->Buffer = std::move(Buffer);
103 C->CountersSectionStart = CountersSection->getAddress();
104 C->CountersSectionEnd = C->CountersSectionStart + CountersSection->getSize();
105
106 auto BitmapSection = getInstrProfSection(Obj, IPSK: IPSK_bitmap);
107 if (auto E = BitmapSection.takeError()) {
108 // It is not an error if NumBitmapBytes of each function is zero.
109 consumeError(Err: std::move(E));
110 C->BitmapSectionStart = 0;
111 C->BitmapSectionEnd = 0;
112 } else {
113 C->BitmapSectionStart = BitmapSection->getAddress();
114 C->BitmapSectionEnd = C->BitmapSectionStart + BitmapSection->getSize();
115 }
116 // In COFF object file, there's a null byte at the beginning of both the
117 // counter and bitmap sections which doesn't exist in raw profile.
118 if (ObjFormat == Triple::COFF) {
119 ++C->CountersSectionStart;
120 if (C->BitmapSectionStart)
121 ++C->BitmapSectionStart;
122 }
123
124 C->ShouldSwapBytes = Obj.isLittleEndian() != sys::IsLittleEndianHost;
125 return Expected<std::unique_ptr<Context>>(std::move(C));
126}
127
128llvm::Expected<std::unique_ptr<InstrProfCorrelator>>
129InstrProfCorrelator::get(StringRef Filename, ProfCorrelatorKind FileKind,
130 const object::BuildIDFetcher *BIDFetcher,
131 const ArrayRef<object::BuildID> BIs) {
132 // Might be overwritten from BuildIDFetcher.
133 std::string EffectiveFilename = Filename.str();
134 if (BIDFetcher) {
135 if (BIs.empty())
136 return make_error<InstrProfError>(
137 Args: instrprof_error::unable_to_correlate_profile,
138 Args: "unsupported profile binary correlation when there is no build ID "
139 "in a profile");
140 if (BIs.size() > 1)
141 return make_error<InstrProfError>(
142 Args: instrprof_error::unable_to_correlate_profile,
143 Args: "unsupported profile binary correlation when there are multiple "
144 "build IDs in a profile");
145
146 Expected<std::string> Path = BIDFetcher->fetch(BuildID: BIs.front());
147 if (!Path) {
148 // Propagate as InstrProf specific error type.
149 consumeError(Err: Path.takeError());
150 return make_error<InstrProfError>(
151 Args: instrprof_error::unable_to_correlate_profile,
152 Args: "Missing build ID: " + llvm::toHex(Input: BIs.front(),
153 /*LowerCase=*/true));
154 }
155 EffectiveFilename = *Path;
156 }
157
158 if (FileKind == DEBUG_INFO) {
159 auto DsymObjectsOrErr =
160 object::MachOObjectFile::findDsymObjectMembers(Path: EffectiveFilename);
161 if (auto Err = DsymObjectsOrErr.takeError())
162 return std::move(Err);
163 if (!DsymObjectsOrErr->empty()) {
164 // TODO: Enable profile correlation when there are multiple objects in a
165 // dSYM bundle.
166 if (DsymObjectsOrErr->size() > 1)
167 return make_error<InstrProfError>(
168 Args: instrprof_error::unable_to_correlate_profile,
169 Args: "using multiple objects is not yet supported");
170 EffectiveFilename = *DsymObjectsOrErr->begin();
171 }
172 auto BufferOrErr =
173 errorOrToExpected(EO: MemoryBuffer::getFile(Filename: EffectiveFilename));
174 if (auto Err = BufferOrErr.takeError())
175 return std::move(Err);
176
177 return get(Buffer: std::move(*BufferOrErr), FileKind);
178 }
179 if (FileKind == BINARY) {
180 auto BufferOrErr =
181 errorOrToExpected(EO: MemoryBuffer::getFile(Filename: EffectiveFilename));
182 if (auto Err = BufferOrErr.takeError())
183 return std::move(Err);
184
185 return get(Buffer: std::move(*BufferOrErr), FileKind);
186 }
187 return make_error<InstrProfError>(
188 Args: instrprof_error::unable_to_correlate_profile,
189 Args: "unsupported correlation kind (only DWARF debug info and Binary format "
190 "(ELF/COFF) are supported)");
191}
192
193llvm::Expected<std::unique_ptr<InstrProfCorrelator>>
194InstrProfCorrelator::get(std::unique_ptr<MemoryBuffer> Buffer,
195 ProfCorrelatorKind FileKind) {
196 auto BinOrErr = object::createBinary(Source: *Buffer);
197 if (auto Err = BinOrErr.takeError())
198 return std::move(Err);
199
200 if (auto *Obj = dyn_cast<object::ObjectFile>(Val: BinOrErr->get())) {
201 auto CtxOrErr = Context::get(Buffer: std::move(Buffer), Obj&: *Obj, FileKind);
202 if (auto Err = CtxOrErr.takeError())
203 return std::move(Err);
204 auto T = Obj->makeTriple();
205 if (T.isArch64Bit())
206 return InstrProfCorrelatorImpl<uint64_t>::get(Ctx: std::move(*CtxOrErr), Obj: *Obj,
207 FileKind);
208 if (T.isArch32Bit())
209 return InstrProfCorrelatorImpl<uint32_t>::get(Ctx: std::move(*CtxOrErr), Obj: *Obj,
210 FileKind);
211 }
212 return make_error<InstrProfError>(
213 Args: instrprof_error::unable_to_correlate_profile, Args: "not an object file");
214}
215
216std::optional<size_t> InstrProfCorrelator::getDataSize() const {
217 if (auto *C = dyn_cast<InstrProfCorrelatorImpl<uint32_t>>(Val: this))
218 return C->getDataSize();
219 if (auto *C = dyn_cast<InstrProfCorrelatorImpl<uint64_t>>(Val: this))
220 return C->getDataSize();
221 return {};
222}
223
224namespace llvm {
225
226template <>
227InstrProfCorrelatorImpl<uint32_t>::InstrProfCorrelatorImpl(
228 std::unique_ptr<InstrProfCorrelator::Context> Ctx)
229 : InstrProfCorrelatorImpl(InstrProfCorrelatorKind::CK_32Bit,
230 std::move(Ctx)) {}
231template <>
232InstrProfCorrelatorImpl<uint64_t>::InstrProfCorrelatorImpl(
233 std::unique_ptr<InstrProfCorrelator::Context> Ctx)
234 : InstrProfCorrelatorImpl(InstrProfCorrelatorKind::CK_64Bit,
235 std::move(Ctx)) {}
236template <>
237bool InstrProfCorrelatorImpl<uint32_t>::classof(const InstrProfCorrelator *C) {
238 return C->getKind() == InstrProfCorrelatorKind::CK_32Bit;
239}
240template <>
241bool InstrProfCorrelatorImpl<uint64_t>::classof(const InstrProfCorrelator *C) {
242 return C->getKind() == InstrProfCorrelatorKind::CK_64Bit;
243}
244
245} // end namespace llvm
246
247template <class IntPtrT>
248llvm::Expected<std::unique_ptr<InstrProfCorrelatorImpl<IntPtrT>>>
249InstrProfCorrelatorImpl<IntPtrT>::get(
250 std::unique_ptr<InstrProfCorrelator::Context> Ctx,
251 const object::ObjectFile &Obj, ProfCorrelatorKind FileKind) {
252 if (FileKind == DEBUG_INFO) {
253 if (Obj.isELF() || Obj.isMachO()) {
254 auto DICtx = DWARFContext::create(Obj);
255 return std::make_unique<DwarfInstrProfCorrelator<IntPtrT>>(
256 std::move(DICtx), std::move(Ctx));
257 }
258 return make_error<InstrProfError>(
259 Args: instrprof_error::unable_to_correlate_profile,
260 Args: "unsupported debug info format (only DWARF is supported)");
261 }
262 if (Obj.isELF() || Obj.isCOFF() || Obj.isMachO())
263 return std::make_unique<BinaryInstrProfCorrelator<IntPtrT>>(std::move(Ctx));
264 return make_error<InstrProfError>(
265 Args: instrprof_error::unable_to_correlate_profile,
266 Args: "unsupported binary format (only ELF, COFF, and Mach-O are supported)");
267}
268
269template <class IntPtrT>
270Error InstrProfCorrelatorImpl<IntPtrT>::correlateProfileData(int MaxWarnings) {
271 assert(Data.empty() && Names.empty() && NamesVec.empty());
272 correlateProfileDataImpl(MaxWarnings);
273 if (this->Data.empty())
274 return make_error<InstrProfError>(
275 Args: instrprof_error::unable_to_correlate_profile,
276 Args: "could not find any profile data metadata in correlated file");
277 Error Result = correlateProfileNameImpl();
278 this->CounterOffsets.clear();
279 this->BitmapOffsets.clear();
280 this->NamesVec.clear();
281 return Result;
282}
283
284template <> struct yaml::MappingTraits<InstrProfCorrelator::CorrelationData> {
285 static void mapping(yaml::IO &io,
286 InstrProfCorrelator::CorrelationData &Data) {
287 io.mapRequired(Key: "Probes", Val&: Data.Probes);
288 }
289};
290
291template <> struct yaml::MappingTraits<InstrProfCorrelator::Probe> {
292 static void mapping(yaml::IO &io, InstrProfCorrelator::Probe &P) {
293 io.mapRequired(Key: "Function Name", Val&: P.FunctionName);
294 io.mapOptional(Key: "Linkage Name", Val&: P.LinkageName);
295 io.mapRequired(Key: "CFG Hash", Val&: P.CFGHash);
296 io.mapRequired(Key: "Counter Offset", Val&: P.CounterOffset);
297 io.mapRequired(Key: "Num Counters", Val&: P.NumCounters);
298 io.mapRequired(Key: "Bitmap Offset", Val&: P.BitmapOffset);
299 io.mapRequired(Key: "Num BitmapBytes", Val&: P.NumBitmapBytes);
300 io.mapOptional(Key: "File", Val&: P.FilePath);
301 io.mapOptional(Key: "Line", Val&: P.LineNumber);
302 }
303};
304
305template <> struct yaml::SequenceElementTraits<InstrProfCorrelator::Probe> {
306 static const bool flow = false;
307};
308
309template <class IntPtrT>
310Error InstrProfCorrelatorImpl<IntPtrT>::dumpYaml(int MaxWarnings,
311 raw_ostream &OS) {
312 InstrProfCorrelator::CorrelationData Data;
313 correlateProfileDataImpl(MaxWarnings, Data: &Data);
314 if (Data.Probes.empty())
315 return make_error<InstrProfError>(
316 Args: instrprof_error::unable_to_correlate_profile,
317 Args: "could not find any profile data metadata in debug info");
318 yaml::Output YamlOS(OS);
319 YamlOS << Data;
320 return Error::success();
321}
322
323template <class IntPtrT>
324void InstrProfCorrelatorImpl<IntPtrT>::addDataProbe(
325 uint64_t NameRef, uint64_t CFGHash, IntPtrT CounterOffset,
326 IntPtrT BitmapOffset, IntPtrT FunctionPtr, uint32_t NumCounters,
327 uint32_t NumBitmapBytes) {
328 // Check if a probe was already added for this counter offset.
329 if (NumCounters && !CounterOffsets.insert(CounterOffset).second)
330 return;
331 // Check if a probe was already added for this bitmap offset.
332 if (NumBitmapBytes && !BitmapOffsets.insert(BitmapOffset).second)
333 return;
334 Data.push_back({
335 maybeSwap<uint64_t>(NameRef),
336 maybeSwap<uint64_t>(CFGHash),
337 // In this mode, CounterPtr actually stores the section relative address
338 // of the counter.
339 maybeSwap<IntPtrT>(CounterOffset),
340 /*UniformCounterPtr=*/maybeSwap<IntPtrT>(0),
341 maybeSwap<IntPtrT>(BitmapOffset),
342 maybeSwap<IntPtrT>(FunctionPtr),
343 // TODO: Value profiling is not yet supported.
344 /*ValuesPtr=*/maybeSwap<IntPtrT>(0),
345 maybeSwap<uint32_t>(NumCounters),
346 /*NumValueSites=*/{maybeSwap<uint16_t>(0), maybeSwap<uint16_t>(0)},
347 /*OffloadDeviceWaveSize=*/maybeSwap<uint16_t>(0),
348 maybeSwap<uint32_t>(NumBitmapBytes),
349 });
350}
351
352template <class IntPtrT>
353std::optional<uint64_t>
354DwarfInstrProfCorrelator<IntPtrT>::getLocation(const DWARFDie &Die) const {
355 auto Locations = Die.getLocations(Attr: dwarf::DW_AT_location);
356 if (!Locations) {
357 consumeError(Err: Locations.takeError());
358 return {};
359 }
360 auto &DU = *Die.getDwarfUnit();
361 auto AddressSize = DU.getAddressByteSize();
362 for (auto &Location : *Locations) {
363 DataExtractor Data(Location.Expr, DICtx->isLittleEndian());
364 DWARFExpression Expr(Data, AddressSize);
365 for (auto &Op : Expr) {
366 if (Op.getCode() == dwarf::DW_OP_addr)
367 return Op.getRawOperand(Idx: 0);
368 if (Op.getCode() == dwarf::DW_OP_addrx) {
369 uint64_t Index = Op.getRawOperand(Idx: 0);
370 if (auto SA = DU.getAddrOffsetSectionItem(Index))
371 return SA->Address;
372 }
373 }
374 }
375 return {};
376}
377
378template <class IntPtrT>
379bool DwarfInstrProfCorrelator<IntPtrT>::isDIEOfProbe(const DWARFDie &Die,
380 StringRef Prefix) {
381 const auto &ParentDie = Die.getParent();
382 if (!Die.isValid() || !ParentDie.isValid() || Die.isNULL())
383 return false;
384 if (Die.getTag() != dwarf::DW_TAG_variable)
385 return false;
386 if (!ParentDie.isSubprogramDIE())
387 return false;
388 if (!Die.hasChildren())
389 return false;
390 if (const char *Name = Die.getName(Kind: DINameKind::ShortName))
391 return StringRef(Name).starts_with(Prefix);
392 return false;
393}
394
395template <class IntPtrT>
396std::optional<std::pair<InstrProfCorrelator::Probe, IntPtrT>>
397DwarfInstrProfCorrelator<IntPtrT>::addCountersToDataProbe(
398 const DWARFDie &Die, const bool UnlimitedWarnings,
399 int &NumSuppressedWarnings) {
400 std::optional<const char *> FunctionName;
401 std::optional<uint64_t> CFGHash;
402 std::optional<uint64_t> CounterPtr = getLocation(Die);
403 auto FnDie = Die.getParent();
404 auto FunctionPtr = dwarf::toAddress(V: FnDie.find(Attr: dwarf::DW_AT_low_pc));
405 std::optional<uint64_t> NumCounters;
406 for (const DWARFDie &Child : Die.children()) {
407 if (Child.getTag() != dwarf::DW_TAG_LLVM_annotation)
408 continue;
409 auto AnnotationFormName = Child.find(Attr: dwarf::DW_AT_name);
410 auto AnnotationFormValue = Child.find(Attr: dwarf::DW_AT_const_value);
411 if (!AnnotationFormName || !AnnotationFormValue)
412 continue;
413 auto AnnotationNameOrErr = AnnotationFormName->getAsCString();
414 if (auto Err = AnnotationNameOrErr.takeError()) {
415 consumeError(Err: std::move(Err));
416 continue;
417 }
418 StringRef AnnotationName = *AnnotationNameOrErr;
419 if (AnnotationName == InstrProfCorrelator::FunctionNameAttributeName) {
420 if (auto EC = AnnotationFormValue->getAsCString().moveInto(Value&: FunctionName))
421 consumeError(Err: std::move(EC));
422 } else if (AnnotationName == InstrProfCorrelator::CFGHashAttributeName) {
423 CFGHash = AnnotationFormValue->getAsUnsignedConstant();
424 } else if (AnnotationName ==
425 InstrProfCorrelator::NumCountersAttributeName) {
426 NumCounters = AnnotationFormValue->getAsUnsignedConstant();
427 }
428 }
429 // If there is no function and no counter, assume it was dead-stripped
430 if (!FunctionPtr && !CounterPtr)
431 return std::nullopt;
432 if (!FunctionName || !CFGHash || !CounterPtr || !NumCounters) {
433 if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) {
434 WithColor::warning() << "Incomplete DIE for function " << FunctionName
435 << ": CFGHash=" << CFGHash
436 << " CounterPtr=" << CounterPtr
437 << " NumCounters=" << NumCounters << "\n";
438 LLVM_DEBUG(Die.dump(dbgs()));
439 }
440 return std::nullopt;
441 }
442 uint64_t CountersStart = this->Ctx->CountersSectionStart;
443 uint64_t CountersEnd = this->Ctx->CountersSectionEnd;
444 if (*CounterPtr < CountersStart || *CounterPtr >= CountersEnd) {
445 if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) {
446 WithColor::warning() << format(
447 Fmt: "CounterPtr out of range for function %s: Actual=0x%x "
448 "Expected=[0x%x, 0x%x)\n",
449 Vals: *FunctionName, Vals: *CounterPtr, Vals: CountersStart, Vals: CountersEnd);
450 LLVM_DEBUG(Die.dump(dbgs()));
451 }
452 return std::nullopt;
453 }
454 if (!FunctionPtr && (UnlimitedWarnings || ++NumSuppressedWarnings < 1)) {
455 WithColor::warning() << format(Fmt: "Could not find address of function %s\n",
456 Vals: *FunctionName);
457 LLVM_DEBUG(Die.dump(dbgs()));
458 }
459 // In debug info correlation mode, the CounterPtr is an absolute address
460 // of the counter, but it's expected to be relative later when iterating
461 // Data.
462 IntPtrT CounterOffset = *CounterPtr - CountersStart;
463 InstrProfCorrelator::Probe P = {};
464 P.FunctionName = *FunctionName;
465 if (const char *Name = FnDie.getName(Kind: DINameKind::LinkageName))
466 P.LinkageName = Name;
467 P.CFGHash = *CFGHash;
468 P.CounterOffset = CounterOffset;
469 P.NumCounters = *NumCounters;
470 auto FilePath = FnDie.getDeclFile(
471 Kind: DILineInfoSpecifier::FileLineInfoKind::RelativeFilePath);
472 if (!FilePath.empty())
473 P.FilePath = FilePath;
474 if (auto LineNumber = FnDie.getDeclLine())
475 P.LineNumber = LineNumber;
476
477 return std::optional<std::pair<InstrProfCorrelator::Probe, IntPtrT>>(
478 {P, FunctionPtr.value_or(u: 0)});
479}
480
481template <class IntPtrT>
482std::optional<std::pair<InstrProfCorrelator::Probe, IntPtrT>>
483DwarfInstrProfCorrelator<IntPtrT>::addBitmapToDataProbe(
484 const DWARFDie &Die, const bool UnlimitedWarnings,
485 int &NumSuppressedWarnings) {
486 std::optional<const char *> FunctionName;
487 std::optional<uint64_t> BitmapPtr = getLocation(Die);
488 uint64_t NumBitmapBytes;
489 for (const DWARFDie &Child : Die.children()) {
490 if (Child.getTag() != dwarf::DW_TAG_LLVM_annotation)
491 continue;
492 auto AnnotationFormName = Child.find(Attr: dwarf::DW_AT_name);
493 auto AnnotationFormValue = Child.find(Attr: dwarf::DW_AT_const_value);
494 if (!AnnotationFormName || !AnnotationFormValue)
495 continue;
496 auto AnnotationNameOrErr = AnnotationFormName->getAsCString();
497 if (auto Err = AnnotationNameOrErr.takeError()) {
498 consumeError(Err: std::move(Err));
499 continue;
500 }
501 StringRef AnnotationName = *AnnotationNameOrErr;
502 if (AnnotationName == InstrProfCorrelator::FunctionNameAttributeName) {
503 if (auto EC = AnnotationFormValue->getAsCString().moveInto(Value&: FunctionName))
504 consumeError(Err: std::move(EC));
505 } else if (AnnotationName ==
506 InstrProfCorrelator::NumBitmapBitsAttributeName) {
507 std::optional<uint64_t> NumBitmapBits =
508 AnnotationFormValue->getAsUnsignedConstant();
509 NumBitmapBytes = alignTo(Value: *NumBitmapBits, CHAR_BIT) / CHAR_BIT;
510 }
511 }
512 if (!FunctionName || !BitmapPtr || !NumBitmapBytes) {
513 if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) {
514 WithColor::warning() << "Incomplete DIE for function " << FunctionName
515 << " BitmapPtr=" << BitmapPtr
516 << " NumBitmapBytes=" << NumBitmapBytes << "\n";
517 LLVM_DEBUG(Die.dump(dbgs()));
518 }
519 return std::nullopt;
520 }
521 uint64_t BitmapStart = this->Ctx->BitmapSectionStart;
522 uint64_t BitmapEnd = this->Ctx->BitmapSectionEnd;
523 if (!BitmapStart && !BitmapEnd && NumBitmapBytes) {
524 auto E = make_error<InstrProfError>(
525 Args: instrprof_error::unable_to_correlate_profile,
526 Args: "could not find profile bitmap section in correlated file");
527 return std::nullopt;
528 }
529 if (*BitmapPtr < BitmapStart || (*BitmapPtr >= BitmapEnd && NumBitmapBytes)) {
530 if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) {
531 WithColor::warning() << format(
532 Fmt: "BitmapPtr out of range for function %s: Actual=0x%x "
533 "Expected=[0x%x, 0x%x)\n",
534 Vals: *FunctionName, Vals: *BitmapPtr, Vals: BitmapStart, Vals: BitmapEnd);
535 LLVM_DEBUG(Die.dump(dbgs()));
536 }
537 return std::nullopt;
538 }
539 // In debug info correlation mode, the BitmapPtr is an absolute address of
540 // the bitmap, but it's expected to be relative later when iterating Data.
541 IntPtrT BitmapOffset = *BitmapPtr - BitmapStart;
542 InstrProfCorrelator::Probe P = {};
543 P.FunctionName = *FunctionName;
544 P.BitmapOffset = BitmapOffset;
545 P.NumBitmapBytes = NumBitmapBytes;
546
547 return std::optional<std::pair<InstrProfCorrelator::Probe, IntPtrT>>({P, 0});
548}
549
550template <class IntPtrT>
551void DwarfInstrProfCorrelator<IntPtrT>::correlateProfileDataImpl(
552 int MaxWarnings, InstrProfCorrelator::CorrelationData *Data) {
553 // Map from FunctionName string to (Probe, FunctionPtr) pair.
554 // We use it to collect data from all functions Counter and Bitmap DIEs.
555 MapVector<std::string, std::pair<InstrProfCorrelator::Probe, IntPtrT>,
556 StringMap<unsigned>>
557 Probes;
558 bool UnlimitedWarnings = (MaxWarnings == 0);
559 // -N suppressed warnings means we can emit up to N (unsuppressed) warnings
560 int NumSuppressedWarnings = -MaxWarnings;
561
562 auto MaybeAddProbe = [&](DWARFDie Die) {
563 std::optional<std::pair<InstrProfCorrelator::Probe, IntPtrT>> ProbeData;
564 if (isDIEOfProbe(Die, Prefix: getInstrProfCountersVarPrefix()))
565 ProbeData =
566 addCountersToDataProbe(Die, UnlimitedWarnings, NumSuppressedWarnings);
567 else if (isDIEOfProbe(Die, Prefix: getInstrProfBitmapVarPrefix()))
568 ProbeData =
569 addBitmapToDataProbe(Die, UnlimitedWarnings, NumSuppressedWarnings);
570 if (!ProbeData)
571 return;
572 auto [Probe, FunctionPtr] = *ProbeData;
573
574 auto [It, Inserted] =
575 Probes.try_emplace(Probe.FunctionName, Probe, FunctionPtr);
576 if (!Inserted) {
577 auto &P = It->second.first;
578 if (isDIEOfProbe(Die, Prefix: getInstrProfCountersVarPrefix())) {
579 P.LinkageName = Probe.LinkageName;
580 P.CFGHash = Probe.CFGHash;
581 P.CounterOffset = Probe.CounterOffset;
582 P.NumCounters = Probe.NumCounters;
583 P.FilePath = Probe.FilePath;
584 P.LineNumber = Probe.LineNumber;
585 } else {
586 P.BitmapOffset = Probe.BitmapOffset;
587 P.NumBitmapBytes = Probe.NumBitmapBytes;
588 }
589 }
590 };
591 for (auto &CU : DICtx->normal_units())
592 for (const auto &Entry : CU->dies())
593 MaybeAddProbe(DWARFDie(CU.get(), &Entry));
594 for (auto &CU : DICtx->dwo_units())
595 for (const auto &Entry : CU->dies())
596 MaybeAddProbe(DWARFDie(CU.get(), &Entry));
597
598 for (const auto &[FunctionName, ProbeData] : Probes) {
599 const auto &[Probe, FunctionPtr] = ProbeData;
600 if (Data)
601 Data->Probes.push_back(Probe);
602 else {
603 this->NamesVec.push_back(FunctionName);
604 uint64_t NameRef = IndexedInstrProf::ComputeHash(FunctionName);
605 this->addDataProbe(NameRef, Probe.CFGHash, Probe.CounterOffset,
606 Probe.BitmapOffset, FunctionPtr, Probe.NumCounters,
607 Probe.NumBitmapBytes);
608 }
609 }
610 if (!UnlimitedWarnings && NumSuppressedWarnings > 0)
611 WithColor::warning() << format(Fmt: "Suppressed %d additional warnings\n",
612 Vals: NumSuppressedWarnings);
613}
614
615template <class IntPtrT>
616Error DwarfInstrProfCorrelator<IntPtrT>::correlateProfileNameImpl() {
617 if (this->NamesVec.empty()) {
618 return make_error<InstrProfError>(
619 Args: instrprof_error::unable_to_correlate_profile,
620 Args: "could not find any profile name metadata in debug info");
621 }
622 auto Result =
623 collectGlobalObjectNameStrings(this->NamesVec,
624 /*doCompression=*/false, this->Names);
625 return Result;
626}
627
628template <class IntPtrT>
629void BinaryInstrProfCorrelator<IntPtrT>::correlateProfileDataImpl(
630 int MaxWarnings, InstrProfCorrelator::CorrelationData *CorrelateData) {
631 using RawProfData = RawInstrProf::ProfileData<IntPtrT>;
632 bool UnlimitedWarnings = (MaxWarnings == 0);
633 // -N suppressed warnings means we can emit up to N (unsuppressed) warnings
634 int NumSuppressedWarnings = -MaxWarnings;
635
636 const RawProfData *DataStart = (const RawProfData *)this->Ctx->DataStart;
637 const RawProfData *DataEnd = (const RawProfData *)this->Ctx->DataEnd;
638 // We need to use < here because the last data record may have no padding.
639 for (const RawProfData *I = DataStart; I < DataEnd; ++I) {
640 uint64_t CounterPtr = this->template maybeSwap<IntPtrT>(I->CounterPtr);
641 uint64_t CountersStart = this->Ctx->CountersSectionStart;
642 uint64_t CountersEnd = this->Ctx->CountersSectionEnd;
643
644 uint64_t BitmapPtr = this->template maybeSwap<IntPtrT>(I->BitmapPtr);
645 uint64_t BitmapStart = this->Ctx->BitmapSectionStart;
646 uint64_t BitmapEnd = this->Ctx->BitmapSectionEnd;
647 if (!BitmapStart && !BitmapEnd && I->NumBitmapBytes) {
648 auto E = make_error<InstrProfError>(
649 Args: instrprof_error::unable_to_correlate_profile,
650 Args: "could not find profile bitmap section in correlated file");
651 return;
652 }
653 if (!this->Ctx->MachOFixups.empty()) {
654 auto GetPtrByOffset = [&](uint64_t Offset, uint64_t &Ptr) {
655 auto It = this->Ctx->MachOFixups.find(Offset);
656 if (It != this->Ctx->MachOFixups.end()) {
657 Ptr = It->second;
658 } else if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) {
659 WithColor::warning() << format(
660 Fmt: "Mach-O fixup not found for covdata offset 0x%llx\n", Vals: Offset);
661 }
662 };
663 uint64_t CounterOffset = (uint64_t)&I->CounterPtr - (uint64_t)DataStart;
664 uint64_t BitmapOffset = (uint64_t)&I->BitmapPtr - (uint64_t)DataStart;
665 GetPtrByOffset(CounterOffset, CounterPtr);
666 GetPtrByOffset(BitmapOffset, BitmapPtr);
667 }
668 if (CounterPtr < CountersStart || CounterPtr >= CountersEnd) {
669 if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) {
670 WithColor::warning()
671 << format("CounterPtr out of range for function: Actual=0x%x "
672 "Expected=[0x%x, 0x%x) at data offset=0x%x\n",
673 CounterPtr, CountersStart, CountersEnd,
674 (I - DataStart) * sizeof(RawProfData));
675 }
676 }
677 if (I->NumBitmapBytes &&
678 (BitmapPtr < BitmapStart || BitmapPtr >= BitmapEnd)) {
679 if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) {
680 WithColor::warning()
681 << format("BitmapPtr out of range for function: Actual=0x%x "
682 "Expected=[0x%x, 0x%x) at data offset=0x%x\n",
683 BitmapPtr, BitmapStart, BitmapEnd,
684 (I - DataStart) * sizeof(RawProfData));
685 }
686 }
687 // In binary correlation mode, CounterPtr and BitmapPtr are absolute
688 // addresses, but they're expected to be relative later when iterating Data.
689 IntPtrT CounterOffset = CounterPtr - CountersStart;
690 IntPtrT BitmapOffset = BitmapPtr - BitmapStart;
691 this->addDataProbe(I->NameRef, I->FuncHash, CounterOffset, BitmapOffset,
692 I->FunctionPointer, I->NumCounters, I->NumBitmapBytes);
693 }
694}
695
696template <class IntPtrT>
697Error BinaryInstrProfCorrelator<IntPtrT>::correlateProfileNameImpl() {
698 if (this->Ctx->NameSize == 0) {
699 return make_error<InstrProfError>(
700 Args: instrprof_error::unable_to_correlate_profile,
701 Args: "could not find any profile data metadata in object file");
702 }
703 this->Names.append(this->Ctx->NameStart, this->Ctx->NameSize);
704 return Error::success();
705}
706