1//===-- InstrProfCorrelator.cpp -------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ProfileData/InstrProfCorrelator.h"
10#include "llvm/DebugInfo/DIContext.h"
11#include "llvm/DebugInfo/DWARF/DWARFContext.h"
12#include "llvm/DebugInfo/DWARF/DWARFDie.h"
13#include "llvm/DebugInfo/DWARF/DWARFExpression.h"
14#include "llvm/DebugInfo/DWARF/DWARFFormValue.h"
15#include "llvm/DebugInfo/DWARF/DWARFLocationExpression.h"
16#include "llvm/DebugInfo/DWARF/DWARFUnit.h"
17#include "llvm/Object/MachO.h"
18#include "llvm/Support/Debug.h"
19#include "llvm/Support/Format.h"
20#include "llvm/Support/WithColor.h"
21#include <optional>
22
23#define DEBUG_TYPE "correlator"
24
25using namespace llvm;
26
27/// Get profile section.
28Expected<object::SectionRef> getInstrProfSection(const object::ObjectFile &Obj,
29 InstrProfSectKind IPSK) {
30 // On COFF, the getInstrProfSectionName returns the section names may followed
31 // by "$M". The linker removes the dollar and everything after it in the final
32 // binary. Do the same to match.
33 Triple::ObjectFormatType ObjFormat = Obj.getTripleObjectFormat();
34 auto StripSuffix = [ObjFormat](StringRef N) {
35 return ObjFormat == Triple::COFF ? N.split(Separator: '$').first : N;
36 };
37 std::string ExpectedSectionName =
38 getInstrProfSectionName(IPSK, OF: ObjFormat,
39 /*AddSegmentInfo=*/false);
40 ExpectedSectionName = StripSuffix(ExpectedSectionName);
41 for (auto &Section : Obj.sections()) {
42 if (auto SectionName = Section.getName())
43 if (*SectionName == ExpectedSectionName)
44 return Section;
45 }
46 return make_error<InstrProfError>(
47 Args: instrprof_error::unable_to_correlate_profile,
48 Args: "could not find section (" + Twine(ExpectedSectionName) + ")");
49}
50
51const char *InstrProfCorrelator::FunctionNameAttributeName = "Function Name";
52const char *InstrProfCorrelator::CFGHashAttributeName = "CFG Hash";
53const char *InstrProfCorrelator::NumCountersAttributeName = "Num Counters";
54
55llvm::Expected<std::unique_ptr<InstrProfCorrelator::Context>>
56InstrProfCorrelator::Context::get(std::unique_ptr<MemoryBuffer> Buffer,
57 const object::ObjectFile &Obj,
58 ProfCorrelatorKind FileKind) {
59 auto C = std::make_unique<Context>();
60 auto CountersSection = getInstrProfSection(Obj, IPSK: IPSK_cnts);
61 if (auto Err = CountersSection.takeError())
62 return std::move(Err);
63 if (FileKind == InstrProfCorrelator::BINARY) {
64 auto DataSection = getInstrProfSection(Obj, IPSK: IPSK_covdata);
65 if (auto Err = DataSection.takeError())
66 return std::move(Err);
67 auto DataOrErr = DataSection->getContents();
68 if (!DataOrErr)
69 return DataOrErr.takeError();
70 auto NameSection = getInstrProfSection(Obj, IPSK: IPSK_covname);
71 if (auto Err = NameSection.takeError())
72 return std::move(Err);
73 auto NameOrErr = NameSection->getContents();
74 if (!NameOrErr)
75 return NameOrErr.takeError();
76 C->DataStart = DataOrErr->data();
77 C->DataEnd = DataOrErr->data() + DataOrErr->size();
78 C->NameStart = NameOrErr->data();
79 C->NameSize = NameOrErr->size();
80 }
81 C->Buffer = std::move(Buffer);
82 C->CountersSectionStart = CountersSection->getAddress();
83 C->CountersSectionEnd = C->CountersSectionStart + CountersSection->getSize();
84 // In COFF object file, there's a null byte at the beginning of the counter
85 // section which doesn't exist in raw profile.
86 if (Obj.getTripleObjectFormat() == Triple::COFF)
87 ++C->CountersSectionStart;
88
89 C->ShouldSwapBytes = Obj.isLittleEndian() != sys::IsLittleEndianHost;
90 return Expected<std::unique_ptr<Context>>(std::move(C));
91}
92
93llvm::Expected<std::unique_ptr<InstrProfCorrelator>>
94InstrProfCorrelator::get(StringRef Filename, ProfCorrelatorKind FileKind) {
95 if (FileKind == DEBUG_INFO) {
96 auto DsymObjectsOrErr =
97 object::MachOObjectFile::findDsymObjectMembers(Path: Filename);
98 if (auto Err = DsymObjectsOrErr.takeError())
99 return std::move(Err);
100 if (!DsymObjectsOrErr->empty()) {
101 // TODO: Enable profile correlation when there are multiple objects in a
102 // dSYM bundle.
103 if (DsymObjectsOrErr->size() > 1)
104 return make_error<InstrProfError>(
105 Args: instrprof_error::unable_to_correlate_profile,
106 Args: "using multiple objects is not yet supported");
107 Filename = *DsymObjectsOrErr->begin();
108 }
109 auto BufferOrErr = errorOrToExpected(EO: MemoryBuffer::getFile(Filename));
110 if (auto Err = BufferOrErr.takeError())
111 return std::move(Err);
112
113 return get(Buffer: std::move(*BufferOrErr), FileKind);
114 }
115 if (FileKind == BINARY) {
116 auto BufferOrErr = errorOrToExpected(EO: MemoryBuffer::getFile(Filename));
117 if (auto Err = BufferOrErr.takeError())
118 return std::move(Err);
119
120 return get(Buffer: std::move(*BufferOrErr), FileKind);
121 }
122 return make_error<InstrProfError>(
123 Args: instrprof_error::unable_to_correlate_profile,
124 Args: "unsupported correlation kind (only DWARF debug info and Binary format "
125 "(ELF/COFF) are supported)");
126}
127
128llvm::Expected<std::unique_ptr<InstrProfCorrelator>>
129InstrProfCorrelator::get(std::unique_ptr<MemoryBuffer> Buffer,
130 ProfCorrelatorKind FileKind) {
131 auto BinOrErr = object::createBinary(Source: *Buffer);
132 if (auto Err = BinOrErr.takeError())
133 return std::move(Err);
134
135 if (auto *Obj = dyn_cast<object::ObjectFile>(Val: BinOrErr->get())) {
136 auto CtxOrErr = Context::get(Buffer: std::move(Buffer), Obj: *Obj, FileKind);
137 if (auto Err = CtxOrErr.takeError())
138 return std::move(Err);
139 auto T = Obj->makeTriple();
140 if (T.isArch64Bit())
141 return InstrProfCorrelatorImpl<uint64_t>::get(Ctx: std::move(*CtxOrErr), Obj: *Obj,
142 FileKind);
143 if (T.isArch32Bit())
144 return InstrProfCorrelatorImpl<uint32_t>::get(Ctx: std::move(*CtxOrErr), Obj: *Obj,
145 FileKind);
146 }
147 return make_error<InstrProfError>(
148 Args: instrprof_error::unable_to_correlate_profile, Args: "not an object file");
149}
150
151std::optional<size_t> InstrProfCorrelator::getDataSize() const {
152 if (auto *C = dyn_cast<InstrProfCorrelatorImpl<uint32_t>>(Val: this)) {
153 return C->getDataSize();
154 } else if (auto *C = dyn_cast<InstrProfCorrelatorImpl<uint64_t>>(Val: this)) {
155 return C->getDataSize();
156 }
157 return {};
158}
159
160namespace llvm {
161
162template <>
163InstrProfCorrelatorImpl<uint32_t>::InstrProfCorrelatorImpl(
164 std::unique_ptr<InstrProfCorrelator::Context> Ctx)
165 : InstrProfCorrelatorImpl(InstrProfCorrelatorKind::CK_32Bit,
166 std::move(Ctx)) {}
167template <>
168InstrProfCorrelatorImpl<uint64_t>::InstrProfCorrelatorImpl(
169 std::unique_ptr<InstrProfCorrelator::Context> Ctx)
170 : InstrProfCorrelatorImpl(InstrProfCorrelatorKind::CK_64Bit,
171 std::move(Ctx)) {}
172template <>
173bool InstrProfCorrelatorImpl<uint32_t>::classof(const InstrProfCorrelator *C) {
174 return C->getKind() == InstrProfCorrelatorKind::CK_32Bit;
175}
176template <>
177bool InstrProfCorrelatorImpl<uint64_t>::classof(const InstrProfCorrelator *C) {
178 return C->getKind() == InstrProfCorrelatorKind::CK_64Bit;
179}
180
181} // end namespace llvm
182
183template <class IntPtrT>
184llvm::Expected<std::unique_ptr<InstrProfCorrelatorImpl<IntPtrT>>>
185InstrProfCorrelatorImpl<IntPtrT>::get(
186 std::unique_ptr<InstrProfCorrelator::Context> Ctx,
187 const object::ObjectFile &Obj, ProfCorrelatorKind FileKind) {
188 if (FileKind == DEBUG_INFO) {
189 if (Obj.isELF() || Obj.isMachO()) {
190 auto DICtx = DWARFContext::create(Obj);
191 return std::make_unique<DwarfInstrProfCorrelator<IntPtrT>>(
192 std::move(DICtx), std::move(Ctx));
193 }
194 return make_error<InstrProfError>(
195 Args: instrprof_error::unable_to_correlate_profile,
196 Args: "unsupported debug info format (only DWARF is supported)");
197 }
198 if (Obj.isELF() || Obj.isCOFF())
199 return std::make_unique<BinaryInstrProfCorrelator<IntPtrT>>(std::move(Ctx));
200 return make_error<InstrProfError>(
201 Args: instrprof_error::unable_to_correlate_profile,
202 Args: "unsupported binary format (only ELF and COFF are supported)");
203}
204
205template <class IntPtrT>
206Error InstrProfCorrelatorImpl<IntPtrT>::correlateProfileData(int MaxWarnings) {
207 assert(Data.empty() && Names.empty() && NamesVec.empty());
208 correlateProfileDataImpl(MaxWarnings);
209 if (this->Data.empty())
210 return make_error<InstrProfError>(
211 Args: instrprof_error::unable_to_correlate_profile,
212 Args: "could not find any profile data metadata in correlated file");
213 Error Result = correlateProfileNameImpl();
214 this->CounterOffsets.clear();
215 this->NamesVec.clear();
216 return Result;
217}
218
219template <> struct yaml::MappingTraits<InstrProfCorrelator::CorrelationData> {
220 static void mapping(yaml::IO &io,
221 InstrProfCorrelator::CorrelationData &Data) {
222 io.mapRequired(Key: "Probes", Val&: Data.Probes);
223 }
224};
225
226template <> struct yaml::MappingTraits<InstrProfCorrelator::Probe> {
227 static void mapping(yaml::IO &io, InstrProfCorrelator::Probe &P) {
228 io.mapRequired(Key: "Function Name", Val&: P.FunctionName);
229 io.mapOptional(Key: "Linkage Name", Val&: P.LinkageName);
230 io.mapRequired(Key: "CFG Hash", Val&: P.CFGHash);
231 io.mapRequired(Key: "Counter Offset", Val&: P.CounterOffset);
232 io.mapRequired(Key: "Num Counters", Val&: P.NumCounters);
233 io.mapOptional(Key: "File", Val&: P.FilePath);
234 io.mapOptional(Key: "Line", Val&: P.LineNumber);
235 }
236};
237
238template <> struct yaml::SequenceElementTraits<InstrProfCorrelator::Probe> {
239 static const bool flow = false;
240};
241
242template <class IntPtrT>
243Error InstrProfCorrelatorImpl<IntPtrT>::dumpYaml(int MaxWarnings,
244 raw_ostream &OS) {
245 InstrProfCorrelator::CorrelationData Data;
246 correlateProfileDataImpl(MaxWarnings, Data: &Data);
247 if (Data.Probes.empty())
248 return make_error<InstrProfError>(
249 Args: instrprof_error::unable_to_correlate_profile,
250 Args: "could not find any profile data metadata in debug info");
251 yaml::Output YamlOS(OS);
252 YamlOS << Data;
253 return Error::success();
254}
255
256template <class IntPtrT>
257void InstrProfCorrelatorImpl<IntPtrT>::addDataProbe(uint64_t NameRef,
258 uint64_t CFGHash,
259 IntPtrT CounterOffset,
260 IntPtrT FunctionPtr,
261 uint32_t NumCounters) {
262 // Check if a probe was already added for this counter offset.
263 if (!CounterOffsets.insert(CounterOffset).second)
264 return;
265 Data.push_back({
266 maybeSwap<uint64_t>(NameRef),
267 maybeSwap<uint64_t>(CFGHash),
268 // In this mode, CounterPtr actually stores the section relative address
269 // of the counter.
270 maybeSwap<IntPtrT>(CounterOffset),
271 // TODO: MC/DC is not yet supported.
272 /*BitmapOffset=*/maybeSwap<IntPtrT>(0),
273 maybeSwap<IntPtrT>(FunctionPtr),
274 // TODO: Value profiling is not yet supported.
275 /*ValuesPtr=*/maybeSwap<IntPtrT>(0),
276 maybeSwap<uint32_t>(NumCounters),
277 /*NumValueSites=*/{maybeSwap<uint16_t>(0), maybeSwap<uint16_t>(0)},
278 // TODO: MC/DC is not yet supported.
279 /*NumBitmapBytes=*/maybeSwap<uint32_t>(0),
280 });
281}
282
283template <class IntPtrT>
284std::optional<uint64_t>
285DwarfInstrProfCorrelator<IntPtrT>::getLocation(const DWARFDie &Die) const {
286 auto Locations = Die.getLocations(Attr: dwarf::DW_AT_location);
287 if (!Locations) {
288 consumeError(Err: Locations.takeError());
289 return {};
290 }
291 auto &DU = *Die.getDwarfUnit();
292 auto AddressSize = DU.getAddressByteSize();
293 for (auto &Location : *Locations) {
294 DataExtractor Data(Location.Expr, DICtx->isLittleEndian(), AddressSize);
295 DWARFExpression Expr(Data, AddressSize);
296 for (auto &Op : Expr) {
297 if (Op.getCode() == dwarf::DW_OP_addr) {
298 return Op.getRawOperand(Idx: 0);
299 } else if (Op.getCode() == dwarf::DW_OP_addrx) {
300 uint64_t Index = Op.getRawOperand(Idx: 0);
301 if (auto SA = DU.getAddrOffsetSectionItem(Index))
302 return SA->Address;
303 }
304 }
305 }
306 return {};
307}
308
309template <class IntPtrT>
310bool DwarfInstrProfCorrelator<IntPtrT>::isDIEOfProbe(const DWARFDie &Die) {
311 const auto &ParentDie = Die.getParent();
312 if (!Die.isValid() || !ParentDie.isValid() || Die.isNULL())
313 return false;
314 if (Die.getTag() != dwarf::DW_TAG_variable)
315 return false;
316 if (!ParentDie.isSubprogramDIE())
317 return false;
318 if (!Die.hasChildren())
319 return false;
320 if (const char *Name = Die.getName(Kind: DINameKind::ShortName))
321 return StringRef(Name).starts_with(Prefix: getInstrProfCountersVarPrefix());
322 return false;
323}
324
325template <class IntPtrT>
326void DwarfInstrProfCorrelator<IntPtrT>::correlateProfileDataImpl(
327 int MaxWarnings, InstrProfCorrelator::CorrelationData *Data) {
328 bool UnlimitedWarnings = (MaxWarnings == 0);
329 // -N suppressed warnings means we can emit up to N (unsuppressed) warnings
330 int NumSuppressedWarnings = -MaxWarnings;
331 auto maybeAddProbe = [&](DWARFDie Die) {
332 if (!isDIEOfProbe(Die))
333 return;
334 std::optional<const char *> FunctionName;
335 std::optional<uint64_t> CFGHash;
336 std::optional<uint64_t> CounterPtr = getLocation(Die);
337 auto FnDie = Die.getParent();
338 auto FunctionPtr = dwarf::toAddress(V: FnDie.find(Attr: dwarf::DW_AT_low_pc));
339 std::optional<uint64_t> NumCounters;
340 for (const DWARFDie &Child : Die.children()) {
341 if (Child.getTag() != dwarf::DW_TAG_LLVM_annotation)
342 continue;
343 auto AnnotationFormName = Child.find(Attr: dwarf::DW_AT_name);
344 auto AnnotationFormValue = Child.find(Attr: dwarf::DW_AT_const_value);
345 if (!AnnotationFormName || !AnnotationFormValue)
346 continue;
347 auto AnnotationNameOrErr = AnnotationFormName->getAsCString();
348 if (auto Err = AnnotationNameOrErr.takeError()) {
349 consumeError(Err: std::move(Err));
350 continue;
351 }
352 StringRef AnnotationName = *AnnotationNameOrErr;
353 if (AnnotationName == InstrProfCorrelator::FunctionNameAttributeName) {
354 if (auto EC =
355 AnnotationFormValue->getAsCString().moveInto(Value&: FunctionName))
356 consumeError(Err: std::move(EC));
357 } else if (AnnotationName == InstrProfCorrelator::CFGHashAttributeName) {
358 CFGHash = AnnotationFormValue->getAsUnsignedConstant();
359 } else if (AnnotationName ==
360 InstrProfCorrelator::NumCountersAttributeName) {
361 NumCounters = AnnotationFormValue->getAsUnsignedConstant();
362 }
363 }
364 if (!FunctionName || !CFGHash || !CounterPtr || !NumCounters) {
365 if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) {
366 WithColor::warning()
367 << "Incomplete DIE for function " << FunctionName
368 << ": CFGHash=" << CFGHash << " CounterPtr=" << CounterPtr
369 << " NumCounters=" << NumCounters << "\n";
370 LLVM_DEBUG(Die.dump(dbgs()));
371 }
372 return;
373 }
374 uint64_t CountersStart = this->Ctx->CountersSectionStart;
375 uint64_t CountersEnd = this->Ctx->CountersSectionEnd;
376 if (*CounterPtr < CountersStart || *CounterPtr >= CountersEnd) {
377 if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) {
378 WithColor::warning()
379 << format(Fmt: "CounterPtr out of range for function %s: Actual=0x%x "
380 "Expected=[0x%x, 0x%x)\n",
381 Vals: *FunctionName, Vals: *CounterPtr, Vals: CountersStart, Vals: CountersEnd);
382 LLVM_DEBUG(Die.dump(dbgs()));
383 }
384 return;
385 }
386 if (!FunctionPtr && (UnlimitedWarnings || ++NumSuppressedWarnings < 1)) {
387 WithColor::warning() << format(Fmt: "Could not find address of function %s\n",
388 Vals: *FunctionName);
389 LLVM_DEBUG(Die.dump(dbgs()));
390 }
391 // In debug info correlation mode, the CounterPtr is an absolute address of
392 // the counter, but it's expected to be relative later when iterating Data.
393 IntPtrT CounterOffset = *CounterPtr - CountersStart;
394 if (Data) {
395 InstrProfCorrelator::Probe P;
396 P.FunctionName = *FunctionName;
397 if (auto Name = FnDie.getName(Kind: DINameKind::LinkageName))
398 P.LinkageName = Name;
399 P.CFGHash = *CFGHash;
400 P.CounterOffset = CounterOffset;
401 P.NumCounters = *NumCounters;
402 auto FilePath = FnDie.getDeclFile(
403 Kind: DILineInfoSpecifier::FileLineInfoKind::RelativeFilePath);
404 if (!FilePath.empty())
405 P.FilePath = FilePath;
406 if (auto LineNumber = FnDie.getDeclLine())
407 P.LineNumber = LineNumber;
408 Data->Probes.push_back(x: P);
409 } else {
410 this->addDataProbe(IndexedInstrProf::ComputeHash(K: *FunctionName), *CFGHash,
411 CounterOffset, FunctionPtr.value_or(u: 0), *NumCounters);
412 this->NamesVec.push_back(*FunctionName);
413 }
414 };
415 for (auto &CU : DICtx->normal_units())
416 for (const auto &Entry : CU->dies())
417 maybeAddProbe(DWARFDie(CU.get(), &Entry));
418 for (auto &CU : DICtx->dwo_units())
419 for (const auto &Entry : CU->dies())
420 maybeAddProbe(DWARFDie(CU.get(), &Entry));
421
422 if (!UnlimitedWarnings && NumSuppressedWarnings > 0)
423 WithColor::warning() << format(Fmt: "Suppressed %d additional warnings\n",
424 Vals: NumSuppressedWarnings);
425}
426
427template <class IntPtrT>
428Error DwarfInstrProfCorrelator<IntPtrT>::correlateProfileNameImpl() {
429 if (this->NamesVec.empty()) {
430 return make_error<InstrProfError>(
431 Args: instrprof_error::unable_to_correlate_profile,
432 Args: "could not find any profile name metadata in debug info");
433 }
434 auto Result =
435 collectGlobalObjectNameStrings(this->NamesVec,
436 /*doCompression=*/false, this->Names);
437 return Result;
438}
439
440template <class IntPtrT>
441void BinaryInstrProfCorrelator<IntPtrT>::correlateProfileDataImpl(
442 int MaxWarnings, InstrProfCorrelator::CorrelationData *CorrelateData) {
443 using RawProfData = RawInstrProf::ProfileData<IntPtrT>;
444 bool UnlimitedWarnings = (MaxWarnings == 0);
445 // -N suppressed warnings means we can emit up to N (unsuppressed) warnings
446 int NumSuppressedWarnings = -MaxWarnings;
447
448 const RawProfData *DataStart = (const RawProfData *)this->Ctx->DataStart;
449 const RawProfData *DataEnd = (const RawProfData *)this->Ctx->DataEnd;
450 // We need to use < here because the last data record may have no padding.
451 for (const RawProfData *I = DataStart; I < DataEnd; ++I) {
452 uint64_t CounterPtr = this->template maybeSwap<IntPtrT>(I->CounterPtr);
453 uint64_t CountersStart = this->Ctx->CountersSectionStart;
454 uint64_t CountersEnd = this->Ctx->CountersSectionEnd;
455 if (CounterPtr < CountersStart || CounterPtr >= CountersEnd) {
456 if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) {
457 WithColor::warning()
458 << format("CounterPtr out of range for function: Actual=0x%x "
459 "Expected=[0x%x, 0x%x) at data offset=0x%x\n",
460 CounterPtr, CountersStart, CountersEnd,
461 (I - DataStart) * sizeof(RawProfData));
462 }
463 }
464 // In binary correlation mode, the CounterPtr is an absolute address of the
465 // counter, but it's expected to be relative later when iterating Data.
466 IntPtrT CounterOffset = CounterPtr - CountersStart;
467 this->addDataProbe(I->NameRef, I->FuncHash, CounterOffset,
468 I->FunctionPointer, I->NumCounters);
469 }
470}
471
472template <class IntPtrT>
473Error BinaryInstrProfCorrelator<IntPtrT>::correlateProfileNameImpl() {
474 if (this->Ctx->NameSize == 0) {
475 return make_error<InstrProfError>(
476 Args: instrprof_error::unable_to_correlate_profile,
477 Args: "could not find any profile data metadata in object file");
478 }
479 this->Names.append(this->Ctx->NameStart, this->Ctx->NameSize);
480 return Error::success();
481}
482