1//===------------- JITLink.cpp - Core Run-time JIT linker APIs ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/JITLink/JITLink.h"
10
11#include "llvm/ADT/StringExtras.h"
12#include "llvm/BinaryFormat/Magic.h"
13#include "llvm/ExecutionEngine/JITLink/COFF.h"
14#include "llvm/ExecutionEngine/JITLink/ELF.h"
15#include "llvm/ExecutionEngine/JITLink/MachO.h"
16#include "llvm/ExecutionEngine/JITLink/XCOFF.h"
17#include "llvm/ExecutionEngine/JITLink/aarch64.h"
18#include "llvm/ExecutionEngine/JITLink/loongarch.h"
19#include "llvm/ExecutionEngine/JITLink/mips.h"
20#include "llvm/ExecutionEngine/JITLink/ppc64.h"
21#include "llvm/ExecutionEngine/JITLink/systemz.h"
22#include "llvm/ExecutionEngine/JITLink/x86.h"
23#include "llvm/ExecutionEngine/JITLink/x86_64.h"
24#include "llvm/Support/raw_ostream.h"
25
26using namespace llvm;
27using namespace llvm::object;
28
29#define DEBUG_TYPE "jitlink"
30
31namespace {
32
33enum JITLinkErrorCode { GenericJITLinkError = 1 };
34
35// FIXME: This class is only here to support the transition to llvm::Error. It
36// will be removed once this transition is complete. Clients should prefer to
37// deal with the Error value directly, rather than converting to error_code.
38class JITLinkerErrorCategory : public std::error_category {
39public:
40 const char *name() const noexcept override { return "runtimedyld"; }
41
42 std::string message(int Condition) const override {
43 switch (static_cast<JITLinkErrorCode>(Condition)) {
44 case GenericJITLinkError:
45 return "Generic JITLink error";
46 }
47 llvm_unreachable("Unrecognized JITLinkErrorCode");
48 }
49};
50
51} // namespace
52
53namespace llvm {
54namespace jitlink {
55
56char JITLinkError::ID = 0;
57
58void JITLinkError::log(raw_ostream &OS) const { OS << ErrMsg; }
59
60std::error_code JITLinkError::convertToErrorCode() const {
61 static JITLinkerErrorCategory TheJITLinkerErrorCategory;
62 return std::error_code(GenericJITLinkError, TheJITLinkerErrorCategory);
63}
64
65const char *getGenericEdgeKindName(Edge::Kind K) {
66 switch (K) {
67 case Edge::Invalid:
68 return "INVALID RELOCATION";
69 case Edge::KeepAlive:
70 return "Keep-Alive";
71 default:
72 return "<Unrecognized edge kind>";
73 }
74}
75
76const char *getLinkageName(Linkage L) {
77 switch (L) {
78 case Linkage::Strong:
79 return "strong";
80 case Linkage::Weak:
81 return "weak";
82 }
83 llvm_unreachable("Unrecognized llvm.jitlink.Linkage enum");
84}
85
86const char *getScopeName(Scope S) {
87 switch (S) {
88 case Scope::Default:
89 return "default";
90 case Scope::Hidden:
91 return "hidden";
92 case Scope::SideEffectsOnly:
93 return "side-effects-only";
94 case Scope::Local:
95 return "local";
96 }
97 llvm_unreachable("Unrecognized llvm.jitlink.Scope enum");
98}
99
100bool isCStringBlock(Block &B) {
101 if (B.getSize() == 0) // Empty blocks are not valid C-strings.
102 return false;
103
104 // Zero-fill blocks of size one are valid empty strings.
105 if (B.isZeroFill())
106 return B.getSize() == 1;
107
108 for (size_t I = 0; I != B.getSize() - 1; ++I)
109 if (B.getContent()[I] == '\0')
110 return false;
111
112 return B.getContent()[B.getSize() - 1] == '\0';
113}
114
115raw_ostream &operator<<(raw_ostream &OS, const Block &B) {
116 return OS << B.getAddress() << " -- " << (B.getAddress() + B.getSize())
117 << ": "
118 << "size = " << formatv(Fmt: "{0:x8}", Vals: B.getSize()) << ", "
119 << (B.isZeroFill() ? "zero-fill" : "content")
120 << ", align = " << B.getAlignment()
121 << ", align-ofs = " << B.getAlignmentOffset()
122 << ", section = " << B.getSection().getName();
123}
124
125raw_ostream &operator<<(raw_ostream &OS, const Symbol &Sym) {
126 OS << Sym.getAddress() << " (" << (Sym.isDefined() ? "block" : "addressable")
127 << " + " << formatv(Fmt: "{0:x8}", Vals: Sym.getOffset())
128 << "): size: " << formatv(Fmt: "{0:x8}", Vals: Sym.getSize())
129 << ", linkage: " << formatv(Fmt: "{0:6}", Vals: getLinkageName(L: Sym.getLinkage()))
130 << ", scope: " << formatv(Fmt: "{0:8}", Vals: getScopeName(S: Sym.getScope())) << ", "
131 << (Sym.isLive() ? "live" : "dead") << " - "
132 << (Sym.hasName() ? *Sym.getName() : "<anonymous symbol>");
133 return OS;
134}
135
136void printEdge(raw_ostream &OS, const Block &B, const Edge &E,
137 StringRef EdgeKindName) {
138 OS << "edge@" << B.getAddress() + E.getOffset() << ": " << B.getAddress()
139 << " + " << formatv(Fmt: "{0:x}", Vals: E.getOffset()) << " -- " << EdgeKindName
140 << " -> ";
141
142 auto &TargetSym = E.getTarget();
143 if (TargetSym.hasName())
144 OS << TargetSym.getName();
145 else {
146 auto &TargetBlock = TargetSym.getBlock();
147 auto &TargetSec = TargetBlock.getSection();
148 orc::ExecutorAddr SecAddress(~uint64_t(0));
149 for (auto *B : TargetSec.blocks())
150 if (B->getAddress() < SecAddress)
151 SecAddress = B->getAddress();
152
153 orc::ExecutorAddrDiff SecDelta = TargetSym.getAddress() - SecAddress;
154 OS << TargetSym.getAddress() << " (section " << TargetSec.getName();
155 if (SecDelta)
156 OS << " + " << formatv(Fmt: "{0:x}", Vals&: SecDelta);
157 OS << " / block " << TargetBlock.getAddress();
158 if (TargetSym.getOffset())
159 OS << " + " << formatv(Fmt: "{0:x}", Vals: TargetSym.getOffset());
160 OS << ")";
161 }
162
163 if (E.getAddend() != 0)
164 OS << " + " << E.getAddend();
165}
166
167Section::~Section() {
168 for (auto *Sym : Symbols)
169 Sym->~Symbol();
170 for (auto *B : Blocks)
171 B->~Block();
172}
173
174LinkGraph::~LinkGraph() {
175 for (auto *Sym : AbsoluteSymbols) {
176 Sym->~Symbol();
177 }
178 for (auto *Sym : external_symbols()) {
179 Sym->~Symbol();
180 }
181 ExternalSymbols.clear();
182}
183
184std::vector<Block *> LinkGraph::splitBlockImpl(std::vector<Block *> Blocks,
185 SplitBlockCache *Cache) {
186 assert(!Blocks.empty() && "Blocks must at least contain the original block");
187
188 // Fix up content of all blocks.
189 ArrayRef<char> Content = Blocks.front()->getContent();
190 for (size_t I = 0; I != Blocks.size() - 1; ++I) {
191 Blocks[I]->setContent(
192 Content.slice(N: Blocks[I]->getAddress() - Blocks[0]->getAddress(),
193 M: Blocks[I + 1]->getAddress() - Blocks[I]->getAddress()));
194 }
195 Blocks.back()->setContent(
196 Content.slice(N: Blocks.back()->getAddress() - Blocks[0]->getAddress()));
197 bool IsMutable = Blocks[0]->ContentMutable;
198 for (auto *B : Blocks)
199 B->ContentMutable = IsMutable;
200
201 // Transfer symbols.
202 {
203 SplitBlockCache LocalBlockSymbolsCache;
204 if (!Cache)
205 Cache = &LocalBlockSymbolsCache;
206
207 // Build cache if required.
208 if (*Cache == std::nullopt) {
209 *Cache = SplitBlockCache::value_type();
210
211 for (auto *Sym : Blocks[0]->getSection().symbols())
212 if (&Sym->getBlock() == Blocks[0])
213 (*Cache)->push_back(Elt: Sym);
214 llvm::sort(C&: **Cache, Comp: [](const Symbol *LHS, const Symbol *RHS) {
215 return LHS->getAddress() > RHS->getAddress();
216 });
217 }
218
219 auto TransferSymbol = [](Symbol &Sym, Block &B) {
220 Sym.setOffset(Sym.getAddress() - B.getAddress());
221 Sym.setBlock(B);
222 if (Sym.getSize() > B.getSize())
223 Sym.setSize(B.getSize() - Sym.getOffset());
224 };
225
226 // Transfer symbols to all blocks except the last one.
227 for (size_t I = 0; I != Blocks.size() - 1; ++I) {
228 if ((*Cache)->empty())
229 break;
230 while (!(*Cache)->empty() &&
231 (*Cache)->back()->getAddress() < Blocks[I + 1]->getAddress()) {
232 TransferSymbol(*(*Cache)->back(), *Blocks[I]);
233 (*Cache)->pop_back();
234 }
235 }
236 // Transfer symbols to the last block, checking that all are in-range.
237 while (!(*Cache)->empty()) {
238 auto &Sym = *(*Cache)->back();
239 (*Cache)->pop_back();
240 assert(Sym.getAddress() >= Blocks.back()->getAddress() &&
241 "Symbol address preceeds block");
242 assert(Sym.getAddress() <= Blocks.back()->getRange().End &&
243 "Symbol address starts past end of block");
244 TransferSymbol(Sym, *Blocks.back());
245 }
246 }
247
248 // Transfer edges.
249 auto &Edges = Blocks[0]->Edges;
250 llvm::sort(C&: Edges, Comp: [](const Edge &LHS, const Edge &RHS) {
251 return LHS.getOffset() < RHS.getOffset();
252 });
253
254 for (size_t I = Blocks.size() - 1; I != 0; --I) {
255
256 // If all edges have been transferred then bail out.
257 if (Edges.empty())
258 break;
259
260 Edge::OffsetT Delta = Blocks[I]->getAddress() - Blocks[0]->getAddress();
261
262 // If no edges to move for this block then move to the next one.
263 if (Edges.back().getOffset() < Delta)
264 continue;
265
266 size_t EI = Edges.size() - 1;
267 while (EI != 0 && Edges[EI - 1].getOffset() >= Delta)
268 --EI;
269
270 for (size_t J = EI; J != Edges.size(); ++J) {
271 Blocks[I]->Edges.push_back(x: std::move(Edges[J]));
272 Blocks[I]->Edges.back().setOffset(Blocks[I]->Edges.back().getOffset() -
273 Delta);
274 }
275
276 while (Edges.size() > EI)
277 Edges.pop_back();
278 }
279
280 return Blocks;
281}
282
283void LinkGraph::dump(raw_ostream &OS) {
284 DenseMap<Block *, std::vector<Symbol *>> BlockSymbols;
285
286 OS << "LinkGraph \"" << getName()
287 << "\" (triple = " << getTargetTriple().str() << ")\n";
288
289 // Map from blocks to the symbols pointing at them.
290 for (auto *Sym : defined_symbols())
291 BlockSymbols[&Sym->getBlock()].push_back(x: Sym);
292
293 // For each block, sort its symbols by something approximating
294 // relevance.
295 for (auto &KV : BlockSymbols)
296 llvm::sort(C&: KV.second, Comp: [](const Symbol *LHS, const Symbol *RHS) {
297 if (LHS->getOffset() != RHS->getOffset())
298 return LHS->getOffset() < RHS->getOffset();
299 if (LHS->getLinkage() != RHS->getLinkage())
300 return LHS->getLinkage() < RHS->getLinkage();
301 if (LHS->getScope() != RHS->getScope())
302 return LHS->getScope() < RHS->getScope();
303 if (LHS->hasName()) {
304 if (!RHS->hasName())
305 return true;
306 return LHS->getName() < RHS->getName();
307 }
308 return false;
309 });
310
311 std::vector<Section *> SortedSections;
312 for (auto &Sec : sections())
313 SortedSections.push_back(x: &Sec);
314 llvm::sort(C&: SortedSections, Comp: [](const Section *LHS, const Section *RHS) {
315 return LHS->getName() < RHS->getName();
316 });
317
318 for (auto *Sec : SortedSections) {
319 OS << "section " << Sec->getName() << ":\n\n";
320
321 std::vector<Block *> SortedBlocks;
322 llvm::append_range(C&: SortedBlocks, R: Sec->blocks());
323 llvm::sort(C&: SortedBlocks, Comp: [](const Block *LHS, const Block *RHS) {
324 return LHS->getAddress() < RHS->getAddress();
325 });
326
327 for (auto *B : SortedBlocks) {
328 OS << " block " << B->getAddress()
329 << " size = " << formatv(Fmt: "{0:x8}", Vals: B->getSize())
330 << ", align = " << B->getAlignment()
331 << ", alignment-offset = " << B->getAlignmentOffset();
332 if (B->isZeroFill())
333 OS << ", zero-fill";
334 OS << "\n";
335
336 auto BlockSymsI = BlockSymbols.find(Val: B);
337 if (BlockSymsI != BlockSymbols.end()) {
338 OS << " symbols:\n";
339 auto &Syms = BlockSymsI->second;
340 for (auto *Sym : Syms)
341 OS << " " << *Sym << "\n";
342 } else
343 OS << " no symbols\n";
344
345 if (!B->edges_empty()) {
346 OS << " edges:\n";
347 std::vector<Edge> SortedEdges;
348 llvm::append_range(C&: SortedEdges, R: B->edges());
349 llvm::sort(C&: SortedEdges, Comp: [](const Edge &LHS, const Edge &RHS) {
350 return LHS.getOffset() < RHS.getOffset();
351 });
352 for (auto &E : SortedEdges) {
353 OS << " " << B->getFixupAddress(E) << " (block + "
354 << formatv(Fmt: "{0:x8}", Vals: E.getOffset()) << "), addend = ";
355 if (E.getAddend() >= 0)
356 OS << formatv(Fmt: "+{0:x8}", Vals: E.getAddend());
357 else
358 OS << formatv(Fmt: "-{0:x8}", Vals: -E.getAddend());
359 OS << ", kind = " << getEdgeKindName(K: E.getKind()) << ", target = ";
360 if (E.getTarget().hasName())
361 OS << E.getTarget().getName();
362 else
363 OS << "addressable@"
364 << formatv(Fmt: "{0:x16}", Vals: E.getTarget().getAddress()) << "+"
365 << formatv(Fmt: "{0:x8}", Vals: E.getTarget().getOffset());
366 OS << "\n";
367 }
368 } else
369 OS << " no edges\n";
370 OS << "\n";
371 }
372 }
373
374 OS << "Absolute symbols:\n";
375 if (!absolute_symbols().empty()) {
376 for (auto *Sym : absolute_symbols())
377 OS << " " << Sym->getAddress() << ": " << *Sym << "\n";
378 } else
379 OS << " none\n";
380
381 OS << "\nExternal symbols:\n";
382 if (!external_symbols().empty()) {
383 for (auto *Sym : external_symbols())
384 OS << " " << Sym->getAddress() << ": " << *Sym
385 << (Sym->isWeaklyReferenced() ? " (weakly referenced)" : "") << "\n";
386 } else
387 OS << " none\n";
388}
389
390raw_ostream &operator<<(raw_ostream &OS, const SymbolLookupFlags &LF) {
391 switch (LF) {
392 case SymbolLookupFlags::RequiredSymbol:
393 return OS << "RequiredSymbol";
394 case SymbolLookupFlags::WeaklyReferencedSymbol:
395 return OS << "WeaklyReferencedSymbol";
396 }
397 llvm_unreachable("Unrecognized lookup flags");
398}
399
400void JITLinkAsyncLookupContinuation::anchor() {}
401
402JITLinkContext::~JITLinkContext() = default;
403
404bool JITLinkContext::shouldAddDefaultTargetPasses(const Triple &TT) const {
405 return true;
406}
407
408LinkGraphPassFunction JITLinkContext::getMarkLivePass(const Triple &TT) const {
409 return LinkGraphPassFunction();
410}
411
412Error JITLinkContext::modifyPassConfig(LinkGraph &G,
413 PassConfiguration &Config) {
414 return Error::success();
415}
416
417Error markAllSymbolsLive(LinkGraph &G) {
418 for (auto *Sym : G.defined_symbols())
419 Sym->setLive(true);
420 return Error::success();
421}
422
423Error makeTargetOutOfRangeError(const LinkGraph &G, const Block &B,
424 const Edge &E) {
425 std::string ErrMsg;
426 {
427 raw_string_ostream ErrStream(ErrMsg);
428 Section &Sec = B.getSection();
429 ErrStream << "In graph " << G.getName() << ", section " << Sec.getName()
430 << ": relocation target "
431 << formatv(Fmt: "{0:x}", Vals: E.getTarget().getAddress() + E.getAddend())
432 << " (";
433 if (E.getTarget().hasName())
434 ErrStream << E.getTarget().getName();
435 else
436 ErrStream << "<anonymous symbol>";
437 if (E.getAddend()) {
438 // Target address includes non-zero added, so break down the arithmetic.
439 ErrStream << formatv(Fmt: ":{0:x}", Vals: E.getTarget().getAddress()) << " + "
440 << formatv(Fmt: "{0:x}", Vals: E.getAddend());
441 }
442 ErrStream << ") is out of range of " << G.getEdgeKindName(K: E.getKind())
443 << " fixup at address "
444 << formatv(Fmt: "{0:x}", Vals: E.getTarget().getAddress()) << " (";
445
446 Symbol *BestSymbolForBlock = nullptr;
447 for (auto *Sym : Sec.symbols())
448 if (&Sym->getBlock() == &B && Sym->hasName() && Sym->getOffset() == 0 &&
449 (!BestSymbolForBlock ||
450 Sym->getScope() < BestSymbolForBlock->getScope() ||
451 Sym->getLinkage() < BestSymbolForBlock->getLinkage()))
452 BestSymbolForBlock = Sym;
453
454 if (BestSymbolForBlock)
455 ErrStream << BestSymbolForBlock->getName() << ", ";
456 else
457 ErrStream << "<anonymous block> @ ";
458
459 ErrStream << formatv(Fmt: "{0:x}", Vals: B.getAddress()) << " + "
460 << formatv(Fmt: "{0:x}", Vals: E.getOffset()) << ")";
461 }
462 return make_error<JITLinkError>(Args: std::move(ErrMsg));
463}
464
465Error makeAlignmentError(llvm::orc::ExecutorAddr Loc, uint64_t Value, int N,
466 const Edge &E) {
467 return make_error<JITLinkError>(Args: "0x" + llvm::utohexstr(X: Loc.getValue()) +
468 " improper alignment for relocation " +
469 formatv(Fmt: "{0:d}", Vals: E.getKind()) + ": 0x" +
470 llvm::utohexstr(X: Value) +
471 " is not aligned to " + Twine(N) + " bytes");
472}
473
474AnonymousPointerCreator getAnonymousPointerCreator(const Triple &TT) {
475 switch (TT.getArch()) {
476 case Triple::aarch64:
477 return aarch64::createAnonymousPointer;
478 case Triple::x86_64:
479 return x86_64::createAnonymousPointer;
480 case Triple::x86:
481 return x86::createAnonymousPointer;
482 case Triple::loongarch32:
483 case Triple::loongarch64:
484 return loongarch::createAnonymousPointer;
485 case Triple::mips:
486 case Triple::mipsel:
487 case Triple::mips64:
488 case Triple::mips64el:
489 return mips::createAnonymousPointer;
490 case Triple::systemz:
491 return systemz::createAnonymousPointer;
492 case Triple::ppc64:
493 case Triple::ppc64le:
494 return ppc64::createAnonymousPointer;
495 default:
496 return nullptr;
497 }
498}
499
500PointerJumpStubCreator getPointerJumpStubCreator(const Triple &TT) {
501 switch (TT.getArch()) {
502 case Triple::aarch64:
503 return aarch64::createAnonymousPointerJumpStub;
504 case Triple::x86_64:
505 return x86_64::createAnonymousPointerJumpStub;
506 case Triple::x86:
507 return x86::createAnonymousPointerJumpStub;
508 case Triple::loongarch32:
509 case Triple::loongarch64:
510 return loongarch::createAnonymousPointerJumpStub;
511 case Triple::mips:
512 case Triple::mipsel:
513 case Triple::mips64:
514 case Triple::mips64el:
515 return mips::createAnonymousPointerJumpStub;
516 case Triple::systemz:
517 return systemz::createAnonymousPointerJumpStub;
518 case Triple::ppc64:
519 return ppc64::createDefaultAnonymousPointerJumpStub<llvm::endianness::big>;
520 case Triple::ppc64le:
521 return ppc64::createDefaultAnonymousPointerJumpStub<
522 llvm::endianness::little>;
523 default:
524 return nullptr;
525 }
526}
527
528Expected<std::unique_ptr<LinkGraph>>
529createLinkGraphFromObject(MemoryBufferRef ObjectBuffer,
530 std::shared_ptr<orc::SymbolStringPool> SSP) {
531 auto Magic = identify_magic(magic: ObjectBuffer.getBuffer());
532 switch (Magic) {
533 case file_magic::macho_object:
534 return createLinkGraphFromMachOObject(ObjectBuffer, SSP: std::move(SSP));
535 case file_magic::elf_relocatable:
536 return createLinkGraphFromELFObject(ObjectBuffer, SSP: std::move(SSP));
537 case file_magic::coff_object:
538 return createLinkGraphFromCOFFObject(ObjectBuffer, SSP: std::move(SSP));
539 case file_magic::xcoff_object_64:
540 return createLinkGraphFromXCOFFObject(ObjectBuffer, SSP: std::move(SSP));
541 default:
542 return make_error<JITLinkError>(Args: "Unsupported file format");
543 };
544}
545
546std::unique_ptr<LinkGraph>
547absoluteSymbolsLinkGraph(Triple TT, std::shared_ptr<orc::SymbolStringPool> SSP,
548 orc::SymbolMap Symbols) {
549 static std::atomic<uint64_t> Counter = {0};
550 auto Index = Counter.fetch_add(i: 1, m: std::memory_order_relaxed);
551 unsigned PointerSize = TT.getArchPointerBitWidth() / 8;
552 auto G = std::make_unique<LinkGraph>(
553 args: "<Absolute Symbols " + std::to_string(val: Index) + ">", args: std::move(SSP),
554 args: std::move(TT), args&: PointerSize, args: SubtargetFeatures(), args&: getGenericEdgeKindName);
555 for (auto &[Name, Def] : Symbols) {
556 auto &Sym =
557 G->addAbsoluteSymbol(Name: *Name, Address: Def.getAddress(), /*Size=*/0,
558 L: Linkage::Strong, S: Scope::Default, /*IsLive=*/true);
559 Sym.setCallable(Def.getFlags().isCallable());
560 }
561
562 return G;
563}
564
565void link(std::unique_ptr<LinkGraph> G, std::unique_ptr<JITLinkContext> Ctx) {
566 switch (G->getTargetTriple().getObjectFormat()) {
567 case Triple::MachO:
568 return link_MachO(G: std::move(G), Ctx: std::move(Ctx));
569 case Triple::ELF:
570 return link_ELF(G: std::move(G), Ctx: std::move(Ctx));
571 case Triple::COFF:
572 return link_COFF(G: std::move(G), Ctx: std::move(Ctx));
573 case Triple::XCOFF:
574 return link_XCOFF(G: std::move(G), Ctx: std::move(Ctx));
575 default:
576 Ctx->notifyFailed(Err: make_error<JITLinkError>(Args: "Unsupported object format"));
577 };
578}
579
580} // end namespace jitlink
581} // end namespace llvm
582