1//===------------- JITLink.cpp - Core Run-time JIT linker APIs ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/JITLink/JITLink.h"
10
11#include "llvm/ADT/StringExtras.h"
12#include "llvm/BinaryFormat/Magic.h"
13#include "llvm/ExecutionEngine/JITLink/COFF.h"
14#include "llvm/ExecutionEngine/JITLink/ELF.h"
15#include "llvm/ExecutionEngine/JITLink/MachO.h"
16#include "llvm/ExecutionEngine/JITLink/XCOFF.h"
17#include "llvm/ExecutionEngine/JITLink/aarch64.h"
18#include "llvm/ExecutionEngine/JITLink/loongarch.h"
19#include "llvm/ExecutionEngine/JITLink/ppc64.h"
20#include "llvm/ExecutionEngine/JITLink/systemz.h"
21#include "llvm/ExecutionEngine/JITLink/x86.h"
22#include "llvm/ExecutionEngine/JITLink/x86_64.h"
23#include "llvm/Support/raw_ostream.h"
24
25using namespace llvm;
26using namespace llvm::object;
27
28#define DEBUG_TYPE "jitlink"
29
30namespace {
31
32enum JITLinkErrorCode { GenericJITLinkError = 1 };
33
34// FIXME: This class is only here to support the transition to llvm::Error. It
35// will be removed once this transition is complete. Clients should prefer to
36// deal with the Error value directly, rather than converting to error_code.
37class JITLinkerErrorCategory : public std::error_category {
38public:
39 const char *name() const noexcept override { return "runtimedyld"; }
40
41 std::string message(int Condition) const override {
42 switch (static_cast<JITLinkErrorCode>(Condition)) {
43 case GenericJITLinkError:
44 return "Generic JITLink error";
45 }
46 llvm_unreachable("Unrecognized JITLinkErrorCode");
47 }
48};
49
50} // namespace
51
52namespace llvm {
53namespace jitlink {
54
55char JITLinkError::ID = 0;
56
57void JITLinkError::log(raw_ostream &OS) const { OS << ErrMsg; }
58
59std::error_code JITLinkError::convertToErrorCode() const {
60 static JITLinkerErrorCategory TheJITLinkerErrorCategory;
61 return std::error_code(GenericJITLinkError, TheJITLinkerErrorCategory);
62}
63
64const char *getGenericEdgeKindName(Edge::Kind K) {
65 switch (K) {
66 case Edge::Invalid:
67 return "INVALID RELOCATION";
68 case Edge::KeepAlive:
69 return "Keep-Alive";
70 default:
71 return "<Unrecognized edge kind>";
72 }
73}
74
75const char *getLinkageName(Linkage L) {
76 switch (L) {
77 case Linkage::Strong:
78 return "strong";
79 case Linkage::Weak:
80 return "weak";
81 }
82 llvm_unreachable("Unrecognized llvm.jitlink.Linkage enum");
83}
84
85const char *getScopeName(Scope S) {
86 switch (S) {
87 case Scope::Default:
88 return "default";
89 case Scope::Hidden:
90 return "hidden";
91 case Scope::SideEffectsOnly:
92 return "side-effects-only";
93 case Scope::Local:
94 return "local";
95 }
96 llvm_unreachable("Unrecognized llvm.jitlink.Scope enum");
97}
98
99bool isCStringBlock(Block &B) {
100 if (B.getSize() == 0) // Empty blocks are not valid C-strings.
101 return false;
102
103 // Zero-fill blocks of size one are valid empty strings.
104 if (B.isZeroFill())
105 return B.getSize() == 1;
106
107 for (size_t I = 0; I != B.getSize() - 1; ++I)
108 if (B.getContent()[I] == '\0')
109 return false;
110
111 return B.getContent()[B.getSize() - 1] == '\0';
112}
113
114raw_ostream &operator<<(raw_ostream &OS, const Block &B) {
115 return OS << B.getAddress() << " -- " << (B.getAddress() + B.getSize())
116 << ": "
117 << "size = " << formatv(Fmt: "{0:x8}", Vals: B.getSize()) << ", "
118 << (B.isZeroFill() ? "zero-fill" : "content")
119 << ", align = " << B.getAlignment()
120 << ", align-ofs = " << B.getAlignmentOffset()
121 << ", section = " << B.getSection().getName();
122}
123
124raw_ostream &operator<<(raw_ostream &OS, const Symbol &Sym) {
125 OS << Sym.getAddress() << " (" << (Sym.isDefined() ? "block" : "addressable")
126 << " + " << formatv(Fmt: "{0:x8}", Vals: Sym.getOffset())
127 << "): size: " << formatv(Fmt: "{0:x8}", Vals: Sym.getSize())
128 << ", linkage: " << formatv(Fmt: "{0:6}", Vals: getLinkageName(L: Sym.getLinkage()))
129 << ", scope: " << formatv(Fmt: "{0:8}", Vals: getScopeName(S: Sym.getScope())) << ", "
130 << (Sym.isLive() ? "live" : "dead") << " - "
131 << (Sym.hasName() ? *Sym.getName() : "<anonymous symbol>");
132 return OS;
133}
134
135void printEdge(raw_ostream &OS, const Block &B, const Edge &E,
136 StringRef EdgeKindName) {
137 OS << "edge@" << B.getAddress() + E.getOffset() << ": " << B.getAddress()
138 << " + " << formatv(Fmt: "{0:x}", Vals: E.getOffset()) << " -- " << EdgeKindName
139 << " -> ";
140
141 auto &TargetSym = E.getTarget();
142 if (TargetSym.hasName())
143 OS << TargetSym.getName();
144 else {
145 auto &TargetBlock = TargetSym.getBlock();
146 auto &TargetSec = TargetBlock.getSection();
147 orc::ExecutorAddr SecAddress(~uint64_t(0));
148 for (auto *B : TargetSec.blocks())
149 if (B->getAddress() < SecAddress)
150 SecAddress = B->getAddress();
151
152 orc::ExecutorAddrDiff SecDelta = TargetSym.getAddress() - SecAddress;
153 OS << TargetSym.getAddress() << " (section " << TargetSec.getName();
154 if (SecDelta)
155 OS << " + " << formatv(Fmt: "{0:x}", Vals&: SecDelta);
156 OS << " / block " << TargetBlock.getAddress();
157 if (TargetSym.getOffset())
158 OS << " + " << formatv(Fmt: "{0:x}", Vals: TargetSym.getOffset());
159 OS << ")";
160 }
161
162 if (E.getAddend() != 0)
163 OS << " + " << E.getAddend();
164}
165
166Section::~Section() {
167 for (auto *Sym : Symbols)
168 Sym->~Symbol();
169 for (auto *B : Blocks)
170 B->~Block();
171}
172
173LinkGraph::~LinkGraph() {
174 for (auto *Sym : AbsoluteSymbols) {
175 Sym->~Symbol();
176 }
177 for (auto *Sym : external_symbols()) {
178 Sym->~Symbol();
179 }
180 ExternalSymbols.clear();
181}
182
183std::vector<Block *> LinkGraph::splitBlockImpl(std::vector<Block *> Blocks,
184 SplitBlockCache *Cache) {
185 assert(!Blocks.empty() && "Blocks must at least contain the original block");
186
187 // Fix up content of all blocks.
188 ArrayRef<char> Content = Blocks.front()->getContent();
189 for (size_t I = 0; I != Blocks.size() - 1; ++I) {
190 Blocks[I]->setContent(
191 Content.slice(N: Blocks[I]->getAddress() - Blocks[0]->getAddress(),
192 M: Blocks[I + 1]->getAddress() - Blocks[I]->getAddress()));
193 }
194 Blocks.back()->setContent(
195 Content.slice(N: Blocks.back()->getAddress() - Blocks[0]->getAddress()));
196 bool IsMutable = Blocks[0]->ContentMutable;
197 for (auto *B : Blocks)
198 B->ContentMutable = IsMutable;
199
200 // Transfer symbols.
201 {
202 SplitBlockCache LocalBlockSymbolsCache;
203 if (!Cache)
204 Cache = &LocalBlockSymbolsCache;
205
206 // Build cache if required.
207 if (*Cache == std::nullopt) {
208 *Cache = SplitBlockCache::value_type();
209
210 for (auto *Sym : Blocks[0]->getSection().symbols())
211 if (&Sym->getBlock() == Blocks[0])
212 (*Cache)->push_back(Elt: Sym);
213 llvm::sort(C&: **Cache, Comp: [](const Symbol *LHS, const Symbol *RHS) {
214 return LHS->getAddress() > RHS->getAddress();
215 });
216 }
217
218 auto TransferSymbol = [](Symbol &Sym, Block &B) {
219 Sym.setOffset(Sym.getAddress() - B.getAddress());
220 Sym.setBlock(B);
221 if (Sym.getSize() > B.getSize())
222 Sym.setSize(B.getSize() - Sym.getOffset());
223 };
224
225 // Transfer symbols to all blocks except the last one.
226 for (size_t I = 0; I != Blocks.size() - 1; ++I) {
227 if ((*Cache)->empty())
228 break;
229 while (!(*Cache)->empty() &&
230 (*Cache)->back()->getAddress() < Blocks[I + 1]->getAddress()) {
231 TransferSymbol(*(*Cache)->back(), *Blocks[I]);
232 (*Cache)->pop_back();
233 }
234 }
235 // Transfer symbols to the last block, checking that all are in-range.
236 while (!(*Cache)->empty()) {
237 auto &Sym = *(*Cache)->back();
238 (*Cache)->pop_back();
239 assert(Sym.getAddress() >= Blocks.back()->getAddress() &&
240 "Symbol address preceeds block");
241 assert(Sym.getAddress() <= Blocks.back()->getRange().End &&
242 "Symbol address starts past end of block");
243 TransferSymbol(Sym, *Blocks.back());
244 }
245 }
246
247 // Transfer edges.
248 auto &Edges = Blocks[0]->Edges;
249 llvm::sort(C&: Edges, Comp: [](const Edge &LHS, const Edge &RHS) {
250 return LHS.getOffset() < RHS.getOffset();
251 });
252
253 for (size_t I = Blocks.size() - 1; I != 0; --I) {
254
255 // If all edges have been transferred then bail out.
256 if (Edges.empty())
257 break;
258
259 Edge::OffsetT Delta = Blocks[I]->getAddress() - Blocks[0]->getAddress();
260
261 // If no edges to move for this block then move to the next one.
262 if (Edges.back().getOffset() < Delta)
263 continue;
264
265 size_t EI = Edges.size() - 1;
266 while (EI != 0 && Edges[EI - 1].getOffset() >= Delta)
267 --EI;
268
269 for (size_t J = EI; J != Edges.size(); ++J) {
270 Blocks[I]->Edges.push_back(x: std::move(Edges[J]));
271 Blocks[I]->Edges.back().setOffset(Blocks[I]->Edges.back().getOffset() -
272 Delta);
273 }
274
275 while (Edges.size() > EI)
276 Edges.pop_back();
277 }
278
279 return Blocks;
280}
281
282void LinkGraph::dump(raw_ostream &OS) {
283 DenseMap<Block *, std::vector<Symbol *>> BlockSymbols;
284
285 OS << "LinkGraph \"" << getName()
286 << "\" (triple = " << getTargetTriple().str() << ")\n";
287
288 // Map from blocks to the symbols pointing at them.
289 for (auto *Sym : defined_symbols())
290 BlockSymbols[&Sym->getBlock()].push_back(x: Sym);
291
292 // For each block, sort its symbols by something approximating
293 // relevance.
294 for (auto &KV : BlockSymbols)
295 llvm::sort(C&: KV.second, Comp: [](const Symbol *LHS, const Symbol *RHS) {
296 if (LHS->getOffset() != RHS->getOffset())
297 return LHS->getOffset() < RHS->getOffset();
298 if (LHS->getLinkage() != RHS->getLinkage())
299 return LHS->getLinkage() < RHS->getLinkage();
300 if (LHS->getScope() != RHS->getScope())
301 return LHS->getScope() < RHS->getScope();
302 if (LHS->hasName()) {
303 if (!RHS->hasName())
304 return true;
305 return LHS->getName() < RHS->getName();
306 }
307 return false;
308 });
309
310 std::vector<Section *> SortedSections;
311 for (auto &Sec : sections())
312 SortedSections.push_back(x: &Sec);
313 llvm::sort(C&: SortedSections, Comp: [](const Section *LHS, const Section *RHS) {
314 return LHS->getName() < RHS->getName();
315 });
316
317 for (auto *Sec : SortedSections) {
318 OS << "section " << Sec->getName() << ":\n\n";
319
320 std::vector<Block *> SortedBlocks;
321 llvm::append_range(C&: SortedBlocks, R: Sec->blocks());
322 llvm::sort(C&: SortedBlocks, Comp: [](const Block *LHS, const Block *RHS) {
323 return LHS->getAddress() < RHS->getAddress();
324 });
325
326 for (auto *B : SortedBlocks) {
327 OS << " block " << B->getAddress()
328 << " size = " << formatv(Fmt: "{0:x8}", Vals: B->getSize())
329 << ", align = " << B->getAlignment()
330 << ", alignment-offset = " << B->getAlignmentOffset();
331 if (B->isZeroFill())
332 OS << ", zero-fill";
333 OS << "\n";
334
335 auto BlockSymsI = BlockSymbols.find(Val: B);
336 if (BlockSymsI != BlockSymbols.end()) {
337 OS << " symbols:\n";
338 auto &Syms = BlockSymsI->second;
339 for (auto *Sym : Syms)
340 OS << " " << *Sym << "\n";
341 } else
342 OS << " no symbols\n";
343
344 if (!B->edges_empty()) {
345 OS << " edges:\n";
346 std::vector<Edge> SortedEdges;
347 llvm::append_range(C&: SortedEdges, R: B->edges());
348 llvm::sort(C&: SortedEdges, Comp: [](const Edge &LHS, const Edge &RHS) {
349 return LHS.getOffset() < RHS.getOffset();
350 });
351 for (auto &E : SortedEdges) {
352 OS << " " << B->getFixupAddress(E) << " (block + "
353 << formatv(Fmt: "{0:x8}", Vals: E.getOffset()) << "), addend = ";
354 if (E.getAddend() >= 0)
355 OS << formatv(Fmt: "+{0:x8}", Vals: E.getAddend());
356 else
357 OS << formatv(Fmt: "-{0:x8}", Vals: -E.getAddend());
358 OS << ", kind = " << getEdgeKindName(K: E.getKind()) << ", target = ";
359 if (E.getTarget().hasName())
360 OS << E.getTarget().getName();
361 else
362 OS << "addressable@"
363 << formatv(Fmt: "{0:x16}", Vals: E.getTarget().getAddress()) << "+"
364 << formatv(Fmt: "{0:x8}", Vals: E.getTarget().getOffset());
365 OS << "\n";
366 }
367 } else
368 OS << " no edges\n";
369 OS << "\n";
370 }
371 }
372
373 OS << "Absolute symbols:\n";
374 if (!absolute_symbols().empty()) {
375 for (auto *Sym : absolute_symbols())
376 OS << " " << Sym->getAddress() << ": " << *Sym << "\n";
377 } else
378 OS << " none\n";
379
380 OS << "\nExternal symbols:\n";
381 if (!external_symbols().empty()) {
382 for (auto *Sym : external_symbols())
383 OS << " " << Sym->getAddress() << ": " << *Sym
384 << (Sym->isWeaklyReferenced() ? " (weakly referenced)" : "") << "\n";
385 } else
386 OS << " none\n";
387}
388
389raw_ostream &operator<<(raw_ostream &OS, const SymbolLookupFlags &LF) {
390 switch (LF) {
391 case SymbolLookupFlags::RequiredSymbol:
392 return OS << "RequiredSymbol";
393 case SymbolLookupFlags::WeaklyReferencedSymbol:
394 return OS << "WeaklyReferencedSymbol";
395 }
396 llvm_unreachable("Unrecognized lookup flags");
397}
398
399void JITLinkAsyncLookupContinuation::anchor() {}
400
401JITLinkContext::~JITLinkContext() = default;
402
403bool JITLinkContext::shouldAddDefaultTargetPasses(const Triple &TT) const {
404 return true;
405}
406
407LinkGraphPassFunction JITLinkContext::getMarkLivePass(const Triple &TT) const {
408 return LinkGraphPassFunction();
409}
410
411Error JITLinkContext::modifyPassConfig(LinkGraph &G,
412 PassConfiguration &Config) {
413 return Error::success();
414}
415
416Error markAllSymbolsLive(LinkGraph &G) {
417 for (auto *Sym : G.defined_symbols())
418 Sym->setLive(true);
419 return Error::success();
420}
421
422Error makeTargetOutOfRangeError(const LinkGraph &G, const Block &B,
423 const Edge &E) {
424 std::string ErrMsg;
425 {
426 raw_string_ostream ErrStream(ErrMsg);
427 Section &Sec = B.getSection();
428 ErrStream << "In graph " << G.getName() << ", section " << Sec.getName()
429 << ": relocation target "
430 << formatv(Fmt: "{0:x}", Vals: E.getTarget().getAddress() + E.getAddend())
431 << " (";
432 if (E.getTarget().hasName())
433 ErrStream << E.getTarget().getName();
434 else
435 ErrStream << "<anonymous symbol>";
436 if (E.getAddend()) {
437 // Target address includes non-zero added, so break down the arithmetic.
438 ErrStream << formatv(Fmt: ":{0:x}", Vals: E.getTarget().getAddress()) << " + "
439 << formatv(Fmt: "{0:x}", Vals: E.getAddend());
440 }
441 ErrStream << ") is out of range of " << G.getEdgeKindName(K: E.getKind())
442 << " fixup at address "
443 << formatv(Fmt: "{0:x}", Vals: E.getTarget().getAddress()) << " (";
444
445 Symbol *BestSymbolForBlock = nullptr;
446 for (auto *Sym : Sec.symbols())
447 if (&Sym->getBlock() == &B && Sym->hasName() && Sym->getOffset() == 0 &&
448 (!BestSymbolForBlock ||
449 Sym->getScope() < BestSymbolForBlock->getScope() ||
450 Sym->getLinkage() < BestSymbolForBlock->getLinkage()))
451 BestSymbolForBlock = Sym;
452
453 if (BestSymbolForBlock)
454 ErrStream << BestSymbolForBlock->getName() << ", ";
455 else
456 ErrStream << "<anonymous block> @ ";
457
458 ErrStream << formatv(Fmt: "{0:x}", Vals: B.getAddress()) << " + "
459 << formatv(Fmt: "{0:x}", Vals: E.getOffset()) << ")";
460 }
461 return make_error<JITLinkError>(Args: std::move(ErrMsg));
462}
463
464Error makeAlignmentError(llvm::orc::ExecutorAddr Loc, uint64_t Value, int N,
465 const Edge &E) {
466 return make_error<JITLinkError>(Args: "0x" + llvm::utohexstr(X: Loc.getValue()) +
467 " improper alignment for relocation " +
468 formatv(Fmt: "{0:d}", Vals: E.getKind()) + ": 0x" +
469 llvm::utohexstr(X: Value) +
470 " is not aligned to " + Twine(N) + " bytes");
471}
472
473AnonymousPointerCreator getAnonymousPointerCreator(const Triple &TT) {
474 switch (TT.getArch()) {
475 case Triple::aarch64:
476 return aarch64::createAnonymousPointer;
477 case Triple::x86_64:
478 return x86_64::createAnonymousPointer;
479 case Triple::x86:
480 return x86::createAnonymousPointer;
481 case Triple::loongarch32:
482 case Triple::loongarch64:
483 return loongarch::createAnonymousPointer;
484 case Triple::systemz:
485 return systemz::createAnonymousPointer;
486 case Triple::ppc64:
487 case Triple::ppc64le:
488 return ppc64::createAnonymousPointer;
489 default:
490 return nullptr;
491 }
492}
493
494PointerJumpStubCreator getPointerJumpStubCreator(const Triple &TT) {
495 switch (TT.getArch()) {
496 case Triple::aarch64:
497 return aarch64::createAnonymousPointerJumpStub;
498 case Triple::x86_64:
499 return x86_64::createAnonymousPointerJumpStub;
500 case Triple::x86:
501 return x86::createAnonymousPointerJumpStub;
502 case Triple::loongarch32:
503 case Triple::loongarch64:
504 return loongarch::createAnonymousPointerJumpStub;
505 case Triple::systemz:
506 return systemz::createAnonymousPointerJumpStub;
507 case Triple::ppc64:
508 return ppc64::createDefaultAnonymousPointerJumpStub<llvm::endianness::big>;
509 case Triple::ppc64le:
510 return ppc64::createDefaultAnonymousPointerJumpStub<
511 llvm::endianness::little>;
512 default:
513 return nullptr;
514 }
515}
516
517Expected<std::unique_ptr<LinkGraph>>
518createLinkGraphFromObject(MemoryBufferRef ObjectBuffer,
519 std::shared_ptr<orc::SymbolStringPool> SSP) {
520 auto Magic = identify_magic(magic: ObjectBuffer.getBuffer());
521 switch (Magic) {
522 case file_magic::macho_object:
523 return createLinkGraphFromMachOObject(ObjectBuffer, SSP: std::move(SSP));
524 case file_magic::elf_relocatable:
525 return createLinkGraphFromELFObject(ObjectBuffer, SSP: std::move(SSP));
526 case file_magic::coff_object:
527 return createLinkGraphFromCOFFObject(ObjectBuffer, SSP: std::move(SSP));
528 case file_magic::xcoff_object_64:
529 return createLinkGraphFromXCOFFObject(ObjectBuffer, SSP: std::move(SSP));
530 default:
531 return make_error<JITLinkError>(Args: "Unsupported file format");
532 };
533}
534
535std::unique_ptr<LinkGraph>
536absoluteSymbolsLinkGraph(Triple TT, std::shared_ptr<orc::SymbolStringPool> SSP,
537 orc::SymbolMap Symbols) {
538 static std::atomic<uint64_t> Counter = {0};
539 auto Index = Counter.fetch_add(i: 1, m: std::memory_order_relaxed);
540 auto G = std::make_unique<LinkGraph>(
541 args: "<Absolute Symbols " + std::to_string(val: Index) + ">", args: std::move(SSP),
542 args: std::move(TT), args: SubtargetFeatures(), args&: getGenericEdgeKindName);
543 for (auto &[Name, Def] : Symbols) {
544 auto &Sym =
545 G->addAbsoluteSymbol(Name: *Name, Address: Def.getAddress(), /*Size=*/0,
546 L: Linkage::Strong, S: Scope::Default, /*IsLive=*/true);
547 Sym.setCallable(Def.getFlags().isCallable());
548 }
549
550 return G;
551}
552
553void link(std::unique_ptr<LinkGraph> G, std::unique_ptr<JITLinkContext> Ctx) {
554 switch (G->getTargetTriple().getObjectFormat()) {
555 case Triple::MachO:
556 return link_MachO(G: std::move(G), Ctx: std::move(Ctx));
557 case Triple::ELF:
558 return link_ELF(G: std::move(G), Ctx: std::move(Ctx));
559 case Triple::COFF:
560 return link_COFF(G: std::move(G), Ctx: std::move(Ctx));
561 case Triple::XCOFF:
562 return link_XCOFF(G: std::move(G), Ctx: std::move(Ctx));
563 default:
564 Ctx->notifyFailed(Err: make_error<JITLinkError>(Args: "Unsupported object format"));
565 };
566}
567
568} // end namespace jitlink
569} // end namespace llvm
570