1//===------------- JITLink.cpp - Core Run-time JIT linker APIs ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/JITLink/JITLink.h"
10
11#include "llvm/ADT/StringExtras.h"
12#include "llvm/BinaryFormat/Magic.h"
13#include "llvm/ExecutionEngine/JITLink/COFF.h"
14#include "llvm/ExecutionEngine/JITLink/ELF.h"
15#include "llvm/ExecutionEngine/JITLink/MachO.h"
16#include "llvm/ExecutionEngine/JITLink/XCOFF.h"
17#include "llvm/ExecutionEngine/JITLink/aarch64.h"
18#include "llvm/ExecutionEngine/JITLink/loongarch.h"
19#include "llvm/ExecutionEngine/JITLink/systemz.h"
20#include "llvm/ExecutionEngine/JITLink/x86.h"
21#include "llvm/ExecutionEngine/JITLink/x86_64.h"
22#include "llvm/Support/raw_ostream.h"
23
24using namespace llvm;
25using namespace llvm::object;
26
27#define DEBUG_TYPE "jitlink"
28
29namespace {
30
31enum JITLinkErrorCode { GenericJITLinkError = 1 };
32
33// FIXME: This class is only here to support the transition to llvm::Error. It
34// will be removed once this transition is complete. Clients should prefer to
35// deal with the Error value directly, rather than converting to error_code.
36class JITLinkerErrorCategory : public std::error_category {
37public:
38 const char *name() const noexcept override { return "runtimedyld"; }
39
40 std::string message(int Condition) const override {
41 switch (static_cast<JITLinkErrorCode>(Condition)) {
42 case GenericJITLinkError:
43 return "Generic JITLink error";
44 }
45 llvm_unreachable("Unrecognized JITLinkErrorCode");
46 }
47};
48
49} // namespace
50
51namespace llvm {
52namespace jitlink {
53
54char JITLinkError::ID = 0;
55
56void JITLinkError::log(raw_ostream &OS) const { OS << ErrMsg; }
57
58std::error_code JITLinkError::convertToErrorCode() const {
59 static JITLinkerErrorCategory TheJITLinkerErrorCategory;
60 return std::error_code(GenericJITLinkError, TheJITLinkerErrorCategory);
61}
62
63const char *getGenericEdgeKindName(Edge::Kind K) {
64 switch (K) {
65 case Edge::Invalid:
66 return "INVALID RELOCATION";
67 case Edge::KeepAlive:
68 return "Keep-Alive";
69 default:
70 return "<Unrecognized edge kind>";
71 }
72}
73
74const char *getLinkageName(Linkage L) {
75 switch (L) {
76 case Linkage::Strong:
77 return "strong";
78 case Linkage::Weak:
79 return "weak";
80 }
81 llvm_unreachable("Unrecognized llvm.jitlink.Linkage enum");
82}
83
84const char *getScopeName(Scope S) {
85 switch (S) {
86 case Scope::Default:
87 return "default";
88 case Scope::Hidden:
89 return "hidden";
90 case Scope::SideEffectsOnly:
91 return "side-effects-only";
92 case Scope::Local:
93 return "local";
94 }
95 llvm_unreachable("Unrecognized llvm.jitlink.Scope enum");
96}
97
98bool isCStringBlock(Block &B) {
99 if (B.getSize() == 0) // Empty blocks are not valid C-strings.
100 return false;
101
102 // Zero-fill blocks of size one are valid empty strings.
103 if (B.isZeroFill())
104 return B.getSize() == 1;
105
106 for (size_t I = 0; I != B.getSize() - 1; ++I)
107 if (B.getContent()[I] == '\0')
108 return false;
109
110 return B.getContent()[B.getSize() - 1] == '\0';
111}
112
113raw_ostream &operator<<(raw_ostream &OS, const Block &B) {
114 return OS << B.getAddress() << " -- " << (B.getAddress() + B.getSize())
115 << ": "
116 << "size = " << formatv(Fmt: "{0:x8}", Vals: B.getSize()) << ", "
117 << (B.isZeroFill() ? "zero-fill" : "content")
118 << ", align = " << B.getAlignment()
119 << ", align-ofs = " << B.getAlignmentOffset()
120 << ", section = " << B.getSection().getName();
121}
122
123raw_ostream &operator<<(raw_ostream &OS, const Symbol &Sym) {
124 OS << Sym.getAddress() << " (" << (Sym.isDefined() ? "block" : "addressable")
125 << " + " << formatv(Fmt: "{0:x8}", Vals: Sym.getOffset())
126 << "): size: " << formatv(Fmt: "{0:x8}", Vals: Sym.getSize())
127 << ", linkage: " << formatv(Fmt: "{0:6}", Vals: getLinkageName(L: Sym.getLinkage()))
128 << ", scope: " << formatv(Fmt: "{0:8}", Vals: getScopeName(S: Sym.getScope())) << ", "
129 << (Sym.isLive() ? "live" : "dead") << " - "
130 << (Sym.hasName() ? *Sym.getName() : "<anonymous symbol>");
131 return OS;
132}
133
134void printEdge(raw_ostream &OS, const Block &B, const Edge &E,
135 StringRef EdgeKindName) {
136 OS << "edge@" << B.getAddress() + E.getOffset() << ": " << B.getAddress()
137 << " + " << formatv(Fmt: "{0:x}", Vals: E.getOffset()) << " -- " << EdgeKindName
138 << " -> ";
139
140 auto &TargetSym = E.getTarget();
141 if (TargetSym.hasName())
142 OS << TargetSym.getName();
143 else {
144 auto &TargetBlock = TargetSym.getBlock();
145 auto &TargetSec = TargetBlock.getSection();
146 orc::ExecutorAddr SecAddress(~uint64_t(0));
147 for (auto *B : TargetSec.blocks())
148 if (B->getAddress() < SecAddress)
149 SecAddress = B->getAddress();
150
151 orc::ExecutorAddrDiff SecDelta = TargetSym.getAddress() - SecAddress;
152 OS << TargetSym.getAddress() << " (section " << TargetSec.getName();
153 if (SecDelta)
154 OS << " + " << formatv(Fmt: "{0:x}", Vals&: SecDelta);
155 OS << " / block " << TargetBlock.getAddress();
156 if (TargetSym.getOffset())
157 OS << " + " << formatv(Fmt: "{0:x}", Vals: TargetSym.getOffset());
158 OS << ")";
159 }
160
161 if (E.getAddend() != 0)
162 OS << " + " << E.getAddend();
163}
164
165Section::~Section() {
166 for (auto *Sym : Symbols)
167 Sym->~Symbol();
168 for (auto *B : Blocks)
169 B->~Block();
170}
171
172LinkGraph::~LinkGraph() {
173 for (auto *Sym : AbsoluteSymbols) {
174 Sym->~Symbol();
175 }
176 for (auto *Sym : external_symbols()) {
177 Sym->~Symbol();
178 }
179 ExternalSymbols.clear();
180}
181
182std::vector<Block *> LinkGraph::splitBlockImpl(std::vector<Block *> Blocks,
183 SplitBlockCache *Cache) {
184 assert(!Blocks.empty() && "Blocks must at least contain the original block");
185
186 // Fix up content of all blocks.
187 ArrayRef<char> Content = Blocks.front()->getContent();
188 for (size_t I = 0; I != Blocks.size() - 1; ++I) {
189 Blocks[I]->setContent(
190 Content.slice(N: Blocks[I]->getAddress() - Blocks[0]->getAddress(),
191 M: Blocks[I + 1]->getAddress() - Blocks[I]->getAddress()));
192 }
193 Blocks.back()->setContent(
194 Content.slice(N: Blocks.back()->getAddress() - Blocks[0]->getAddress()));
195 bool IsMutable = Blocks[0]->ContentMutable;
196 for (auto *B : Blocks)
197 B->ContentMutable = IsMutable;
198
199 // Transfer symbols.
200 {
201 SplitBlockCache LocalBlockSymbolsCache;
202 if (!Cache)
203 Cache = &LocalBlockSymbolsCache;
204
205 // Build cache if required.
206 if (*Cache == std::nullopt) {
207 *Cache = SplitBlockCache::value_type();
208
209 for (auto *Sym : Blocks[0]->getSection().symbols())
210 if (&Sym->getBlock() == Blocks[0])
211 (*Cache)->push_back(Elt: Sym);
212 llvm::sort(C&: **Cache, Comp: [](const Symbol *LHS, const Symbol *RHS) {
213 return LHS->getAddress() > RHS->getAddress();
214 });
215 }
216
217 auto TransferSymbol = [](Symbol &Sym, Block &B) {
218 Sym.setOffset(Sym.getAddress() - B.getAddress());
219 Sym.setBlock(B);
220 if (Sym.getSize() > B.getSize())
221 Sym.setSize(B.getSize() - Sym.getOffset());
222 };
223
224 // Transfer symbols to all blocks except the last one.
225 for (size_t I = 0; I != Blocks.size() - 1; ++I) {
226 if ((*Cache)->empty())
227 break;
228 while (!(*Cache)->empty() &&
229 (*Cache)->back()->getAddress() < Blocks[I + 1]->getAddress()) {
230 TransferSymbol(*(*Cache)->back(), *Blocks[I]);
231 (*Cache)->pop_back();
232 }
233 }
234 // Transfer symbols to the last block, checking that all are in-range.
235 while (!(*Cache)->empty()) {
236 auto &Sym = *(*Cache)->back();
237 (*Cache)->pop_back();
238 assert(Sym.getAddress() >= Blocks.back()->getAddress() &&
239 "Symbol address preceeds block");
240 assert(Sym.getAddress() <= Blocks.back()->getRange().End &&
241 "Symbol address starts past end of block");
242 TransferSymbol(Sym, *Blocks.back());
243 }
244 }
245
246 // Transfer edges.
247 auto &Edges = Blocks[0]->Edges;
248 llvm::sort(C&: Edges, Comp: [](const Edge &LHS, const Edge &RHS) {
249 return LHS.getOffset() < RHS.getOffset();
250 });
251
252 for (size_t I = Blocks.size() - 1; I != 0; --I) {
253
254 // If all edges have been transferred then bail out.
255 if (Edges.empty())
256 break;
257
258 Edge::OffsetT Delta = Blocks[I]->getAddress() - Blocks[0]->getAddress();
259
260 // If no edges to move for this block then move to the next one.
261 if (Edges.back().getOffset() < Delta)
262 continue;
263
264 size_t EI = Edges.size() - 1;
265 while (EI != 0 && Edges[EI - 1].getOffset() >= Delta)
266 --EI;
267
268 for (size_t J = EI; J != Edges.size(); ++J) {
269 Blocks[I]->Edges.push_back(x: std::move(Edges[J]));
270 Blocks[I]->Edges.back().setOffset(Blocks[I]->Edges.back().getOffset() -
271 Delta);
272 }
273
274 while (Edges.size() > EI)
275 Edges.pop_back();
276 }
277
278 return Blocks;
279}
280
281void LinkGraph::dump(raw_ostream &OS) {
282 DenseMap<Block *, std::vector<Symbol *>> BlockSymbols;
283
284 OS << "LinkGraph \"" << getName()
285 << "\" (triple = " << getTargetTriple().str() << ")\n";
286
287 // Map from blocks to the symbols pointing at them.
288 for (auto *Sym : defined_symbols())
289 BlockSymbols[&Sym->getBlock()].push_back(x: Sym);
290
291 // For each block, sort its symbols by something approximating
292 // relevance.
293 for (auto &KV : BlockSymbols)
294 llvm::sort(C&: KV.second, Comp: [](const Symbol *LHS, const Symbol *RHS) {
295 if (LHS->getOffset() != RHS->getOffset())
296 return LHS->getOffset() < RHS->getOffset();
297 if (LHS->getLinkage() != RHS->getLinkage())
298 return LHS->getLinkage() < RHS->getLinkage();
299 if (LHS->getScope() != RHS->getScope())
300 return LHS->getScope() < RHS->getScope();
301 if (LHS->hasName()) {
302 if (!RHS->hasName())
303 return true;
304 return LHS->getName() < RHS->getName();
305 }
306 return false;
307 });
308
309 std::vector<Section *> SortedSections;
310 for (auto &Sec : sections())
311 SortedSections.push_back(x: &Sec);
312 llvm::sort(C&: SortedSections, Comp: [](const Section *LHS, const Section *RHS) {
313 return LHS->getName() < RHS->getName();
314 });
315
316 for (auto *Sec : SortedSections) {
317 OS << "section " << Sec->getName() << ":\n\n";
318
319 std::vector<Block *> SortedBlocks;
320 llvm::append_range(C&: SortedBlocks, R: Sec->blocks());
321 llvm::sort(C&: SortedBlocks, Comp: [](const Block *LHS, const Block *RHS) {
322 return LHS->getAddress() < RHS->getAddress();
323 });
324
325 for (auto *B : SortedBlocks) {
326 OS << " block " << B->getAddress()
327 << " size = " << formatv(Fmt: "{0:x8}", Vals: B->getSize())
328 << ", align = " << B->getAlignment()
329 << ", alignment-offset = " << B->getAlignmentOffset();
330 if (B->isZeroFill())
331 OS << ", zero-fill";
332 OS << "\n";
333
334 auto BlockSymsI = BlockSymbols.find(Val: B);
335 if (BlockSymsI != BlockSymbols.end()) {
336 OS << " symbols:\n";
337 auto &Syms = BlockSymsI->second;
338 for (auto *Sym : Syms)
339 OS << " " << *Sym << "\n";
340 } else
341 OS << " no symbols\n";
342
343 if (!B->edges_empty()) {
344 OS << " edges:\n";
345 std::vector<Edge> SortedEdges;
346 llvm::append_range(C&: SortedEdges, R: B->edges());
347 llvm::sort(C&: SortedEdges, Comp: [](const Edge &LHS, const Edge &RHS) {
348 return LHS.getOffset() < RHS.getOffset();
349 });
350 for (auto &E : SortedEdges) {
351 OS << " " << B->getFixupAddress(E) << " (block + "
352 << formatv(Fmt: "{0:x8}", Vals: E.getOffset()) << "), addend = ";
353 if (E.getAddend() >= 0)
354 OS << formatv(Fmt: "+{0:x8}", Vals: E.getAddend());
355 else
356 OS << formatv(Fmt: "-{0:x8}", Vals: -E.getAddend());
357 OS << ", kind = " << getEdgeKindName(K: E.getKind()) << ", target = ";
358 if (E.getTarget().hasName())
359 OS << E.getTarget().getName();
360 else
361 OS << "addressable@"
362 << formatv(Fmt: "{0:x16}", Vals: E.getTarget().getAddress()) << "+"
363 << formatv(Fmt: "{0:x8}", Vals: E.getTarget().getOffset());
364 OS << "\n";
365 }
366 } else
367 OS << " no edges\n";
368 OS << "\n";
369 }
370 }
371
372 OS << "Absolute symbols:\n";
373 if (!absolute_symbols().empty()) {
374 for (auto *Sym : absolute_symbols())
375 OS << " " << Sym->getAddress() << ": " << *Sym << "\n";
376 } else
377 OS << " none\n";
378
379 OS << "\nExternal symbols:\n";
380 if (!external_symbols().empty()) {
381 for (auto *Sym : external_symbols())
382 OS << " " << Sym->getAddress() << ": " << *Sym
383 << (Sym->isWeaklyReferenced() ? " (weakly referenced)" : "") << "\n";
384 } else
385 OS << " none\n";
386}
387
388raw_ostream &operator<<(raw_ostream &OS, const SymbolLookupFlags &LF) {
389 switch (LF) {
390 case SymbolLookupFlags::RequiredSymbol:
391 return OS << "RequiredSymbol";
392 case SymbolLookupFlags::WeaklyReferencedSymbol:
393 return OS << "WeaklyReferencedSymbol";
394 }
395 llvm_unreachable("Unrecognized lookup flags");
396}
397
398void JITLinkAsyncLookupContinuation::anchor() {}
399
400JITLinkContext::~JITLinkContext() = default;
401
402bool JITLinkContext::shouldAddDefaultTargetPasses(const Triple &TT) const {
403 return true;
404}
405
406LinkGraphPassFunction JITLinkContext::getMarkLivePass(const Triple &TT) const {
407 return LinkGraphPassFunction();
408}
409
410Error JITLinkContext::modifyPassConfig(LinkGraph &G,
411 PassConfiguration &Config) {
412 return Error::success();
413}
414
415Error markAllSymbolsLive(LinkGraph &G) {
416 for (auto *Sym : G.defined_symbols())
417 Sym->setLive(true);
418 return Error::success();
419}
420
421Error makeTargetOutOfRangeError(const LinkGraph &G, const Block &B,
422 const Edge &E) {
423 std::string ErrMsg;
424 {
425 raw_string_ostream ErrStream(ErrMsg);
426 Section &Sec = B.getSection();
427 ErrStream << "In graph " << G.getName() << ", section " << Sec.getName()
428 << ": relocation target "
429 << formatv(Fmt: "{0:x}", Vals: E.getTarget().getAddress() + E.getAddend())
430 << " (";
431 if (E.getTarget().hasName())
432 ErrStream << E.getTarget().getName();
433 else
434 ErrStream << "<anonymous symbol>";
435 if (E.getAddend()) {
436 // Target address includes non-zero added, so break down the arithmetic.
437 ErrStream << formatv(Fmt: ":{0:x}", Vals: E.getTarget().getAddress()) << " + "
438 << formatv(Fmt: "{0:x}", Vals: E.getAddend());
439 }
440 ErrStream << ") is out of range of " << G.getEdgeKindName(K: E.getKind())
441 << " fixup at address "
442 << formatv(Fmt: "{0:x}", Vals: E.getTarget().getAddress()) << " (";
443
444 Symbol *BestSymbolForBlock = nullptr;
445 for (auto *Sym : Sec.symbols())
446 if (&Sym->getBlock() == &B && Sym->hasName() && Sym->getOffset() == 0 &&
447 (!BestSymbolForBlock ||
448 Sym->getScope() < BestSymbolForBlock->getScope() ||
449 Sym->getLinkage() < BestSymbolForBlock->getLinkage()))
450 BestSymbolForBlock = Sym;
451
452 if (BestSymbolForBlock)
453 ErrStream << BestSymbolForBlock->getName() << ", ";
454 else
455 ErrStream << "<anonymous block> @ ";
456
457 ErrStream << formatv(Fmt: "{0:x}", Vals: B.getAddress()) << " + "
458 << formatv(Fmt: "{0:x}", Vals: E.getOffset()) << ")";
459 }
460 return make_error<JITLinkError>(Args: std::move(ErrMsg));
461}
462
463Error makeAlignmentError(llvm::orc::ExecutorAddr Loc, uint64_t Value, int N,
464 const Edge &E) {
465 return make_error<JITLinkError>(Args: "0x" + llvm::utohexstr(X: Loc.getValue()) +
466 " improper alignment for relocation " +
467 formatv(Fmt: "{0:d}", Vals: E.getKind()) + ": 0x" +
468 llvm::utohexstr(X: Value) +
469 " is not aligned to " + Twine(N) + " bytes");
470}
471
472AnonymousPointerCreator getAnonymousPointerCreator(const Triple &TT) {
473 switch (TT.getArch()) {
474 case Triple::aarch64:
475 return aarch64::createAnonymousPointer;
476 case Triple::x86_64:
477 return x86_64::createAnonymousPointer;
478 case Triple::x86:
479 return x86::createAnonymousPointer;
480 case Triple::loongarch32:
481 case Triple::loongarch64:
482 return loongarch::createAnonymousPointer;
483 case Triple::systemz:
484 return systemz::createAnonymousPointer;
485 default:
486 return nullptr;
487 }
488}
489
490PointerJumpStubCreator getPointerJumpStubCreator(const Triple &TT) {
491 switch (TT.getArch()) {
492 case Triple::aarch64:
493 return aarch64::createAnonymousPointerJumpStub;
494 case Triple::x86_64:
495 return x86_64::createAnonymousPointerJumpStub;
496 case Triple::x86:
497 return x86::createAnonymousPointerJumpStub;
498 case Triple::loongarch32:
499 case Triple::loongarch64:
500 return loongarch::createAnonymousPointerJumpStub;
501 case Triple::systemz:
502 return systemz::createAnonymousPointerJumpStub;
503 default:
504 return nullptr;
505 }
506}
507
508Expected<std::unique_ptr<LinkGraph>>
509createLinkGraphFromObject(MemoryBufferRef ObjectBuffer,
510 std::shared_ptr<orc::SymbolStringPool> SSP) {
511 auto Magic = identify_magic(magic: ObjectBuffer.getBuffer());
512 switch (Magic) {
513 case file_magic::macho_object:
514 return createLinkGraphFromMachOObject(ObjectBuffer, SSP: std::move(SSP));
515 case file_magic::elf_relocatable:
516 return createLinkGraphFromELFObject(ObjectBuffer, SSP: std::move(SSP));
517 case file_magic::coff_object:
518 return createLinkGraphFromCOFFObject(ObjectBuffer, SSP: std::move(SSP));
519 case file_magic::xcoff_object_64:
520 return createLinkGraphFromXCOFFObject(ObjectBuffer, SSP: std::move(SSP));
521 default:
522 return make_error<JITLinkError>(Args: "Unsupported file format");
523 };
524}
525
526std::unique_ptr<LinkGraph>
527absoluteSymbolsLinkGraph(Triple TT, std::shared_ptr<orc::SymbolStringPool> SSP,
528 orc::SymbolMap Symbols) {
529 static std::atomic<uint64_t> Counter = {0};
530 auto Index = Counter.fetch_add(i: 1, m: std::memory_order_relaxed);
531 auto G = std::make_unique<LinkGraph>(
532 args: "<Absolute Symbols " + std::to_string(val: Index) + ">", args: std::move(SSP),
533 args: std::move(TT), args: SubtargetFeatures(), args&: getGenericEdgeKindName);
534 for (auto &[Name, Def] : Symbols) {
535 auto &Sym =
536 G->addAbsoluteSymbol(Name: *Name, Address: Def.getAddress(), /*Size=*/0,
537 L: Linkage::Strong, S: Scope::Default, /*IsLive=*/true);
538 Sym.setCallable(Def.getFlags().isCallable());
539 }
540
541 return G;
542}
543
544void link(std::unique_ptr<LinkGraph> G, std::unique_ptr<JITLinkContext> Ctx) {
545 switch (G->getTargetTriple().getObjectFormat()) {
546 case Triple::MachO:
547 return link_MachO(G: std::move(G), Ctx: std::move(Ctx));
548 case Triple::ELF:
549 return link_ELF(G: std::move(G), Ctx: std::move(Ctx));
550 case Triple::COFF:
551 return link_COFF(G: std::move(G), Ctx: std::move(Ctx));
552 case Triple::XCOFF:
553 return link_XCOFF(G: std::move(G), Ctx: std::move(Ctx));
554 default:
555 Ctx->notifyFailed(Err: make_error<JITLinkError>(Args: "Unsupported object format"));
556 };
557}
558
559} // end namespace jitlink
560} // end namespace llvm
561