1//===------------- JITLink.cpp - Core Run-time JIT linker APIs ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/JITLink/JITLink.h"
10
11#include "llvm/ADT/StringExtras.h"
12#include "llvm/BinaryFormat/Magic.h"
13#include "llvm/ExecutionEngine/JITLink/COFF.h"
14#include "llvm/ExecutionEngine/JITLink/ELF.h"
15#include "llvm/ExecutionEngine/JITLink/MachO.h"
16#include "llvm/ExecutionEngine/JITLink/XCOFF.h"
17#include "llvm/ExecutionEngine/JITLink/aarch64.h"
18#include "llvm/ExecutionEngine/JITLink/loongarch.h"
19#include "llvm/ExecutionEngine/JITLink/x86.h"
20#include "llvm/ExecutionEngine/JITLink/x86_64.h"
21#include "llvm/Support/raw_ostream.h"
22
23using namespace llvm;
24using namespace llvm::object;
25
26#define DEBUG_TYPE "jitlink"
27
28namespace {
29
30enum JITLinkErrorCode { GenericJITLinkError = 1 };
31
32// FIXME: This class is only here to support the transition to llvm::Error. It
33// will be removed once this transition is complete. Clients should prefer to
34// deal with the Error value directly, rather than converting to error_code.
35class JITLinkerErrorCategory : public std::error_category {
36public:
37 const char *name() const noexcept override { return "runtimedyld"; }
38
39 std::string message(int Condition) const override {
40 switch (static_cast<JITLinkErrorCode>(Condition)) {
41 case GenericJITLinkError:
42 return "Generic JITLink error";
43 }
44 llvm_unreachable("Unrecognized JITLinkErrorCode");
45 }
46};
47
48} // namespace
49
50namespace llvm {
51namespace jitlink {
52
53char JITLinkError::ID = 0;
54
55void JITLinkError::log(raw_ostream &OS) const { OS << ErrMsg; }
56
57std::error_code JITLinkError::convertToErrorCode() const {
58 static JITLinkerErrorCategory TheJITLinkerErrorCategory;
59 return std::error_code(GenericJITLinkError, TheJITLinkerErrorCategory);
60}
61
62const char *getGenericEdgeKindName(Edge::Kind K) {
63 switch (K) {
64 case Edge::Invalid:
65 return "INVALID RELOCATION";
66 case Edge::KeepAlive:
67 return "Keep-Alive";
68 default:
69 return "<Unrecognized edge kind>";
70 }
71}
72
73const char *getLinkageName(Linkage L) {
74 switch (L) {
75 case Linkage::Strong:
76 return "strong";
77 case Linkage::Weak:
78 return "weak";
79 }
80 llvm_unreachable("Unrecognized llvm.jitlink.Linkage enum");
81}
82
83const char *getScopeName(Scope S) {
84 switch (S) {
85 case Scope::Default:
86 return "default";
87 case Scope::Hidden:
88 return "hidden";
89 case Scope::SideEffectsOnly:
90 return "side-effects-only";
91 case Scope::Local:
92 return "local";
93 }
94 llvm_unreachable("Unrecognized llvm.jitlink.Scope enum");
95}
96
97bool isCStringBlock(Block &B) {
98 if (B.getSize() == 0) // Empty blocks are not valid C-strings.
99 return false;
100
101 // Zero-fill blocks of size one are valid empty strings.
102 if (B.isZeroFill())
103 return B.getSize() == 1;
104
105 for (size_t I = 0; I != B.getSize() - 1; ++I)
106 if (B.getContent()[I] == '\0')
107 return false;
108
109 return B.getContent()[B.getSize() - 1] == '\0';
110}
111
112raw_ostream &operator<<(raw_ostream &OS, const Block &B) {
113 return OS << B.getAddress() << " -- " << (B.getAddress() + B.getSize())
114 << ": "
115 << "size = " << formatv(Fmt: "{0:x8}", Vals: B.getSize()) << ", "
116 << (B.isZeroFill() ? "zero-fill" : "content")
117 << ", align = " << B.getAlignment()
118 << ", align-ofs = " << B.getAlignmentOffset()
119 << ", section = " << B.getSection().getName();
120}
121
122raw_ostream &operator<<(raw_ostream &OS, const Symbol &Sym) {
123 OS << Sym.getAddress() << " (" << (Sym.isDefined() ? "block" : "addressable")
124 << " + " << formatv(Fmt: "{0:x8}", Vals: Sym.getOffset())
125 << "): size: " << formatv(Fmt: "{0:x8}", Vals: Sym.getSize())
126 << ", linkage: " << formatv(Fmt: "{0:6}", Vals: getLinkageName(L: Sym.getLinkage()))
127 << ", scope: " << formatv(Fmt: "{0:8}", Vals: getScopeName(S: Sym.getScope())) << ", "
128 << (Sym.isLive() ? "live" : "dead") << " - "
129 << (Sym.hasName() ? *Sym.getName() : "<anonymous symbol>");
130 return OS;
131}
132
133void printEdge(raw_ostream &OS, const Block &B, const Edge &E,
134 StringRef EdgeKindName) {
135 OS << "edge@" << B.getAddress() + E.getOffset() << ": " << B.getAddress()
136 << " + " << formatv(Fmt: "{0:x}", Vals: E.getOffset()) << " -- " << EdgeKindName
137 << " -> ";
138
139 auto &TargetSym = E.getTarget();
140 if (TargetSym.hasName())
141 OS << TargetSym.getName();
142 else {
143 auto &TargetBlock = TargetSym.getBlock();
144 auto &TargetSec = TargetBlock.getSection();
145 orc::ExecutorAddr SecAddress(~uint64_t(0));
146 for (auto *B : TargetSec.blocks())
147 if (B->getAddress() < SecAddress)
148 SecAddress = B->getAddress();
149
150 orc::ExecutorAddrDiff SecDelta = TargetSym.getAddress() - SecAddress;
151 OS << TargetSym.getAddress() << " (section " << TargetSec.getName();
152 if (SecDelta)
153 OS << " + " << formatv(Fmt: "{0:x}", Vals&: SecDelta);
154 OS << " / block " << TargetBlock.getAddress();
155 if (TargetSym.getOffset())
156 OS << " + " << formatv(Fmt: "{0:x}", Vals: TargetSym.getOffset());
157 OS << ")";
158 }
159
160 if (E.getAddend() != 0)
161 OS << " + " << E.getAddend();
162}
163
164Section::~Section() {
165 for (auto *Sym : Symbols)
166 Sym->~Symbol();
167 for (auto *B : Blocks)
168 B->~Block();
169}
170
171LinkGraph::~LinkGraph() {
172 for (auto *Sym : AbsoluteSymbols) {
173 Sym->~Symbol();
174 }
175 for (auto *Sym : external_symbols()) {
176 Sym->~Symbol();
177 }
178 ExternalSymbols.clear();
179}
180
181std::vector<Block *> LinkGraph::splitBlockImpl(std::vector<Block *> Blocks,
182 SplitBlockCache *Cache) {
183 assert(!Blocks.empty() && "Blocks must at least contain the original block");
184
185 // Fix up content of all blocks.
186 ArrayRef<char> Content = Blocks.front()->getContent();
187 for (size_t I = 0; I != Blocks.size() - 1; ++I) {
188 Blocks[I]->setContent(
189 Content.slice(N: Blocks[I]->getAddress() - Blocks[0]->getAddress(),
190 M: Blocks[I + 1]->getAddress() - Blocks[I]->getAddress()));
191 }
192 Blocks.back()->setContent(
193 Content.slice(N: Blocks.back()->getAddress() - Blocks[0]->getAddress()));
194 bool IsMutable = Blocks[0]->ContentMutable;
195 for (auto *B : Blocks)
196 B->ContentMutable = IsMutable;
197
198 // Transfer symbols.
199 {
200 SplitBlockCache LocalBlockSymbolsCache;
201 if (!Cache)
202 Cache = &LocalBlockSymbolsCache;
203
204 // Build cache if required.
205 if (*Cache == std::nullopt) {
206 *Cache = SplitBlockCache::value_type();
207
208 for (auto *Sym : Blocks[0]->getSection().symbols())
209 if (&Sym->getBlock() == Blocks[0])
210 (*Cache)->push_back(Elt: Sym);
211 llvm::sort(C&: **Cache, Comp: [](const Symbol *LHS, const Symbol *RHS) {
212 return LHS->getAddress() > RHS->getAddress();
213 });
214 }
215
216 auto TransferSymbol = [](Symbol &Sym, Block &B) {
217 Sym.setOffset(Sym.getAddress() - B.getAddress());
218 Sym.setBlock(B);
219 if (Sym.getSize() > B.getSize())
220 Sym.setSize(B.getSize() - Sym.getOffset());
221 };
222
223 // Transfer symbols to all blocks except the last one.
224 for (size_t I = 0; I != Blocks.size() - 1; ++I) {
225 if ((*Cache)->empty())
226 break;
227 while (!(*Cache)->empty() &&
228 (*Cache)->back()->getAddress() < Blocks[I + 1]->getAddress()) {
229 TransferSymbol(*(*Cache)->back(), *Blocks[I]);
230 (*Cache)->pop_back();
231 }
232 }
233 // Transfer symbols to the last block, checking that all are in-range.
234 while (!(*Cache)->empty()) {
235 auto &Sym = *(*Cache)->back();
236 (*Cache)->pop_back();
237 assert(Sym.getAddress() >= Blocks.back()->getAddress() &&
238 "Symbol address preceeds block");
239 assert(Sym.getAddress() <= Blocks.back()->getRange().End &&
240 "Symbol address starts past end of block");
241 TransferSymbol(Sym, *Blocks.back());
242 }
243 }
244
245 // Transfer edges.
246 auto &Edges = Blocks[0]->Edges;
247 llvm::sort(C&: Edges, Comp: [](const Edge &LHS, const Edge &RHS) {
248 return LHS.getOffset() < RHS.getOffset();
249 });
250
251 for (size_t I = Blocks.size() - 1; I != 0; --I) {
252
253 // If all edges have been transferred then bail out.
254 if (Edges.empty())
255 break;
256
257 Edge::OffsetT Delta = Blocks[I]->getAddress() - Blocks[0]->getAddress();
258
259 // If no edges to move for this block then move to the next one.
260 if (Edges.back().getOffset() < Delta)
261 continue;
262
263 size_t EI = Edges.size() - 1;
264 while (EI != 0 && Edges[EI - 1].getOffset() >= Delta)
265 --EI;
266
267 for (size_t J = EI; J != Edges.size(); ++J) {
268 Blocks[I]->Edges.push_back(x: std::move(Edges[J]));
269 Blocks[I]->Edges.back().setOffset(Blocks[I]->Edges.back().getOffset() -
270 Delta);
271 }
272
273 while (Edges.size() > EI)
274 Edges.pop_back();
275 }
276
277 return Blocks;
278}
279
280void LinkGraph::dump(raw_ostream &OS) {
281 DenseMap<Block *, std::vector<Symbol *>> BlockSymbols;
282
283 // Map from blocks to the symbols pointing at them.
284 for (auto *Sym : defined_symbols())
285 BlockSymbols[&Sym->getBlock()].push_back(x: Sym);
286
287 // For each block, sort its symbols by something approximating
288 // relevance.
289 for (auto &KV : BlockSymbols)
290 llvm::sort(C&: KV.second, Comp: [](const Symbol *LHS, const Symbol *RHS) {
291 if (LHS->getOffset() != RHS->getOffset())
292 return LHS->getOffset() < RHS->getOffset();
293 if (LHS->getLinkage() != RHS->getLinkage())
294 return LHS->getLinkage() < RHS->getLinkage();
295 if (LHS->getScope() != RHS->getScope())
296 return LHS->getScope() < RHS->getScope();
297 if (LHS->hasName()) {
298 if (!RHS->hasName())
299 return true;
300 return LHS->getName() < RHS->getName();
301 }
302 return false;
303 });
304
305 std::vector<Section *> SortedSections;
306 for (auto &Sec : sections())
307 SortedSections.push_back(x: &Sec);
308 llvm::sort(C&: SortedSections, Comp: [](const Section *LHS, const Section *RHS) {
309 return LHS->getName() < RHS->getName();
310 });
311
312 for (auto *Sec : SortedSections) {
313 OS << "section " << Sec->getName() << ":\n\n";
314
315 std::vector<Block *> SortedBlocks;
316 llvm::append_range(C&: SortedBlocks, R: Sec->blocks());
317 llvm::sort(C&: SortedBlocks, Comp: [](const Block *LHS, const Block *RHS) {
318 return LHS->getAddress() < RHS->getAddress();
319 });
320
321 for (auto *B : SortedBlocks) {
322 OS << " block " << B->getAddress()
323 << " size = " << formatv(Fmt: "{0:x8}", Vals: B->getSize())
324 << ", align = " << B->getAlignment()
325 << ", alignment-offset = " << B->getAlignmentOffset();
326 if (B->isZeroFill())
327 OS << ", zero-fill";
328 OS << "\n";
329
330 auto BlockSymsI = BlockSymbols.find(Val: B);
331 if (BlockSymsI != BlockSymbols.end()) {
332 OS << " symbols:\n";
333 auto &Syms = BlockSymsI->second;
334 for (auto *Sym : Syms)
335 OS << " " << *Sym << "\n";
336 } else
337 OS << " no symbols\n";
338
339 if (!B->edges_empty()) {
340 OS << " edges:\n";
341 std::vector<Edge> SortedEdges;
342 llvm::append_range(C&: SortedEdges, R: B->edges());
343 llvm::sort(C&: SortedEdges, Comp: [](const Edge &LHS, const Edge &RHS) {
344 return LHS.getOffset() < RHS.getOffset();
345 });
346 for (auto &E : SortedEdges) {
347 OS << " " << B->getFixupAddress(E) << " (block + "
348 << formatv(Fmt: "{0:x8}", Vals: E.getOffset()) << "), addend = ";
349 if (E.getAddend() >= 0)
350 OS << formatv(Fmt: "+{0:x8}", Vals: E.getAddend());
351 else
352 OS << formatv(Fmt: "-{0:x8}", Vals: -E.getAddend());
353 OS << ", kind = " << getEdgeKindName(K: E.getKind()) << ", target = ";
354 if (E.getTarget().hasName())
355 OS << E.getTarget().getName();
356 else
357 OS << "addressable@"
358 << formatv(Fmt: "{0:x16}", Vals: E.getTarget().getAddress()) << "+"
359 << formatv(Fmt: "{0:x8}", Vals: E.getTarget().getOffset());
360 OS << "\n";
361 }
362 } else
363 OS << " no edges\n";
364 OS << "\n";
365 }
366 }
367
368 OS << "Absolute symbols:\n";
369 if (!absolute_symbols().empty()) {
370 for (auto *Sym : absolute_symbols())
371 OS << " " << Sym->getAddress() << ": " << *Sym << "\n";
372 } else
373 OS << " none\n";
374
375 OS << "\nExternal symbols:\n";
376 if (!external_symbols().empty()) {
377 for (auto *Sym : external_symbols())
378 OS << " " << Sym->getAddress() << ": " << *Sym
379 << (Sym->isWeaklyReferenced() ? " (weakly referenced)" : "") << "\n";
380 } else
381 OS << " none\n";
382}
383
384raw_ostream &operator<<(raw_ostream &OS, const SymbolLookupFlags &LF) {
385 switch (LF) {
386 case SymbolLookupFlags::RequiredSymbol:
387 return OS << "RequiredSymbol";
388 case SymbolLookupFlags::WeaklyReferencedSymbol:
389 return OS << "WeaklyReferencedSymbol";
390 }
391 llvm_unreachable("Unrecognized lookup flags");
392}
393
394void JITLinkAsyncLookupContinuation::anchor() {}
395
396JITLinkContext::~JITLinkContext() = default;
397
398bool JITLinkContext::shouldAddDefaultTargetPasses(const Triple &TT) const {
399 return true;
400}
401
402LinkGraphPassFunction JITLinkContext::getMarkLivePass(const Triple &TT) const {
403 return LinkGraphPassFunction();
404}
405
406Error JITLinkContext::modifyPassConfig(LinkGraph &G,
407 PassConfiguration &Config) {
408 return Error::success();
409}
410
411Error markAllSymbolsLive(LinkGraph &G) {
412 for (auto *Sym : G.defined_symbols())
413 Sym->setLive(true);
414 return Error::success();
415}
416
417Error makeTargetOutOfRangeError(const LinkGraph &G, const Block &B,
418 const Edge &E) {
419 std::string ErrMsg;
420 {
421 raw_string_ostream ErrStream(ErrMsg);
422 Section &Sec = B.getSection();
423 ErrStream << "In graph " << G.getName() << ", section " << Sec.getName()
424 << ": relocation target "
425 << formatv(Fmt: "{0:x}", Vals: E.getTarget().getAddress() + E.getAddend())
426 << " (";
427 if (E.getTarget().hasName())
428 ErrStream << E.getTarget().getName();
429 else
430 ErrStream << "<anonymous symbol>";
431 if (E.getAddend()) {
432 // Target address includes non-zero added, so break down the arithmetic.
433 ErrStream << formatv(Fmt: ":{0:x}", Vals: E.getTarget().getAddress()) << " + "
434 << formatv(Fmt: "{0:x}", Vals: E.getAddend());
435 }
436 ErrStream << ") is out of range of " << G.getEdgeKindName(K: E.getKind())
437 << " fixup at address "
438 << formatv(Fmt: "{0:x}", Vals: E.getTarget().getAddress()) << " (";
439
440 Symbol *BestSymbolForBlock = nullptr;
441 for (auto *Sym : Sec.symbols())
442 if (&Sym->getBlock() == &B && Sym->hasName() && Sym->getOffset() == 0 &&
443 (!BestSymbolForBlock ||
444 Sym->getScope() < BestSymbolForBlock->getScope() ||
445 Sym->getLinkage() < BestSymbolForBlock->getLinkage()))
446 BestSymbolForBlock = Sym;
447
448 if (BestSymbolForBlock)
449 ErrStream << BestSymbolForBlock->getName() << ", ";
450 else
451 ErrStream << "<anonymous block> @ ";
452
453 ErrStream << formatv(Fmt: "{0:x}", Vals: B.getAddress()) << " + "
454 << formatv(Fmt: "{0:x}", Vals: E.getOffset()) << ")";
455 }
456 return make_error<JITLinkError>(Args: std::move(ErrMsg));
457}
458
459Error makeAlignmentError(llvm::orc::ExecutorAddr Loc, uint64_t Value, int N,
460 const Edge &E) {
461 return make_error<JITLinkError>(Args: "0x" + llvm::utohexstr(X: Loc.getValue()) +
462 " improper alignment for relocation " +
463 formatv(Fmt: "{0:d}", Vals: E.getKind()) + ": 0x" +
464 llvm::utohexstr(X: Value) +
465 " is not aligned to " + Twine(N) + " bytes");
466}
467
468AnonymousPointerCreator getAnonymousPointerCreator(const Triple &TT) {
469 switch (TT.getArch()) {
470 case Triple::aarch64:
471 return aarch64::createAnonymousPointer;
472 case Triple::x86_64:
473 return x86_64::createAnonymousPointer;
474 case Triple::x86:
475 return x86::createAnonymousPointer;
476 case Triple::loongarch32:
477 case Triple::loongarch64:
478 return loongarch::createAnonymousPointer;
479 default:
480 return nullptr;
481 }
482}
483
484PointerJumpStubCreator getPointerJumpStubCreator(const Triple &TT) {
485 switch (TT.getArch()) {
486 case Triple::aarch64:
487 return aarch64::createAnonymousPointerJumpStub;
488 case Triple::x86_64:
489 return x86_64::createAnonymousPointerJumpStub;
490 case Triple::x86:
491 return x86::createAnonymousPointerJumpStub;
492 case Triple::loongarch32:
493 case Triple::loongarch64:
494 return loongarch::createAnonymousPointerJumpStub;
495 default:
496 return nullptr;
497 }
498}
499
500Expected<std::unique_ptr<LinkGraph>>
501createLinkGraphFromObject(MemoryBufferRef ObjectBuffer,
502 std::shared_ptr<orc::SymbolStringPool> SSP) {
503 auto Magic = identify_magic(magic: ObjectBuffer.getBuffer());
504 switch (Magic) {
505 case file_magic::macho_object:
506 return createLinkGraphFromMachOObject(ObjectBuffer, SSP: std::move(SSP));
507 case file_magic::elf_relocatable:
508 return createLinkGraphFromELFObject(ObjectBuffer, SSP: std::move(SSP));
509 case file_magic::coff_object:
510 return createLinkGraphFromCOFFObject(ObjectBuffer, SSP: std::move(SSP));
511 case file_magic::xcoff_object_64:
512 return createLinkGraphFromXCOFFObject(ObjectBuffer, SSP: std::move(SSP));
513 default:
514 return make_error<JITLinkError>(Args: "Unsupported file format");
515 };
516}
517
518std::unique_ptr<LinkGraph>
519absoluteSymbolsLinkGraph(Triple TT, std::shared_ptr<orc::SymbolStringPool> SSP,
520 orc::SymbolMap Symbols) {
521 static std::atomic<uint64_t> Counter = {0};
522 auto Index = Counter.fetch_add(i: 1, m: std::memory_order_relaxed);
523 auto G = std::make_unique<LinkGraph>(
524 args: "<Absolute Symbols " + std::to_string(val: Index) + ">", args: std::move(SSP),
525 args: std::move(TT), args: SubtargetFeatures(), args&: getGenericEdgeKindName);
526 for (auto &[Name, Def] : Symbols) {
527 auto &Sym =
528 G->addAbsoluteSymbol(Name: *Name, Address: Def.getAddress(), /*Size=*/0,
529 L: Linkage::Strong, S: Scope::Default, /*IsLive=*/true);
530 Sym.setCallable(Def.getFlags().isCallable());
531 }
532
533 return G;
534}
535
536void link(std::unique_ptr<LinkGraph> G, std::unique_ptr<JITLinkContext> Ctx) {
537 switch (G->getTargetTriple().getObjectFormat()) {
538 case Triple::MachO:
539 return link_MachO(G: std::move(G), Ctx: std::move(Ctx));
540 case Triple::ELF:
541 return link_ELF(G: std::move(G), Ctx: std::move(Ctx));
542 case Triple::COFF:
543 return link_COFF(G: std::move(G), Ctx: std::move(Ctx));
544 case Triple::XCOFF:
545 return link_XCOFF(G: std::move(G), Ctx: std::move(Ctx));
546 default:
547 Ctx->notifyFailed(Err: make_error<JITLinkError>(Args: "Unsupported object format"));
548 };
549}
550
551} // end namespace jitlink
552} // end namespace llvm
553