1//===----- ELFNixPlatform.cpp - Utilities for executing ELFNix in Orc -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/Orc/ELFNixPlatform.h"
10
11#include "llvm/ExecutionEngine/JITLink/aarch64.h"
12#include "llvm/ExecutionEngine/JITLink/loongarch.h"
13#include "llvm/ExecutionEngine/JITLink/ppc64.h"
14#include "llvm/ExecutionEngine/JITLink/x86_64.h"
15#include "llvm/ExecutionEngine/Orc/AbsoluteSymbols.h"
16#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
17#include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h"
18#include "llvm/Support/Debug.h"
19#include <optional>
20
21#define DEBUG_TYPE "orc"
22
23using namespace llvm;
24using namespace llvm::orc;
25using namespace llvm::orc::shared;
26
27namespace {
28
29template <typename SPSSerializer, typename... ArgTs>
30shared::WrapperFunctionCall::ArgDataBufferType
31getArgDataBufferType(const ArgTs &...Args) {
32 shared::WrapperFunctionCall::ArgDataBufferType ArgData;
33 ArgData.resize(SPSSerializer::size(Args...));
34 SPSOutputBuffer OB(ArgData.empty() ? nullptr : ArgData.data(),
35 ArgData.size());
36 if (SPSSerializer::serialize(OB, Args...))
37 return ArgData;
38 return {};
39}
40
41std::unique_ptr<jitlink::LinkGraph> createPlatformGraph(ELFNixPlatform &MOP,
42 std::string Name) {
43 auto &ES = MOP.getExecutionSession();
44 return std::make_unique<jitlink::LinkGraph>(
45 args: std::move(Name), args: ES.getSymbolStringPool(), args: ES.getTargetTriple(),
46 args: SubtargetFeatures(), args&: jitlink::getGenericEdgeKindName);
47}
48
49// Creates a Bootstrap-Complete LinkGraph to run deferred actions.
50class ELFNixPlatformCompleteBootstrapMaterializationUnit
51 : public MaterializationUnit {
52public:
53 ELFNixPlatformCompleteBootstrapMaterializationUnit(
54 ELFNixPlatform &MOP, StringRef PlatformJDName,
55 SymbolStringPtr CompleteBootstrapSymbol, DeferredRuntimeFnMap DeferredAAs,
56 ExecutorAddr ELFNixHeaderAddr, ExecutorAddr PlatformBootstrap,
57 ExecutorAddr PlatformShutdown, ExecutorAddr RegisterJITDylib,
58 ExecutorAddr DeregisterJITDylib)
59 : MaterializationUnit(
60 {{{CompleteBootstrapSymbol, JITSymbolFlags::None}}, nullptr}),
61 MOP(MOP), PlatformJDName(PlatformJDName),
62 CompleteBootstrapSymbol(std::move(CompleteBootstrapSymbol)),
63 DeferredAAsMap(std::move(DeferredAAs)),
64 ELFNixHeaderAddr(ELFNixHeaderAddr),
65 PlatformBootstrap(PlatformBootstrap),
66 PlatformShutdown(PlatformShutdown), RegisterJITDylib(RegisterJITDylib),
67 DeregisterJITDylib(DeregisterJITDylib) {}
68
69 StringRef getName() const override {
70 return "ELFNixPlatformCompleteBootstrap";
71 }
72
73 void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
74 using namespace jitlink;
75 auto G = createPlatformGraph(MOP, Name: "<OrcRTCompleteBootstrap>");
76 auto &PlaceholderSection =
77 G->createSection(Name: "__orc_rt_cplt_bs", Prot: MemProt::Read);
78 auto &PlaceholderBlock =
79 G->createZeroFillBlock(Parent&: PlaceholderSection, Size: 1, Address: ExecutorAddr(), Alignment: 1, AlignmentOffset: 0);
80 G->addDefinedSymbol(Content&: PlaceholderBlock, Offset: 0, Name: *CompleteBootstrapSymbol, Size: 1,
81 L: Linkage::Strong, S: Scope::Hidden, IsCallable: false, IsLive: true);
82
83 // 1. Bootstrap the platform support code.
84 G->allocActions().push_back(
85 x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
86 FnAddr: PlatformBootstrap, Args: ELFNixHeaderAddr)),
87 .Dealloc: cantFail(
88 ValOrErr: WrapperFunctionCall::Create<SPSArgList<>>(FnAddr: PlatformShutdown))});
89
90 // 2. Register the platform JITDylib.
91 G->allocActions().push_back(
92 x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<
93 SPSArgList<SPSString, SPSExecutorAddr>>(
94 FnAddr: RegisterJITDylib, Args: PlatformJDName, Args: ELFNixHeaderAddr)),
95 .Dealloc: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
96 FnAddr: DeregisterJITDylib, Args: ELFNixHeaderAddr))});
97
98 // 4. Add the deferred actions to the graph.
99 for (auto &[Fn, CallDatas] : DeferredAAsMap) {
100 for (auto &CallData : CallDatas) {
101 G->allocActions().push_back(
102 x: {.Finalize: WrapperFunctionCall(Fn.first->Addr, std::move(CallData.first)),
103 .Dealloc: WrapperFunctionCall(Fn.second->Addr, std::move(CallData.second))});
104 }
105 }
106
107 MOP.getObjectLinkingLayer().emit(R: std::move(R), G: std::move(G));
108 }
109
110 void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
111
112private:
113 ELFNixPlatform &MOP;
114 StringRef PlatformJDName;
115 SymbolStringPtr CompleteBootstrapSymbol;
116 DeferredRuntimeFnMap DeferredAAsMap;
117 ExecutorAddr ELFNixHeaderAddr;
118 ExecutorAddr PlatformBootstrap;
119 ExecutorAddr PlatformShutdown;
120 ExecutorAddr RegisterJITDylib;
121 ExecutorAddr DeregisterJITDylib;
122};
123
124class DSOHandleMaterializationUnit : public MaterializationUnit {
125public:
126 DSOHandleMaterializationUnit(ELFNixPlatform &ENP,
127 const SymbolStringPtr &DSOHandleSymbol)
128 : MaterializationUnit(
129 createDSOHandleSectionInterface(ENP, DSOHandleSymbol)),
130 ENP(ENP) {}
131
132 StringRef getName() const override { return "DSOHandleMU"; }
133
134 void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
135
136 auto &ES = ENP.getExecutionSession();
137
138 jitlink::Edge::Kind EdgeKind;
139
140 switch (ES.getTargetTriple().getArch()) {
141 case Triple::x86_64:
142 EdgeKind = jitlink::x86_64::Pointer64;
143 break;
144 case Triple::aarch64:
145 EdgeKind = jitlink::aarch64::Pointer64;
146 break;
147 case Triple::ppc64:
148 EdgeKind = jitlink::ppc64::Pointer64;
149 break;
150 case Triple::ppc64le:
151 EdgeKind = jitlink::ppc64::Pointer64;
152 break;
153 case Triple::loongarch64:
154 EdgeKind = jitlink::loongarch::Pointer64;
155 break;
156 default:
157 llvm_unreachable("Unrecognized architecture");
158 }
159
160 // void *__dso_handle = &__dso_handle;
161 auto G = std::make_unique<jitlink::LinkGraph>(
162 args: "<DSOHandleMU>", args: ES.getSymbolStringPool(), args: ES.getTargetTriple(),
163 args: SubtargetFeatures(), args&: jitlink::getGenericEdgeKindName);
164 auto &DSOHandleSection =
165 G->createSection(Name: ".data.__dso_handle", Prot: MemProt::Read);
166 auto &DSOHandleBlock = G->createContentBlock(
167 Parent&: DSOHandleSection, Content: getDSOHandleContent(PointerSize: G->getPointerSize()),
168 Address: orc::ExecutorAddr(), Alignment: 8, AlignmentOffset: 0);
169 auto &DSOHandleSymbol = G->addDefinedSymbol(
170 Content&: DSOHandleBlock, Offset: 0, Name: *R->getInitializerSymbol(), Size: DSOHandleBlock.getSize(),
171 L: jitlink::Linkage::Strong, S: jitlink::Scope::Default, IsCallable: false, IsLive: true);
172 DSOHandleBlock.addEdge(K: EdgeKind, Offset: 0, Target&: DSOHandleSymbol, Addend: 0);
173
174 ENP.getObjectLinkingLayer().emit(R: std::move(R), G: std::move(G));
175 }
176
177 void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
178
179private:
180 static MaterializationUnit::Interface
181 createDSOHandleSectionInterface(ELFNixPlatform &ENP,
182 const SymbolStringPtr &DSOHandleSymbol) {
183 SymbolFlagsMap SymbolFlags;
184 SymbolFlags[DSOHandleSymbol] = JITSymbolFlags::Exported;
185 return MaterializationUnit::Interface(std::move(SymbolFlags),
186 DSOHandleSymbol);
187 }
188
189 ArrayRef<char> getDSOHandleContent(size_t PointerSize) {
190 static const char Content[8] = {0};
191 assert(PointerSize <= sizeof Content);
192 return {Content, PointerSize};
193 }
194
195 ELFNixPlatform &ENP;
196};
197
198} // end anonymous namespace
199
200namespace llvm {
201namespace orc {
202
203Expected<std::unique_ptr<ELFNixPlatform>>
204ELFNixPlatform::Create(ObjectLinkingLayer &ObjLinkingLayer,
205 JITDylib &PlatformJD,
206 std::unique_ptr<DefinitionGenerator> OrcRuntime,
207 std::optional<SymbolAliasMap> RuntimeAliases) {
208
209 auto &ES = ObjLinkingLayer.getExecutionSession();
210
211 // If the target is not supported then bail out immediately.
212 if (!supportedTarget(TT: ES.getTargetTriple()))
213 return make_error<StringError>(Args: "Unsupported ELFNixPlatform triple: " +
214 ES.getTargetTriple().str(),
215 Args: inconvertibleErrorCode());
216
217 auto &EPC = ES.getExecutorProcessControl();
218
219 // Create default aliases if the caller didn't supply any.
220 if (!RuntimeAliases) {
221 auto StandardRuntimeAliases = standardPlatformAliases(ES, PlatformJD);
222 if (!StandardRuntimeAliases)
223 return StandardRuntimeAliases.takeError();
224 RuntimeAliases = std::move(*StandardRuntimeAliases);
225 }
226
227 // Define the aliases.
228 if (auto Err = PlatformJD.define(MU: symbolAliases(Aliases: std::move(*RuntimeAliases))))
229 return std::move(Err);
230
231 // Add JIT-dispatch function support symbols.
232 if (auto Err = PlatformJD.define(
233 MU: absoluteSymbols(Symbols: {{ES.intern(SymName: "__orc_rt_jit_dispatch"),
234 {EPC.getJITDispatchInfo().JITDispatchFunction,
235 JITSymbolFlags::Exported}},
236 {ES.intern(SymName: "__orc_rt_jit_dispatch_ctx"),
237 {EPC.getJITDispatchInfo().JITDispatchContext,
238 JITSymbolFlags::Exported}}})))
239 return std::move(Err);
240
241 // Create the instance.
242 Error Err = Error::success();
243 auto P = std::unique_ptr<ELFNixPlatform>(new ELFNixPlatform(
244 ObjLinkingLayer, PlatformJD, std::move(OrcRuntime), Err));
245 if (Err)
246 return std::move(Err);
247 return std::move(P);
248}
249
250Expected<std::unique_ptr<ELFNixPlatform>>
251ELFNixPlatform::Create(ObjectLinkingLayer &ObjLinkingLayer,
252 JITDylib &PlatformJD, const char *OrcRuntimePath,
253 std::optional<SymbolAliasMap> RuntimeAliases) {
254
255 // Create a generator for the ORC runtime archive.
256 auto OrcRuntimeArchiveGenerator =
257 StaticLibraryDefinitionGenerator::Load(L&: ObjLinkingLayer, FileName: OrcRuntimePath);
258 if (!OrcRuntimeArchiveGenerator)
259 return OrcRuntimeArchiveGenerator.takeError();
260
261 return Create(ObjLinkingLayer, PlatformJD,
262 OrcRuntime: std::move(*OrcRuntimeArchiveGenerator),
263 RuntimeAliases: std::move(RuntimeAliases));
264}
265
266Error ELFNixPlatform::setupJITDylib(JITDylib &JD) {
267 if (auto Err = JD.define(MU: std::make_unique<DSOHandleMaterializationUnit>(
268 args&: *this, args&: DSOHandleSymbol)))
269 return Err;
270
271 return ES.lookup(SearchOrder: {&JD}, Symbol: DSOHandleSymbol).takeError();
272}
273
274Error ELFNixPlatform::teardownJITDylib(JITDylib &JD) {
275 std::lock_guard<std::mutex> Lock(PlatformMutex);
276 auto I = JITDylibToHandleAddr.find(Val: &JD);
277 if (I != JITDylibToHandleAddr.end()) {
278 assert(HandleAddrToJITDylib.count(I->second) &&
279 "HandleAddrToJITDylib missing entry");
280 HandleAddrToJITDylib.erase(Val: I->second);
281 JITDylibToHandleAddr.erase(I);
282 }
283 return Error::success();
284}
285
286Error ELFNixPlatform::notifyAdding(ResourceTracker &RT,
287 const MaterializationUnit &MU) {
288
289 auto &JD = RT.getJITDylib();
290 const auto &InitSym = MU.getInitializerSymbol();
291 if (!InitSym)
292 return Error::success();
293
294 RegisteredInitSymbols[&JD].add(Name: InitSym,
295 Flags: SymbolLookupFlags::WeaklyReferencedSymbol);
296 LLVM_DEBUG({
297 dbgs() << "ELFNixPlatform: Registered init symbol " << *InitSym
298 << " for MU " << MU.getName() << "\n";
299 });
300 return Error::success();
301}
302
303Error ELFNixPlatform::notifyRemoving(ResourceTracker &RT) {
304 llvm_unreachable("Not supported yet");
305}
306
307static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases,
308 ArrayRef<std::pair<const char *, const char *>> AL) {
309 for (auto &KV : AL) {
310 auto AliasName = ES.intern(SymName: KV.first);
311 assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map");
312 Aliases[std::move(AliasName)] = {ES.intern(SymName: KV.second),
313 JITSymbolFlags::Exported};
314 }
315}
316
317Expected<SymbolAliasMap>
318ELFNixPlatform::standardPlatformAliases(ExecutionSession &ES,
319 JITDylib &PlatformJD) {
320 SymbolAliasMap Aliases;
321 addAliases(ES, Aliases, AL: requiredCXXAliases());
322 addAliases(ES, Aliases, AL: standardRuntimeUtilityAliases());
323 addAliases(ES, Aliases, AL: standardLazyCompilationAliases());
324 return Aliases;
325}
326
327ArrayRef<std::pair<const char *, const char *>>
328ELFNixPlatform::requiredCXXAliases() {
329 static const std::pair<const char *, const char *> RequiredCXXAliases[] = {
330 {"__cxa_atexit", "__orc_rt_elfnix_cxa_atexit"},
331 {"atexit", "__orc_rt_elfnix_atexit"}};
332
333 return ArrayRef<std::pair<const char *, const char *>>(RequiredCXXAliases);
334}
335
336ArrayRef<std::pair<const char *, const char *>>
337ELFNixPlatform::standardRuntimeUtilityAliases() {
338 static const std::pair<const char *, const char *>
339 StandardRuntimeUtilityAliases[] = {
340 {"__orc_rt_run_program", "__orc_rt_elfnix_run_program"},
341 {"__orc_rt_jit_dlerror", "__orc_rt_elfnix_jit_dlerror"},
342 {"__orc_rt_jit_dlopen", "__orc_rt_elfnix_jit_dlopen"},
343 {"__orc_rt_jit_dlupdate", "__orc_rt_elfnix_jit_dlupdate"},
344 {"__orc_rt_jit_dlclose", "__orc_rt_elfnix_jit_dlclose"},
345 {"__orc_rt_jit_dlsym", "__orc_rt_elfnix_jit_dlsym"},
346 {"__orc_rt_log_error", "__orc_rt_log_error_to_stderr"}};
347
348 return ArrayRef<std::pair<const char *, const char *>>(
349 StandardRuntimeUtilityAliases);
350}
351
352ArrayRef<std::pair<const char *, const char *>>
353ELFNixPlatform::standardLazyCompilationAliases() {
354 static const std::pair<const char *, const char *>
355 StandardLazyCompilationAliases[] = {
356 {"__orc_rt_reenter", "__orc_rt_sysv_reenter"}};
357
358 return ArrayRef<std::pair<const char *, const char *>>(
359 StandardLazyCompilationAliases);
360}
361
362bool ELFNixPlatform::supportedTarget(const Triple &TT) {
363 switch (TT.getArch()) {
364 case Triple::x86_64:
365 case Triple::aarch64:
366 // FIXME: jitlink for ppc64 hasn't been well tested, leave it unsupported
367 // right now.
368 case Triple::ppc64le:
369 case Triple::loongarch64:
370 return true;
371 default:
372 return false;
373 }
374}
375
376ELFNixPlatform::ELFNixPlatform(
377 ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD,
378 std::unique_ptr<DefinitionGenerator> OrcRuntimeGenerator, Error &Err)
379 : ES(ObjLinkingLayer.getExecutionSession()), PlatformJD(PlatformJD),
380 ObjLinkingLayer(ObjLinkingLayer),
381 DSOHandleSymbol(ES.intern(SymName: "__dso_handle")) {
382 ErrorAsOutParameter _(Err);
383 ObjLinkingLayer.addPlugin(P: std::make_unique<ELFNixPlatformPlugin>(args&: *this));
384
385 PlatformJD.addGenerator(DefGenerator: std::move(OrcRuntimeGenerator));
386
387 BootstrapInfo BI;
388 Bootstrap = &BI;
389
390 // PlatformJD hasn't been 'set-up' by the platform yet (since we're creating
391 // the platform now), so set it up.
392 if (auto E2 = setupJITDylib(PlatformJD)) {
393 Err = std::move(E2);
394 return;
395 }
396
397 // Step (2) Request runtime registration functions to trigger
398 // materialization..
399 if ((Err = ES.lookup(
400 SearchOrder: makeJITDylibSearchOrder(JDs: &PlatformJD),
401 Symbols: SymbolLookupSet(
402 {PlatformBootstrap.Name, PlatformShutdown.Name,
403 RegisterJITDylib.Name, DeregisterJITDylib.Name,
404 RegisterInitSections.Name, DeregisterInitSections.Name,
405 RegisterObjectSections.Name,
406 DeregisterObjectSections.Name, CreatePThreadKey.Name}))
407 .takeError()))
408 return;
409
410 // Step (3) Wait for any incidental linker work to complete.
411 {
412 std::unique_lock<std::mutex> Lock(BI.Mutex);
413 BI.CV.wait(lock&: Lock, p: [&]() { return BI.ActiveGraphs == 0; });
414 Bootstrap = nullptr;
415 }
416
417 // Step (4) Add complete-bootstrap materialization unit and request.
418 auto BootstrapCompleteSymbol =
419 ES.intern(SymName: "__orc_rt_elfnix_complete_bootstrap");
420 if ((Err = PlatformJD.define(
421 MU: std::make_unique<ELFNixPlatformCompleteBootstrapMaterializationUnit>(
422 args&: *this, args: PlatformJD.getName(), args&: BootstrapCompleteSymbol,
423 args: std::move(BI.DeferredRTFnMap), args&: BI.ELFNixHeaderAddr,
424 args&: PlatformBootstrap.Addr, args&: PlatformShutdown.Addr,
425 args&: RegisterJITDylib.Addr, args&: DeregisterJITDylib.Addr))))
426 return;
427 if ((Err = ES.lookup(SearchOrder: makeJITDylibSearchOrder(
428 JDs: &PlatformJD, Flags: JITDylibLookupFlags::MatchAllSymbols),
429 Symbol: std::move(BootstrapCompleteSymbol))
430 .takeError()))
431 return;
432
433 // Associate wrapper function tags with JIT-side function implementations.
434 if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) {
435 Err = std::move(E2);
436 return;
437 }
438}
439
440Error ELFNixPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) {
441 ExecutionSession::JITDispatchHandlerAssociationMap WFs;
442
443 using RecordInitializersSPSSig =
444 SPSExpected<SPSELFNixJITDylibDepInfoMap>(SPSExecutorAddr);
445 WFs[ES.intern(SymName: "__orc_rt_elfnix_push_initializers_tag")] =
446 ES.wrapAsyncWithSPS<RecordInitializersSPSSig>(
447 Instance: this, Method: &ELFNixPlatform::rt_recordInitializers);
448
449 using LookupSymbolSPSSig =
450 SPSExpected<SPSExecutorAddr>(SPSExecutorAddr, SPSString);
451 WFs[ES.intern(SymName: "__orc_rt_elfnix_symbol_lookup_tag")] =
452 ES.wrapAsyncWithSPS<LookupSymbolSPSSig>(Instance: this,
453 Method: &ELFNixPlatform::rt_lookupSymbol);
454
455 return ES.registerJITDispatchHandlers(JD&: PlatformJD, WFs: std::move(WFs));
456}
457
458void ELFNixPlatform::pushInitializersLoop(
459 PushInitializersSendResultFn SendResult, JITDylibSP JD) {
460 DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols;
461 DenseMap<JITDylib *, SmallVector<JITDylib *>> JDDepMap;
462 SmallVector<JITDylib *, 16> Worklist({JD.get()});
463
464 ES.runSessionLocked(F: [&]() {
465 while (!Worklist.empty()) {
466 // FIXME: Check for defunct dylibs.
467
468 auto DepJD = Worklist.back();
469 Worklist.pop_back();
470
471 // If we've already visited this JITDylib on this iteration then continue.
472 auto [It, Inserted] = JDDepMap.try_emplace(Key: DepJD);
473 if (!Inserted)
474 continue;
475
476 // Add dep info.
477 auto &DM = It->second;
478 DepJD->withLinkOrderDo(F: [&](const JITDylibSearchOrder &O) {
479 for (auto &KV : O) {
480 if (KV.first == DepJD)
481 continue;
482 DM.push_back(Elt: KV.first);
483 Worklist.push_back(Elt: KV.first);
484 }
485 });
486
487 // Add any registered init symbols.
488 auto RISItr = RegisteredInitSymbols.find(Val: DepJD);
489 if (RISItr != RegisteredInitSymbols.end()) {
490 NewInitSymbols[DepJD] = std::move(RISItr->second);
491 RegisteredInitSymbols.erase(I: RISItr);
492 }
493 }
494 });
495
496 // If there are no further init symbols to look up then send the link order
497 // (as a list of header addresses) to the caller.
498 if (NewInitSymbols.empty()) {
499
500 // To make the list intelligible to the runtime we need to convert all
501 // JITDylib pointers to their header addresses. Only include JITDylibs
502 // that appear in the JITDylibToHandleAddr map (i.e. those that have been
503 // through setupJITDylib) -- bare JITDylibs aren't managed by the platform.
504 DenseMap<JITDylib *, ExecutorAddr> HeaderAddrs;
505 HeaderAddrs.reserve(NumEntries: JDDepMap.size());
506 {
507 std::lock_guard<std::mutex> Lock(PlatformMutex);
508 for (auto &KV : JDDepMap) {
509 auto I = JITDylibToHandleAddr.find(Val: KV.first);
510 if (I != JITDylibToHandleAddr.end())
511 HeaderAddrs[KV.first] = I->second;
512 }
513 }
514
515 // Build the dep info map to return.
516 ELFNixJITDylibDepInfoMap DIM;
517 DIM.reserve(n: JDDepMap.size());
518 for (auto &KV : JDDepMap) {
519 auto HI = HeaderAddrs.find(Val: KV.first);
520 // Skip unmanaged JITDylibs.
521 if (HI == HeaderAddrs.end())
522 continue;
523 auto H = HI->second;
524 ELFNixJITDylibDepInfo DepInfo;
525 for (auto &Dep : KV.second) {
526 auto HJ = HeaderAddrs.find(Val: Dep);
527 if (HJ != HeaderAddrs.end())
528 DepInfo.push_back(x: HJ->second);
529 }
530 DIM.push_back(x: std::make_pair(x&: H, y: std::move(DepInfo)));
531 }
532 SendResult(DIM);
533 return;
534 }
535
536 // Otherwise issue a lookup and re-run this phase when it completes.
537 lookupInitSymbolsAsync(
538 OnComplete: [this, SendResult = std::move(SendResult), JD](Error Err) mutable {
539 if (Err)
540 SendResult(std::move(Err));
541 else
542 pushInitializersLoop(SendResult: std::move(SendResult), JD);
543 },
544 ES, InitSyms: std::move(NewInitSymbols));
545}
546
547void ELFNixPlatform::rt_recordInitializers(
548 PushInitializersSendResultFn SendResult, ExecutorAddr JDHeaderAddr) {
549 JITDylibSP JD;
550 {
551 std::lock_guard<std::mutex> Lock(PlatformMutex);
552 auto I = HandleAddrToJITDylib.find(Val: JDHeaderAddr);
553 if (I != HandleAddrToJITDylib.end())
554 JD = I->second;
555 }
556
557 LLVM_DEBUG({
558 dbgs() << "ELFNixPlatform::rt_recordInitializers(" << JDHeaderAddr << ") ";
559 if (JD)
560 dbgs() << "pushing initializers for " << JD->getName() << "\n";
561 else
562 dbgs() << "No JITDylib for header address.\n";
563 });
564
565 if (!JD) {
566 SendResult(make_error<StringError>(Args: "No JITDylib with header addr " +
567 formatv(Fmt: "{0:x}", Vals&: JDHeaderAddr),
568 Args: inconvertibleErrorCode()));
569 return;
570 }
571
572 pushInitializersLoop(SendResult: std::move(SendResult), JD);
573}
574
575void ELFNixPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult,
576 ExecutorAddr Handle,
577 StringRef SymbolName) {
578 LLVM_DEBUG({
579 dbgs() << "ELFNixPlatform::rt_lookupSymbol(\"" << Handle << "\")\n";
580 });
581
582 JITDylib *JD = nullptr;
583
584 {
585 std::lock_guard<std::mutex> Lock(PlatformMutex);
586 auto I = HandleAddrToJITDylib.find(Val: Handle);
587 if (I != HandleAddrToJITDylib.end())
588 JD = I->second;
589 }
590
591 if (!JD) {
592 LLVM_DEBUG(dbgs() << " No JITDylib for handle " << Handle << "\n");
593 SendResult(make_error<StringError>(Args: "No JITDylib associated with handle " +
594 formatv(Fmt: "{0:x}", Vals&: Handle),
595 Args: inconvertibleErrorCode()));
596 return;
597 }
598
599 // Use functor class to work around XL build compiler issue on AIX.
600 class RtLookupNotifyComplete {
601 public:
602 RtLookupNotifyComplete(SendSymbolAddressFn &&SendResult)
603 : SendResult(std::move(SendResult)) {}
604 void operator()(Expected<SymbolMap> Result) {
605 if (Result) {
606 assert(Result->size() == 1 && "Unexpected result map count");
607 SendResult(Result->begin()->second.getAddress());
608 } else {
609 SendResult(Result.takeError());
610 }
611 }
612
613 private:
614 SendSymbolAddressFn SendResult;
615 };
616
617 ES.lookup(
618 K: LookupKind::DLSym, SearchOrder: {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}},
619 Symbols: SymbolLookupSet(ES.intern(SymName: SymbolName)), RequiredState: SymbolState::Ready,
620 NotifyComplete: RtLookupNotifyComplete(std::move(SendResult)), RegisterDependencies: NoDependenciesToRegister);
621}
622
623Error ELFNixPlatform::ELFNixPlatformPlugin::bootstrapPipelineStart(
624 jitlink::LinkGraph &G) {
625 // Increment the active graphs count in BootstrapInfo.
626 std::lock_guard<std::mutex> Lock(MP.Bootstrap.load()->Mutex);
627 ++MP.Bootstrap.load()->ActiveGraphs;
628 return Error::success();
629}
630
631Error ELFNixPlatform::ELFNixPlatformPlugin::
632 bootstrapPipelineRecordRuntimeFunctions(jitlink::LinkGraph &G) {
633 // Record bootstrap function names.
634 std::pair<StringRef, ExecutorAddr *> RuntimeSymbols[] = {
635 {*MP.DSOHandleSymbol, &MP.Bootstrap.load()->ELFNixHeaderAddr},
636 {*MP.PlatformBootstrap.Name, &MP.PlatformBootstrap.Addr},
637 {*MP.PlatformShutdown.Name, &MP.PlatformShutdown.Addr},
638 {*MP.RegisterJITDylib.Name, &MP.RegisterJITDylib.Addr},
639 {*MP.DeregisterJITDylib.Name, &MP.DeregisterJITDylib.Addr},
640 {*MP.RegisterObjectSections.Name, &MP.RegisterObjectSections.Addr},
641 {*MP.DeregisterObjectSections.Name, &MP.DeregisterObjectSections.Addr},
642 {*MP.RegisterInitSections.Name, &MP.RegisterInitSections.Addr},
643 {*MP.DeregisterInitSections.Name, &MP.DeregisterInitSections.Addr},
644 {*MP.CreatePThreadKey.Name, &MP.CreatePThreadKey.Addr}};
645
646 bool RegisterELFNixHeader = false;
647
648 for (auto *Sym : G.defined_symbols()) {
649 for (auto &RTSym : RuntimeSymbols) {
650 if (Sym->hasName() && *Sym->getName() == RTSym.first) {
651 if (*RTSym.second)
652 return make_error<StringError>(
653 Args: "Duplicate " + RTSym.first +
654 " detected during ELFNixPlatform bootstrap",
655 Args: inconvertibleErrorCode());
656
657 if (*Sym->getName() == *MP.DSOHandleSymbol)
658 RegisterELFNixHeader = true;
659
660 *RTSym.second = Sym->getAddress();
661 }
662 }
663 }
664
665 if (RegisterELFNixHeader) {
666 // If this graph defines the elfnix header symbol then create the internal
667 // mapping between it and PlatformJD.
668 std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
669 MP.JITDylibToHandleAddr[&MP.PlatformJD] =
670 MP.Bootstrap.load()->ELFNixHeaderAddr;
671 MP.HandleAddrToJITDylib[MP.Bootstrap.load()->ELFNixHeaderAddr] =
672 &MP.PlatformJD;
673 }
674
675 return Error::success();
676}
677
678Error ELFNixPlatform::ELFNixPlatformPlugin::bootstrapPipelineEnd(
679 jitlink::LinkGraph &G) {
680 std::lock_guard<std::mutex> Lock(MP.Bootstrap.load()->Mutex);
681 assert(MP.Bootstrap && "DeferredAAs reset before bootstrap completed");
682 --MP.Bootstrap.load()->ActiveGraphs;
683 // Notify Bootstrap->CV while holding the mutex because the mutex is
684 // also keeping Bootstrap->CV alive.
685 if (MP.Bootstrap.load()->ActiveGraphs == 0)
686 MP.Bootstrap.load()->CV.notify_all();
687 return Error::success();
688}
689
690Error ELFNixPlatform::registerPerObjectSections(
691 jitlink::LinkGraph &G, const ELFPerObjectSectionsToRegister &POSR,
692 bool IsBootstrapping) {
693 using SPSRegisterPerObjSectionsArgs =
694 SPSArgList<SPSELFPerObjectSectionsToRegister>;
695
696 if (LLVM_UNLIKELY(IsBootstrapping)) {
697 Bootstrap.load()->addArgumentsToRTFnMap(
698 func1: &RegisterObjectSections, func2: &DeregisterObjectSections,
699 arg1: getArgDataBufferType<SPSRegisterPerObjSectionsArgs>(Args: POSR),
700 arg2: getArgDataBufferType<SPSRegisterPerObjSectionsArgs>(Args: POSR));
701 return Error::success();
702 }
703
704 G.allocActions().push_back(
705 x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSRegisterPerObjSectionsArgs>(
706 FnAddr: RegisterObjectSections.Addr, Args: POSR)),
707 .Dealloc: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSRegisterPerObjSectionsArgs>(
708 FnAddr: DeregisterObjectSections.Addr, Args: POSR))});
709
710 return Error::success();
711}
712
713Expected<uint64_t> ELFNixPlatform::createPThreadKey() {
714 if (!CreatePThreadKey.Addr)
715 return make_error<StringError>(
716 Args: "Attempting to create pthread key in target, but runtime support has "
717 "not been loaded yet",
718 Args: inconvertibleErrorCode());
719
720 Expected<uint64_t> Result(0);
721 if (auto Err = ES.callSPSWrapper<SPSExpected<uint64_t>(void)>(
722 WrapperFnAddr: CreatePThreadKey.Addr, WrapperCallArgs&: Result))
723 return std::move(Err);
724 return Result;
725}
726
727void ELFNixPlatform::ELFNixPlatformPlugin::modifyPassConfig(
728 MaterializationResponsibility &MR, jitlink::LinkGraph &LG,
729 jitlink::PassConfiguration &Config) {
730 using namespace jitlink;
731
732 bool InBootstrapPhase =
733 &MR.getTargetJITDylib() == &MP.PlatformJD && MP.Bootstrap;
734
735 // If we're in the bootstrap phase then increment the active graphs.
736 if (InBootstrapPhase) {
737 Config.PrePrunePasses.push_back(
738 x: [this](LinkGraph &G) { return bootstrapPipelineStart(G); });
739 Config.PostAllocationPasses.push_back(x: [this](LinkGraph &G) {
740 return bootstrapPipelineRecordRuntimeFunctions(G);
741 });
742 }
743
744 // If the initializer symbol is the __dso_handle symbol then just add
745 // the DSO handle support passes.
746 if (auto InitSymbol = MR.getInitializerSymbol()) {
747 if (InitSymbol == MP.DSOHandleSymbol && !InBootstrapPhase) {
748 addDSOHandleSupportPasses(MR, Config);
749 // The DSOHandle materialization unit doesn't require any other
750 // support, so we can bail out early.
751 return;
752 }
753
754 /// Preserve init sections.
755 Config.PrePrunePasses.push_back(
756 x: [this, &MR](jitlink::LinkGraph &G) -> Error {
757 if (auto Err = preserveInitSections(G, MR))
758 return Err;
759 return Error::success();
760 });
761 }
762
763 // Add passes for eh-frame and TLV support.
764 addEHAndTLVSupportPasses(MR, Config, IsBootstrapping: InBootstrapPhase);
765
766 // If the object contains initializers then add passes to record them.
767 Config.PostFixupPasses.push_back(x: [this, &JD = MR.getTargetJITDylib(),
768 InBootstrapPhase](jitlink::LinkGraph &G) {
769 return registerInitSections(G, JD, IsBootstrapping: InBootstrapPhase);
770 });
771
772 // If we're in the bootstrap phase then steal allocation actions and then
773 // decrement the active graphs.
774 if (InBootstrapPhase)
775 Config.PostFixupPasses.push_back(
776 x: [this](LinkGraph &G) { return bootstrapPipelineEnd(G); });
777}
778
779void ELFNixPlatform::ELFNixPlatformPlugin::addDSOHandleSupportPasses(
780 MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) {
781
782 Config.PostAllocationPasses.push_back(x: [this, &JD = MR.getTargetJITDylib()](
783 jitlink::LinkGraph &G) -> Error {
784 auto I = llvm::find_if(Range: G.defined_symbols(), P: [this](jitlink::Symbol *Sym) {
785 return Sym->getName() == MP.DSOHandleSymbol;
786 });
787 assert(I != G.defined_symbols().end() && "Missing DSO handle symbol");
788 {
789 std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
790 auto HandleAddr = (*I)->getAddress();
791 MP.HandleAddrToJITDylib[HandleAddr] = &JD;
792 MP.JITDylibToHandleAddr[&JD] = HandleAddr;
793
794 G.allocActions().push_back(
795 x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<
796 SPSArgList<SPSString, SPSExecutorAddr>>(
797 FnAddr: MP.RegisterJITDylib.Addr, Args: JD.getName(), Args: HandleAddr)),
798 .Dealloc: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
799 FnAddr: MP.DeregisterJITDylib.Addr, Args: HandleAddr))});
800 }
801 return Error::success();
802 });
803}
804
805void ELFNixPlatform::ELFNixPlatformPlugin::addEHAndTLVSupportPasses(
806 MaterializationResponsibility &MR, jitlink::PassConfiguration &Config,
807 bool IsBootstrapping) {
808
809 // Insert TLV lowering at the start of the PostPrunePasses, since we want
810 // it to run before GOT/PLT lowering.
811
812 // TODO: Check that before the fixTLVSectionsAndEdges pass, the GOT/PLT build
813 // pass has done. Because the TLS descriptor need to be allocate in GOT.
814 Config.PostPrunePasses.push_back(
815 x: [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
816 return fixTLVSectionsAndEdges(G, JD);
817 });
818
819 // Add a pass to register the final addresses of the eh-frame and TLV sections
820 // with the runtime.
821 Config.PostFixupPasses.push_back(x: [this, IsBootstrapping](
822 jitlink::LinkGraph &G) -> Error {
823 ELFPerObjectSectionsToRegister POSR;
824
825 if (auto *EHFrameSection = G.findSectionByName(Name: ELFEHFrameSectionName)) {
826 jitlink::SectionRange R(*EHFrameSection);
827 if (!R.empty())
828 POSR.EHFrameSection = R.getRange();
829 }
830
831 // Get a pointer to the thread data section if there is one. It will be used
832 // below.
833 jitlink::Section *ThreadDataSection =
834 G.findSectionByName(Name: ELFThreadDataSectionName);
835
836 // Handle thread BSS section if there is one.
837 if (auto *ThreadBSSSection = G.findSectionByName(Name: ELFThreadBSSSectionName)) {
838 // If there's already a thread data section in this graph then merge the
839 // thread BSS section content into it, otherwise just treat the thread
840 // BSS section as the thread data section.
841 if (ThreadDataSection)
842 G.mergeSections(DstSection&: *ThreadDataSection, SrcSection&: *ThreadBSSSection);
843 else
844 ThreadDataSection = ThreadBSSSection;
845 }
846
847 // Having merged thread BSS (if present) and thread data (if present),
848 // record the resulting section range.
849 if (ThreadDataSection) {
850 jitlink::SectionRange R(*ThreadDataSection);
851 if (!R.empty())
852 POSR.ThreadDataSection = R.getRange();
853 }
854
855 if (POSR.EHFrameSection.Start || POSR.ThreadDataSection.Start) {
856 if (auto Err = MP.registerPerObjectSections(G, POSR, IsBootstrapping))
857 return Err;
858 }
859
860 return Error::success();
861 });
862}
863
864Error ELFNixPlatform::ELFNixPlatformPlugin::preserveInitSections(
865 jitlink::LinkGraph &G, MaterializationResponsibility &MR) {
866
867 if (const auto &InitSymName = MR.getInitializerSymbol()) {
868
869 jitlink::Symbol *InitSym = nullptr;
870
871 for (auto &InitSection : G.sections()) {
872 // Skip non-init sections.
873 if (!isELFInitializerSection(SecName: InitSection.getName()) ||
874 InitSection.empty())
875 continue;
876
877 // Create the init symbol if it has not been created already and attach it
878 // to the first block.
879 if (!InitSym) {
880 auto &B = **InitSection.blocks().begin();
881 InitSym = &G.addDefinedSymbol(
882 Content&: B, Offset: 0, Name: *InitSymName, Size: B.getSize(), L: jitlink::Linkage::Strong,
883 S: jitlink::Scope::SideEffectsOnly, IsCallable: false, IsLive: true);
884 }
885
886 // Add keep-alive edges to anonymous symbols in all other init blocks.
887 for (auto *B : InitSection.blocks()) {
888 if (B == &InitSym->getBlock())
889 continue;
890
891 auto &S = G.addAnonymousSymbol(Content&: *B, Offset: 0, Size: B->getSize(), IsCallable: false, IsLive: true);
892 InitSym->getBlock().addEdge(K: jitlink::Edge::KeepAlive, Offset: 0, Target&: S, Addend: 0);
893 }
894 }
895 }
896
897 return Error::success();
898}
899
900Error ELFNixPlatform::ELFNixPlatformPlugin::registerInitSections(
901 jitlink::LinkGraph &G, JITDylib &JD, bool IsBootstrapping) {
902 SmallVector<ExecutorAddrRange> ELFNixPlatformSecs;
903 LLVM_DEBUG(dbgs() << "ELFNixPlatform::registerInitSections\n");
904
905 SmallVector<jitlink::Section *> OrderedInitSections;
906 for (auto &Sec : G.sections())
907 if (isELFInitializerSection(SecName: Sec.getName()))
908 OrderedInitSections.push_back(Elt: &Sec);
909
910 // FIXME: This handles priority order within the current graph, but we'll need
911 // to include priority information in the initializer allocation
912 // actions in order to respect the ordering across multiple graphs.
913 llvm::sort(C&: OrderedInitSections, Comp: [](const jitlink::Section *LHS,
914 const jitlink::Section *RHS) {
915 if (LHS->getName().starts_with(Prefix: ".init_array")) {
916 if (RHS->getName().starts_with(Prefix: ".init_array")) {
917 StringRef LHSPrioStr(LHS->getName());
918 StringRef RHSPrioStr(RHS->getName());
919 uint64_t LHSPriority;
920 bool LHSHasPriority = LHSPrioStr.consume_front(Prefix: ".init_array.") &&
921 !LHSPrioStr.getAsInteger(Radix: 10, Result&: LHSPriority);
922 uint64_t RHSPriority;
923 bool RHSHasPriority = RHSPrioStr.consume_front(Prefix: ".init_array.") &&
924 !RHSPrioStr.getAsInteger(Radix: 10, Result&: RHSPriority);
925 if (LHSHasPriority)
926 return RHSHasPriority ? LHSPriority < RHSPriority : true;
927 else if (RHSHasPriority)
928 return false;
929 // If we get here we'll fall through to the
930 // LHS->getName() < RHS->getName() test below.
931 } else {
932 // .init_array[.N] comes before any non-.init_array[.N] section.
933 return true;
934 }
935 }
936 return LHS->getName() < RHS->getName();
937 });
938
939 for (auto &Sec : OrderedInitSections)
940 ELFNixPlatformSecs.push_back(Elt: jitlink::SectionRange(*Sec).getRange());
941
942 // Dump the scraped inits.
943 LLVM_DEBUG({
944 dbgs() << "ELFNixPlatform: Scraped " << G.getName() << " init sections:\n";
945 for (auto &Sec : G.sections()) {
946 jitlink::SectionRange R(Sec);
947 dbgs() << " " << Sec.getName() << ": " << R.getRange() << "\n";
948 }
949 });
950
951 ExecutorAddr HeaderAddr;
952 {
953 std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
954 auto I = MP.JITDylibToHandleAddr.find(Val: &JD);
955 assert(I != MP.JITDylibToHandleAddr.end() && "No header registered for JD");
956 assert(I->second && "Null header registered for JD");
957 HeaderAddr = I->second;
958 }
959
960 using SPSRegisterInitSectionsArgs =
961 SPSArgList<SPSExecutorAddr, SPSSequence<SPSExecutorAddrRange>>;
962
963 if (LLVM_UNLIKELY(IsBootstrapping)) {
964 MP.Bootstrap.load()->addArgumentsToRTFnMap(
965 func1: &MP.RegisterInitSections, func2: &MP.DeregisterInitSections,
966 arg1: getArgDataBufferType<SPSRegisterInitSectionsArgs>(Args: HeaderAddr,
967 Args: ELFNixPlatformSecs),
968 arg2: getArgDataBufferType<SPSRegisterInitSectionsArgs>(Args: HeaderAddr,
969 Args: ELFNixPlatformSecs));
970 return Error::success();
971 }
972
973 G.allocActions().push_back(
974 x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSRegisterInitSectionsArgs>(
975 FnAddr: MP.RegisterInitSections.Addr, Args: HeaderAddr, Args: ELFNixPlatformSecs)),
976 .Dealloc: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSRegisterInitSectionsArgs>(
977 FnAddr: MP.DeregisterInitSections.Addr, Args: HeaderAddr, Args: ELFNixPlatformSecs))});
978
979 return Error::success();
980}
981
982Error ELFNixPlatform::ELFNixPlatformPlugin::fixTLVSectionsAndEdges(
983 jitlink::LinkGraph &G, JITDylib &JD) {
984 auto TLSGetAddrSymbolName = G.intern(SymbolName: "__tls_get_addr");
985 auto TLSDescResolveSymbolName = G.intern(SymbolName: "__tlsdesc_resolver");
986 for (auto *Sym : G.external_symbols()) {
987 if (Sym->getName() == TLSGetAddrSymbolName) {
988 auto TLSGetAddr =
989 MP.getExecutionSession().intern(SymName: "___orc_rt_elfnix_tls_get_addr");
990 Sym->setName(std::move(TLSGetAddr));
991 } else if (Sym->getName() == TLSDescResolveSymbolName) {
992 auto TLSGetAddr =
993 MP.getExecutionSession().intern(SymName: "___orc_rt_elfnix_tlsdesc_resolver");
994 Sym->setName(std::move(TLSGetAddr));
995 }
996 }
997
998 auto *TLSInfoEntrySection = G.findSectionByName(Name: "$__TLSINFO");
999
1000 if (TLSInfoEntrySection) {
1001 std::optional<uint64_t> Key;
1002 {
1003 std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
1004 auto I = MP.JITDylibToPThreadKey.find(Val: &JD);
1005 if (I != MP.JITDylibToPThreadKey.end())
1006 Key = I->second;
1007 }
1008 if (!Key) {
1009 if (auto KeyOrErr = MP.createPThreadKey())
1010 Key = *KeyOrErr;
1011 else
1012 return KeyOrErr.takeError();
1013 }
1014
1015 uint64_t PlatformKeyBits =
1016 support::endian::byte_swap(value: *Key, endian: G.getEndianness());
1017
1018 for (auto *B : TLSInfoEntrySection->blocks()) {
1019 // FIXME: The TLS descriptor byte length may different with different
1020 // ISA
1021 assert(B->getSize() == (G.getPointerSize() * 2) &&
1022 "TLS descriptor must be 2 words length");
1023 auto TLSInfoEntryContent = B->getMutableContent(G);
1024 memcpy(dest: TLSInfoEntryContent.data(), src: &PlatformKeyBits, n: G.getPointerSize());
1025 }
1026 }
1027
1028 return Error::success();
1029}
1030
1031} // End namespace orc.
1032} // End namespace llvm.
1033