1//===----- ELFNixPlatform.cpp - Utilities for executing ELFNix in Orc -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/Orc/ELFNixPlatform.h"
10
11#include "llvm/ExecutionEngine/JITLink/aarch64.h"
12#include "llvm/ExecutionEngine/JITLink/loongarch.h"
13#include "llvm/ExecutionEngine/JITLink/ppc64.h"
14#include "llvm/ExecutionEngine/JITLink/systemz.h"
15#include "llvm/ExecutionEngine/JITLink/x86_64.h"
16#include "llvm/ExecutionEngine/Orc/AbsoluteSymbols.h"
17#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
18#include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h"
19#include "llvm/Support/Debug.h"
20#include <optional>
21
22#define DEBUG_TYPE "orc"
23
24using namespace llvm;
25using namespace llvm::orc;
26using namespace llvm::orc::shared;
27
28namespace {
29
30template <typename SPSSerializer, typename... ArgTs>
31shared::WrapperFunctionCall::ArgDataBufferType
32getArgDataBufferType(const ArgTs &...Args) {
33 shared::WrapperFunctionCall::ArgDataBufferType ArgData;
34 ArgData.resize(SPSSerializer::size(Args...));
35 SPSOutputBuffer OB(ArgData.empty() ? nullptr : ArgData.data(),
36 ArgData.size());
37 if (SPSSerializer::serialize(OB, Args...))
38 return ArgData;
39 return {};
40}
41
42std::unique_ptr<jitlink::LinkGraph> createPlatformGraph(ELFNixPlatform &MOP,
43 std::string Name) {
44 auto &ES = MOP.getExecutionSession();
45 return std::make_unique<jitlink::LinkGraph>(
46 args: std::move(Name), args: ES.getSymbolStringPool(), args: ES.getTargetTriple(),
47 args: SubtargetFeatures(), args&: jitlink::getGenericEdgeKindName);
48}
49
50// Creates a Bootstrap-Complete LinkGraph to run deferred actions.
51class ELFNixPlatformCompleteBootstrapMaterializationUnit
52 : public MaterializationUnit {
53public:
54 ELFNixPlatformCompleteBootstrapMaterializationUnit(
55 ELFNixPlatform &MOP, StringRef PlatformJDName,
56 SymbolStringPtr CompleteBootstrapSymbol, DeferredRuntimeFnMap DeferredAAs,
57 ExecutorAddr ELFNixHeaderAddr, ExecutorAddr PlatformBootstrap,
58 ExecutorAddr PlatformShutdown, ExecutorAddr RegisterJITDylib,
59 ExecutorAddr DeregisterJITDylib)
60 : MaterializationUnit(
61 {{{CompleteBootstrapSymbol, JITSymbolFlags::None}}, nullptr}),
62 MOP(MOP), PlatformJDName(PlatformJDName),
63 CompleteBootstrapSymbol(std::move(CompleteBootstrapSymbol)),
64 DeferredAAsMap(std::move(DeferredAAs)),
65 ELFNixHeaderAddr(ELFNixHeaderAddr),
66 PlatformBootstrap(PlatformBootstrap),
67 PlatformShutdown(PlatformShutdown), RegisterJITDylib(RegisterJITDylib),
68 DeregisterJITDylib(DeregisterJITDylib) {}
69
70 StringRef getName() const override {
71 return "ELFNixPlatformCompleteBootstrap";
72 }
73
74 void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
75 using namespace jitlink;
76 auto G = createPlatformGraph(MOP, Name: "<OrcRTCompleteBootstrap>");
77 auto &PlaceholderSection =
78 G->createSection(Name: "__orc_rt_cplt_bs", Prot: MemProt::Read);
79 auto &PlaceholderBlock =
80 G->createZeroFillBlock(Parent&: PlaceholderSection, Size: 1, Address: ExecutorAddr(), Alignment: 1, AlignmentOffset: 0);
81 G->addDefinedSymbol(Content&: PlaceholderBlock, Offset: 0, Name: *CompleteBootstrapSymbol, Size: 1,
82 L: Linkage::Strong, S: Scope::Hidden, IsCallable: false, IsLive: true);
83
84 // 1. Bootstrap the platform support code.
85 G->allocActions().push_back(
86 x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
87 FnAddr: PlatformBootstrap, Args: ELFNixHeaderAddr)),
88 .Dealloc: cantFail(
89 ValOrErr: WrapperFunctionCall::Create<SPSArgList<>>(FnAddr: PlatformShutdown))});
90
91 // 2. Register the platform JITDylib.
92 G->allocActions().push_back(
93 x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<
94 SPSArgList<SPSString, SPSExecutorAddr>>(
95 FnAddr: RegisterJITDylib, Args: PlatformJDName, Args: ELFNixHeaderAddr)),
96 .Dealloc: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
97 FnAddr: DeregisterJITDylib, Args: ELFNixHeaderAddr))});
98
99 // 4. Add the deferred actions to the graph.
100 for (auto &[Fn, CallDatas] : DeferredAAsMap) {
101 for (auto &CallData : CallDatas) {
102 G->allocActions().push_back(
103 x: {.Finalize: WrapperFunctionCall(Fn.first->Addr, std::move(CallData.first)),
104 .Dealloc: WrapperFunctionCall(Fn.second->Addr, std::move(CallData.second))});
105 }
106 }
107
108 MOP.getObjectLinkingLayer().emit(R: std::move(R), G: std::move(G));
109 }
110
111 void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
112
113private:
114 ELFNixPlatform &MOP;
115 StringRef PlatformJDName;
116 SymbolStringPtr CompleteBootstrapSymbol;
117 DeferredRuntimeFnMap DeferredAAsMap;
118 ExecutorAddr ELFNixHeaderAddr;
119 ExecutorAddr PlatformBootstrap;
120 ExecutorAddr PlatformShutdown;
121 ExecutorAddr RegisterJITDylib;
122 ExecutorAddr DeregisterJITDylib;
123};
124
125class DSOHandleMaterializationUnit : public MaterializationUnit {
126public:
127 DSOHandleMaterializationUnit(ELFNixPlatform &ENP,
128 const SymbolStringPtr &DSOHandleSymbol)
129 : MaterializationUnit(
130 createDSOHandleSectionInterface(ENP, DSOHandleSymbol)),
131 ENP(ENP) {}
132
133 StringRef getName() const override { return "DSOHandleMU"; }
134
135 void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
136
137 auto &ES = ENP.getExecutionSession();
138
139 jitlink::Edge::Kind EdgeKind;
140
141 switch (ES.getTargetTriple().getArch()) {
142 case Triple::x86_64:
143 EdgeKind = jitlink::x86_64::Pointer64;
144 break;
145 case Triple::aarch64:
146 EdgeKind = jitlink::aarch64::Pointer64;
147 break;
148 case Triple::ppc64:
149 EdgeKind = jitlink::ppc64::Pointer64;
150 break;
151 case Triple::ppc64le:
152 EdgeKind = jitlink::ppc64::Pointer64;
153 break;
154 case Triple::loongarch64:
155 EdgeKind = jitlink::loongarch::Pointer64;
156 break;
157 case Triple::systemz:
158 EdgeKind = jitlink::systemz::Pointer64;
159 break;
160 default:
161 llvm_unreachable("Unrecognized architecture");
162 }
163
164 // void *__dso_handle = &__dso_handle;
165 auto G = std::make_unique<jitlink::LinkGraph>(
166 args: "<DSOHandleMU>", args: ES.getSymbolStringPool(), args: ES.getTargetTriple(),
167 args: SubtargetFeatures(), args&: jitlink::getGenericEdgeKindName);
168 auto &DSOHandleSection =
169 G->createSection(Name: ".data.__dso_handle", Prot: MemProt::Read);
170 auto &DSOHandleBlock = G->createContentBlock(
171 Parent&: DSOHandleSection, Content: getDSOHandleContent(PointerSize: G->getPointerSize()),
172 Address: orc::ExecutorAddr(), Alignment: 8, AlignmentOffset: 0);
173 auto &DSOHandleSymbol = G->addDefinedSymbol(
174 Content&: DSOHandleBlock, Offset: 0, Name: *R->getInitializerSymbol(), Size: DSOHandleBlock.getSize(),
175 L: jitlink::Linkage::Strong, S: jitlink::Scope::Default, IsCallable: false, IsLive: true);
176 DSOHandleBlock.addEdge(K: EdgeKind, Offset: 0, Target&: DSOHandleSymbol, Addend: 0);
177
178 ENP.getObjectLinkingLayer().emit(R: std::move(R), G: std::move(G));
179 }
180
181 void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
182
183private:
184 static MaterializationUnit::Interface
185 createDSOHandleSectionInterface(ELFNixPlatform &ENP,
186 const SymbolStringPtr &DSOHandleSymbol) {
187 SymbolFlagsMap SymbolFlags;
188 SymbolFlags[DSOHandleSymbol] = JITSymbolFlags::Exported;
189 return MaterializationUnit::Interface(std::move(SymbolFlags),
190 DSOHandleSymbol);
191 }
192
193 ArrayRef<char> getDSOHandleContent(size_t PointerSize) {
194 static const char Content[8] = {0};
195 assert(PointerSize <= sizeof Content);
196 return {Content, PointerSize};
197 }
198
199 ELFNixPlatform &ENP;
200};
201
202} // end anonymous namespace
203
204namespace llvm {
205namespace orc {
206
207Expected<std::unique_ptr<ELFNixPlatform>>
208ELFNixPlatform::Create(ObjectLinkingLayer &ObjLinkingLayer,
209 JITDylib &PlatformJD,
210 std::unique_ptr<DefinitionGenerator> OrcRuntime,
211 std::optional<SymbolAliasMap> RuntimeAliases) {
212
213 auto &ES = ObjLinkingLayer.getExecutionSession();
214
215 // If the target is not supported then bail out immediately.
216 if (!supportedTarget(TT: ES.getTargetTriple()))
217 return make_error<StringError>(Args: "Unsupported ELFNixPlatform triple: " +
218 ES.getTargetTriple().str(),
219 Args: inconvertibleErrorCode());
220
221 auto &EPC = ES.getExecutorProcessControl();
222
223 // Create default aliases if the caller didn't supply any.
224 if (!RuntimeAliases) {
225 auto StandardRuntimeAliases = standardPlatformAliases(ES, PlatformJD);
226 if (!StandardRuntimeAliases)
227 return StandardRuntimeAliases.takeError();
228 RuntimeAliases = std::move(*StandardRuntimeAliases);
229 }
230
231 // Define the aliases.
232 if (auto Err = PlatformJD.define(MU: symbolAliases(Aliases: std::move(*RuntimeAliases))))
233 return std::move(Err);
234
235 // Add JIT-dispatch function support symbols.
236 if (auto Err = PlatformJD.define(
237 MU: absoluteSymbols(Symbols: {{ES.intern(SymName: "__orc_rt_jit_dispatch"),
238 {EPC.getJITDispatchInfo().JITDispatchFunction,
239 JITSymbolFlags::Exported}},
240 {ES.intern(SymName: "__orc_rt_jit_dispatch_ctx"),
241 {EPC.getJITDispatchInfo().JITDispatchContext,
242 JITSymbolFlags::Exported}}})))
243 return std::move(Err);
244
245 // Create the instance.
246 Error Err = Error::success();
247 auto P = std::unique_ptr<ELFNixPlatform>(new ELFNixPlatform(
248 ObjLinkingLayer, PlatformJD, std::move(OrcRuntime), Err));
249 if (Err)
250 return std::move(Err);
251 return std::move(P);
252}
253
254Expected<std::unique_ptr<ELFNixPlatform>>
255ELFNixPlatform::Create(ObjectLinkingLayer &ObjLinkingLayer,
256 JITDylib &PlatformJD, const char *OrcRuntimePath,
257 std::optional<SymbolAliasMap> RuntimeAliases) {
258
259 // Create a generator for the ORC runtime archive.
260 auto OrcRuntimeArchiveGenerator =
261 StaticLibraryDefinitionGenerator::Load(L&: ObjLinkingLayer, FileName: OrcRuntimePath);
262 if (!OrcRuntimeArchiveGenerator)
263 return OrcRuntimeArchiveGenerator.takeError();
264
265 return Create(ObjLinkingLayer, PlatformJD,
266 OrcRuntime: std::move(*OrcRuntimeArchiveGenerator),
267 RuntimeAliases: std::move(RuntimeAliases));
268}
269
270Error ELFNixPlatform::setupJITDylib(JITDylib &JD) {
271 if (auto Err = JD.define(MU: std::make_unique<DSOHandleMaterializationUnit>(
272 args&: *this, args&: DSOHandleSymbol)))
273 return Err;
274
275 return ES.lookup(SearchOrder: {&JD}, Symbol: DSOHandleSymbol).takeError();
276}
277
278Error ELFNixPlatform::teardownJITDylib(JITDylib &JD) {
279 std::lock_guard<std::mutex> Lock(PlatformMutex);
280 auto I = JITDylibToHandleAddr.find(Val: &JD);
281 if (I != JITDylibToHandleAddr.end()) {
282 assert(HandleAddrToJITDylib.count(I->second) &&
283 "HandleAddrToJITDylib missing entry");
284 HandleAddrToJITDylib.erase(Val: I->second);
285 JITDylibToHandleAddr.erase(I);
286 }
287 return Error::success();
288}
289
290Error ELFNixPlatform::notifyAdding(ResourceTracker &RT,
291 const MaterializationUnit &MU) {
292
293 auto &JD = RT.getJITDylib();
294 const auto &InitSym = MU.getInitializerSymbol();
295 if (!InitSym)
296 return Error::success();
297
298 RegisteredInitSymbols[&JD].add(Name: InitSym,
299 Flags: SymbolLookupFlags::WeaklyReferencedSymbol);
300 LLVM_DEBUG({
301 dbgs() << "ELFNixPlatform: Registered init symbol " << *InitSym
302 << " for MU " << MU.getName() << "\n";
303 });
304 return Error::success();
305}
306
307Error ELFNixPlatform::notifyRemoving(ResourceTracker &RT) {
308 llvm_unreachable("Not supported yet");
309}
310
311static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases,
312 ArrayRef<std::pair<const char *, const char *>> AL) {
313 for (auto &KV : AL) {
314 auto AliasName = ES.intern(SymName: KV.first);
315 assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map");
316 Aliases[std::move(AliasName)] = {ES.intern(SymName: KV.second),
317 JITSymbolFlags::Exported};
318 }
319}
320
321Expected<SymbolAliasMap>
322ELFNixPlatform::standardPlatformAliases(ExecutionSession &ES,
323 JITDylib &PlatformJD) {
324 SymbolAliasMap Aliases;
325 addAliases(ES, Aliases, AL: requiredCXXAliases());
326 addAliases(ES, Aliases, AL: standardRuntimeUtilityAliases());
327 addAliases(ES, Aliases, AL: standardLazyCompilationAliases());
328 return Aliases;
329}
330
331ArrayRef<std::pair<const char *, const char *>>
332ELFNixPlatform::requiredCXXAliases() {
333 static const std::pair<const char *, const char *> RequiredCXXAliases[] = {
334 {"__cxa_atexit", "__orc_rt_elfnix_cxa_atexit"},
335 {"atexit", "__orc_rt_elfnix_atexit"}};
336
337 return ArrayRef<std::pair<const char *, const char *>>(RequiredCXXAliases);
338}
339
340ArrayRef<std::pair<const char *, const char *>>
341ELFNixPlatform::standardRuntimeUtilityAliases() {
342 static const std::pair<const char *, const char *>
343 StandardRuntimeUtilityAliases[] = {
344 {"__orc_rt_run_program", "__orc_rt_elfnix_run_program"},
345 {"__orc_rt_jit_dlerror", "__orc_rt_elfnix_jit_dlerror"},
346 {"__orc_rt_jit_dlopen", "__orc_rt_elfnix_jit_dlopen"},
347 {"__orc_rt_jit_dlupdate", "__orc_rt_elfnix_jit_dlupdate"},
348 {"__orc_rt_jit_dlclose", "__orc_rt_elfnix_jit_dlclose"},
349 {"__orc_rt_jit_dlsym", "__orc_rt_elfnix_jit_dlsym"},
350 {"__orc_rt_log_error", "__orc_rt_log_error_to_stderr"}};
351
352 return ArrayRef<std::pair<const char *, const char *>>(
353 StandardRuntimeUtilityAliases);
354}
355
356ArrayRef<std::pair<const char *, const char *>>
357ELFNixPlatform::standardLazyCompilationAliases() {
358 static const std::pair<const char *, const char *>
359 StandardLazyCompilationAliases[] = {
360 {"__orc_rt_reenter", "__orc_rt_sysv_reenter"}};
361
362 return ArrayRef<std::pair<const char *, const char *>>(
363 StandardLazyCompilationAliases);
364}
365
366bool ELFNixPlatform::supportedTarget(const Triple &TT) {
367 switch (TT.getArch()) {
368 case Triple::x86_64:
369 case Triple::aarch64:
370 // FIXME: jitlink for ppc64 hasn't been well tested, leave it unsupported
371 // right now.
372 case Triple::ppc64le:
373 case Triple::loongarch64:
374 case Triple::systemz:
375 return true;
376 default:
377 return false;
378 }
379}
380
381ELFNixPlatform::ELFNixPlatform(
382 ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD,
383 std::unique_ptr<DefinitionGenerator> OrcRuntimeGenerator, Error &Err)
384 : ES(ObjLinkingLayer.getExecutionSession()), PlatformJD(PlatformJD),
385 ObjLinkingLayer(ObjLinkingLayer),
386 DSOHandleSymbol(ES.intern(SymName: "__dso_handle")) {
387 ErrorAsOutParameter _(Err);
388 ObjLinkingLayer.addPlugin(P: std::make_unique<ELFNixPlatformPlugin>(args&: *this));
389
390 PlatformJD.addGenerator(DefGenerator: std::move(OrcRuntimeGenerator));
391
392 BootstrapInfo BI;
393 Bootstrap = &BI;
394
395 // PlatformJD hasn't been 'set-up' by the platform yet (since we're creating
396 // the platform now), so set it up.
397 if (auto E2 = setupJITDylib(PlatformJD)) {
398 Err = std::move(E2);
399 return;
400 }
401
402 // Step (2) Request runtime registration functions to trigger
403 // materialization..
404 if ((Err = ES.lookup(
405 SearchOrder: makeJITDylibSearchOrder(JDs: &PlatformJD),
406 Symbols: SymbolLookupSet(
407 {PlatformBootstrap.Name, PlatformShutdown.Name,
408 RegisterJITDylib.Name, DeregisterJITDylib.Name,
409 RegisterInitSections.Name, DeregisterInitSections.Name,
410 RegisterFiniSections.Name, DeregisterFiniSections.Name,
411 RegisterObjectSections.Name,
412 DeregisterObjectSections.Name, CreatePThreadKey.Name}))
413 .takeError()))
414 return;
415
416 // Step (3) Wait for any incidental linker work to complete.
417 {
418 std::unique_lock<std::mutex> Lock(BI.Mutex);
419 BI.CV.wait(lock&: Lock, p: [&]() { return BI.ActiveGraphs == 0; });
420 Bootstrap = nullptr;
421 }
422
423 // Step (4) Add complete-bootstrap materialization unit and request.
424 auto BootstrapCompleteSymbol =
425 ES.intern(SymName: "__orc_rt_elfnix_complete_bootstrap");
426 if ((Err = PlatformJD.define(
427 MU: std::make_unique<ELFNixPlatformCompleteBootstrapMaterializationUnit>(
428 args&: *this, args: PlatformJD.getName(), args&: BootstrapCompleteSymbol,
429 args: std::move(BI.DeferredRTFnMap), args&: BI.ELFNixHeaderAddr,
430 args&: PlatformBootstrap.Addr, args&: PlatformShutdown.Addr,
431 args&: RegisterJITDylib.Addr, args&: DeregisterJITDylib.Addr))))
432 return;
433 if ((Err = ES.lookup(SearchOrder: makeJITDylibSearchOrder(
434 JDs: &PlatformJD, Flags: JITDylibLookupFlags::MatchAllSymbols),
435 Symbol: std::move(BootstrapCompleteSymbol))
436 .takeError()))
437 return;
438
439 // Associate wrapper function tags with JIT-side function implementations.
440 if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) {
441 Err = std::move(E2);
442 return;
443 }
444}
445
446Error ELFNixPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) {
447 ExecutionSession::JITDispatchHandlerAssociationMap WFs;
448
449 using RecordInitializersSPSSig =
450 SPSExpected<SPSELFNixJITDylibDepInfoMap>(SPSExecutorAddr);
451 WFs[ES.intern(SymName: "__orc_rt_elfnix_push_initializers_tag")] =
452 ES.wrapAsyncWithSPS<RecordInitializersSPSSig>(
453 Instance: this, Method: &ELFNixPlatform::rt_recordInitializers);
454
455 using LookupSymbolSPSSig =
456 SPSExpected<SPSExecutorAddr>(SPSExecutorAddr, SPSString);
457 WFs[ES.intern(SymName: "__orc_rt_elfnix_symbol_lookup_tag")] =
458 ES.wrapAsyncWithSPS<LookupSymbolSPSSig>(Instance: this,
459 Method: &ELFNixPlatform::rt_lookupSymbol);
460
461 return ES.registerJITDispatchHandlers(JD&: PlatformJD, WFs: std::move(WFs));
462}
463
464void ELFNixPlatform::pushInitializersLoop(
465 PushInitializersSendResultFn SendResult, JITDylibSP JD) {
466 DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols;
467 DenseMap<JITDylib *, SmallVector<JITDylib *>> JDDepMap;
468 SmallVector<JITDylib *, 16> Worklist({JD.get()});
469
470 ES.runSessionLocked(F: [&]() {
471 while (!Worklist.empty()) {
472 // FIXME: Check for defunct dylibs.
473
474 auto DepJD = Worklist.back();
475 Worklist.pop_back();
476
477 // If we've already visited this JITDylib on this iteration then continue.
478 auto [It, Inserted] = JDDepMap.try_emplace(Key: DepJD);
479 if (!Inserted)
480 continue;
481
482 // Add dep info.
483 auto &DM = It->second;
484 DepJD->withLinkOrderDo(F: [&](const JITDylibSearchOrder &O) {
485 for (auto &KV : O) {
486 if (KV.first == DepJD)
487 continue;
488 DM.push_back(Elt: KV.first);
489 Worklist.push_back(Elt: KV.first);
490 }
491 });
492
493 // Add any registered init symbols.
494 auto RISItr = RegisteredInitSymbols.find(Val: DepJD);
495 if (RISItr != RegisteredInitSymbols.end()) {
496 NewInitSymbols[DepJD] = std::move(RISItr->second);
497 RegisteredInitSymbols.erase(I: RISItr);
498 }
499 }
500 });
501
502 // If there are no further init symbols to look up then send the link order
503 // (as a list of header addresses) to the caller.
504 if (NewInitSymbols.empty()) {
505
506 // To make the list intelligible to the runtime we need to convert all
507 // JITDylib pointers to their header addresses. Only include JITDylibs
508 // that appear in the JITDylibToHandleAddr map (i.e. those that have been
509 // through setupJITDylib) -- bare JITDylibs aren't managed by the platform.
510 DenseMap<JITDylib *, ExecutorAddr> HeaderAddrs;
511 HeaderAddrs.reserve(NumEntries: JDDepMap.size());
512 {
513 std::lock_guard<std::mutex> Lock(PlatformMutex);
514 for (auto &KV : JDDepMap) {
515 auto I = JITDylibToHandleAddr.find(Val: KV.first);
516 if (I != JITDylibToHandleAddr.end())
517 HeaderAddrs[KV.first] = I->second;
518 }
519 }
520
521 // Build the dep info map to return.
522 ELFNixJITDylibDepInfoMap DIM;
523 DIM.reserve(n: JDDepMap.size());
524 for (auto &KV : JDDepMap) {
525 auto HI = HeaderAddrs.find(Val: KV.first);
526 // Skip unmanaged JITDylibs.
527 if (HI == HeaderAddrs.end())
528 continue;
529 auto H = HI->second;
530 ELFNixJITDylibDepInfo DepInfo;
531 for (auto &Dep : KV.second) {
532 auto HJ = HeaderAddrs.find(Val: Dep);
533 if (HJ != HeaderAddrs.end())
534 DepInfo.push_back(x: HJ->second);
535 }
536 DIM.push_back(x: std::make_pair(x&: H, y: std::move(DepInfo)));
537 }
538 SendResult(DIM);
539 return;
540 }
541
542 // Otherwise issue a lookup and re-run this phase when it completes.
543 lookupInitSymbolsAsync(
544 OnComplete: [this, SendResult = std::move(SendResult), JD](Error Err) mutable {
545 if (Err)
546 SendResult(std::move(Err));
547 else
548 pushInitializersLoop(SendResult: std::move(SendResult), JD);
549 },
550 ES, InitSyms: std::move(NewInitSymbols));
551}
552
553void ELFNixPlatform::rt_recordInitializers(
554 PushInitializersSendResultFn SendResult, ExecutorAddr JDHeaderAddr) {
555 JITDylibSP JD;
556 {
557 std::lock_guard<std::mutex> Lock(PlatformMutex);
558 auto I = HandleAddrToJITDylib.find(Val: JDHeaderAddr);
559 if (I != HandleAddrToJITDylib.end())
560 JD = I->second;
561 }
562
563 LLVM_DEBUG({
564 dbgs() << "ELFNixPlatform::rt_recordInitializers(" << JDHeaderAddr << ") ";
565 if (JD)
566 dbgs() << "pushing initializers for " << JD->getName() << "\n";
567 else
568 dbgs() << "No JITDylib for header address.\n";
569 });
570
571 if (!JD) {
572 SendResult(make_error<StringError>(Args: "No JITDylib with header addr " +
573 formatv(Fmt: "{0:x}", Vals&: JDHeaderAddr),
574 Args: inconvertibleErrorCode()));
575 return;
576 }
577
578 pushInitializersLoop(SendResult: std::move(SendResult), JD);
579}
580
581void ELFNixPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult,
582 ExecutorAddr Handle,
583 StringRef SymbolName) {
584 LLVM_DEBUG({
585 dbgs() << "ELFNixPlatform::rt_lookupSymbol(\"" << Handle << "\")\n";
586 });
587
588 JITDylib *JD = nullptr;
589
590 {
591 std::lock_guard<std::mutex> Lock(PlatformMutex);
592 auto I = HandleAddrToJITDylib.find(Val: Handle);
593 if (I != HandleAddrToJITDylib.end())
594 JD = I->second;
595 }
596
597 if (!JD) {
598 LLVM_DEBUG(dbgs() << " No JITDylib for handle " << Handle << "\n");
599 SendResult(make_error<StringError>(Args: "No JITDylib associated with handle " +
600 formatv(Fmt: "{0:x}", Vals&: Handle),
601 Args: inconvertibleErrorCode()));
602 return;
603 }
604
605 // Use functor class to work around XL build compiler issue on AIX.
606 class RtLookupNotifyComplete {
607 public:
608 RtLookupNotifyComplete(SendSymbolAddressFn &&SendResult)
609 : SendResult(std::move(SendResult)) {}
610 void operator()(Expected<SymbolMap> Result) {
611 if (Result) {
612 assert(Result->size() == 1 && "Unexpected result map count");
613 SendResult(Result->begin()->second.getAddress());
614 } else {
615 SendResult(Result.takeError());
616 }
617 }
618
619 private:
620 SendSymbolAddressFn SendResult;
621 };
622
623 ES.lookup(
624 K: LookupKind::DLSym, SearchOrder: {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}},
625 Symbols: SymbolLookupSet(ES.intern(SymName: SymbolName)), RequiredState: SymbolState::Ready,
626 NotifyComplete: RtLookupNotifyComplete(std::move(SendResult)), RegisterDependencies: NoDependenciesToRegister);
627}
628
629Error ELFNixPlatform::ELFNixPlatformPlugin::bootstrapPipelineStart(
630 jitlink::LinkGraph &G) {
631 // Increment the active graphs count in BootstrapInfo.
632 std::lock_guard<std::mutex> Lock(MP.Bootstrap.load()->Mutex);
633 ++MP.Bootstrap.load()->ActiveGraphs;
634 return Error::success();
635}
636
637Error ELFNixPlatform::ELFNixPlatformPlugin::
638 bootstrapPipelineRecordRuntimeFunctions(jitlink::LinkGraph &G) {
639 // Record bootstrap function names.
640 std::pair<StringRef, ExecutorAddr *> RuntimeSymbols[] = {
641 {*MP.DSOHandleSymbol, &MP.Bootstrap.load()->ELFNixHeaderAddr},
642 {*MP.PlatformBootstrap.Name, &MP.PlatformBootstrap.Addr},
643 {*MP.PlatformShutdown.Name, &MP.PlatformShutdown.Addr},
644 {*MP.RegisterJITDylib.Name, &MP.RegisterJITDylib.Addr},
645 {*MP.DeregisterJITDylib.Name, &MP.DeregisterJITDylib.Addr},
646 {*MP.RegisterObjectSections.Name, &MP.RegisterObjectSections.Addr},
647 {*MP.DeregisterObjectSections.Name, &MP.DeregisterObjectSections.Addr},
648 {*MP.RegisterInitSections.Name, &MP.RegisterInitSections.Addr},
649 {*MP.DeregisterInitSections.Name, &MP.DeregisterInitSections.Addr},
650 {*MP.RegisterFiniSections.Name, &MP.RegisterFiniSections.Addr},
651 {*MP.DeregisterFiniSections.Name, &MP.DeregisterFiniSections.Addr},
652 {*MP.CreatePThreadKey.Name, &MP.CreatePThreadKey.Addr}};
653
654 bool RegisterELFNixHeader = false;
655
656 for (auto *Sym : G.defined_symbols()) {
657 for (auto &RTSym : RuntimeSymbols) {
658 if (Sym->hasName() && *Sym->getName() == RTSym.first) {
659 if (*RTSym.second)
660 return make_error<StringError>(
661 Args: "Duplicate " + RTSym.first +
662 " detected during ELFNixPlatform bootstrap",
663 Args: inconvertibleErrorCode());
664
665 if (*Sym->getName() == *MP.DSOHandleSymbol)
666 RegisterELFNixHeader = true;
667
668 *RTSym.second = Sym->getAddress();
669 }
670 }
671 }
672
673 if (RegisterELFNixHeader) {
674 // If this graph defines the elfnix header symbol then create the internal
675 // mapping between it and PlatformJD.
676 std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
677 MP.JITDylibToHandleAddr[&MP.PlatformJD] =
678 MP.Bootstrap.load()->ELFNixHeaderAddr;
679 MP.HandleAddrToJITDylib[MP.Bootstrap.load()->ELFNixHeaderAddr] =
680 &MP.PlatformJD;
681 }
682
683 return Error::success();
684}
685
686Error ELFNixPlatform::ELFNixPlatformPlugin::bootstrapPipelineEnd(
687 jitlink::LinkGraph &G) {
688 std::lock_guard<std::mutex> Lock(MP.Bootstrap.load()->Mutex);
689 assert(MP.Bootstrap && "DeferredAAs reset before bootstrap completed");
690 --MP.Bootstrap.load()->ActiveGraphs;
691 // Notify Bootstrap->CV while holding the mutex because the mutex is
692 // also keeping Bootstrap->CV alive.
693 if (MP.Bootstrap.load()->ActiveGraphs == 0)
694 MP.Bootstrap.load()->CV.notify_all();
695 return Error::success();
696}
697
698Error ELFNixPlatform::registerPerObjectSections(
699 jitlink::LinkGraph &G, const ELFPerObjectSectionsToRegister &POSR,
700 bool IsBootstrapping) {
701 using SPSRegisterPerObjSectionsArgs =
702 SPSArgList<SPSELFPerObjectSectionsToRegister>;
703
704 if (LLVM_UNLIKELY(IsBootstrapping)) {
705 Bootstrap.load()->addArgumentsToRTFnMap(
706 func1: &RegisterObjectSections, func2: &DeregisterObjectSections,
707 arg1: getArgDataBufferType<SPSRegisterPerObjSectionsArgs>(Args: POSR),
708 arg2: getArgDataBufferType<SPSRegisterPerObjSectionsArgs>(Args: POSR));
709 return Error::success();
710 }
711
712 G.allocActions().push_back(
713 x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSRegisterPerObjSectionsArgs>(
714 FnAddr: RegisterObjectSections.Addr, Args: POSR)),
715 .Dealloc: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSRegisterPerObjSectionsArgs>(
716 FnAddr: DeregisterObjectSections.Addr, Args: POSR))});
717
718 return Error::success();
719}
720
721Expected<uint64_t> ELFNixPlatform::createPThreadKey() {
722 if (!CreatePThreadKey.Addr)
723 return make_error<StringError>(
724 Args: "Attempting to create pthread key in target, but runtime support has "
725 "not been loaded yet",
726 Args: inconvertibleErrorCode());
727
728 Expected<uint64_t> Result(0);
729 if (auto Err = ES.callSPSWrapper<SPSExpected<uint64_t>(void)>(
730 WrapperFnAddr: CreatePThreadKey.Addr, WrapperCallArgs&: Result))
731 return std::move(Err);
732 return Result;
733}
734
735void ELFNixPlatform::ELFNixPlatformPlugin::modifyPassConfig(
736 MaterializationResponsibility &MR, jitlink::LinkGraph &LG,
737 jitlink::PassConfiguration &Config) {
738 using namespace jitlink;
739
740 bool InBootstrapPhase =
741 &MR.getTargetJITDylib() == &MP.PlatformJD && MP.Bootstrap;
742
743 // If we're in the bootstrap phase then increment the active graphs.
744 if (InBootstrapPhase) {
745 Config.PrePrunePasses.push_back(
746 x: [this](LinkGraph &G) { return bootstrapPipelineStart(G); });
747 Config.PostAllocationPasses.push_back(x: [this](LinkGraph &G) {
748 return bootstrapPipelineRecordRuntimeFunctions(G);
749 });
750 }
751
752 // If the initializer symbol is the __dso_handle symbol then just add
753 // the DSO handle support passes.
754 if (auto InitSymbol = MR.getInitializerSymbol()) {
755 if (InitSymbol == MP.DSOHandleSymbol && !InBootstrapPhase) {
756 addDSOHandleSupportPasses(MR, Config);
757 // The DSOHandle materialization unit doesn't require any other
758 // support, so we can bail out early.
759 return;
760 }
761
762 /// Preserve init sections.
763 Config.PrePrunePasses.push_back(
764 x: [this, &MR](jitlink::LinkGraph &G) -> Error {
765 if (auto Err = preserveInitSections(G, MR))
766 return Err;
767 return Error::success();
768 });
769 }
770
771 // Add passes for eh-frame and TLV support.
772 addEHAndTLVSupportPasses(MR, Config, IsBootstrapping: InBootstrapPhase);
773
774 // If the object contains initializers then add passes to record them.
775 Config.PostFixupPasses.push_back(x: [this, &JD = MR.getTargetJITDylib(),
776 InBootstrapPhase](jitlink::LinkGraph &G) {
777 return registerInitSections(G, JD, IsBootstrapping: InBootstrapPhase);
778 });
779
780 // If the object contains finalizers then add passes to record them.
781 Config.PostFixupPasses.push_back(x: [this, &JD = MR.getTargetJITDylib(),
782 InBootstrapPhase](jitlink::LinkGraph &G) {
783 return registerFiniSections(G, JD, IsBootstrapping: InBootstrapPhase);
784 });
785
786 // If we're in the bootstrap phase then steal allocation actions and then
787 // decrement the active graphs.
788 if (InBootstrapPhase)
789 Config.PostFixupPasses.push_back(
790 x: [this](LinkGraph &G) { return bootstrapPipelineEnd(G); });
791}
792
793void ELFNixPlatform::ELFNixPlatformPlugin::addDSOHandleSupportPasses(
794 MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) {
795
796 Config.PostAllocationPasses.push_back(x: [this, &JD = MR.getTargetJITDylib()](
797 jitlink::LinkGraph &G) -> Error {
798 auto I = llvm::find_if(Range: G.defined_symbols(), P: [this](jitlink::Symbol *Sym) {
799 return Sym->getName() == MP.DSOHandleSymbol;
800 });
801 assert(I != G.defined_symbols().end() && "Missing DSO handle symbol");
802 {
803 std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
804 auto HandleAddr = (*I)->getAddress();
805 MP.HandleAddrToJITDylib[HandleAddr] = &JD;
806 MP.JITDylibToHandleAddr[&JD] = HandleAddr;
807
808 G.allocActions().push_back(
809 x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<
810 SPSArgList<SPSString, SPSExecutorAddr>>(
811 FnAddr: MP.RegisterJITDylib.Addr, Args: JD.getName(), Args: HandleAddr)),
812 .Dealloc: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
813 FnAddr: MP.DeregisterJITDylib.Addr, Args: HandleAddr))});
814 }
815 return Error::success();
816 });
817}
818
819void ELFNixPlatform::ELFNixPlatformPlugin::addEHAndTLVSupportPasses(
820 MaterializationResponsibility &MR, jitlink::PassConfiguration &Config,
821 bool IsBootstrapping) {
822
823 // Insert TLV lowering at the start of the PostPrunePasses, since we want
824 // it to run before GOT/PLT lowering.
825
826 // TODO: Check that before the fixTLVSectionsAndEdges pass, the GOT/PLT build
827 // pass has done. Because the TLS descriptor need to be allocate in GOT.
828 Config.PostPrunePasses.push_back(
829 x: [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
830 return fixTLVSectionsAndEdges(G, JD);
831 });
832
833 // Add a pass to register the final addresses of the eh-frame and TLV sections
834 // with the runtime.
835 Config.PostFixupPasses.push_back(x: [this, IsBootstrapping](
836 jitlink::LinkGraph &G) -> Error {
837 ELFPerObjectSectionsToRegister POSR;
838
839 if (auto *EHFrameSection = G.findSectionByName(Name: ELFEHFrameSectionName)) {
840 jitlink::SectionRange R(*EHFrameSection);
841 if (!R.empty())
842 POSR.EHFrameSection = R.getRange();
843 }
844
845 // Get a pointer to the thread data section if there is one. It will be used
846 // below.
847 jitlink::Section *ThreadDataSection =
848 G.findSectionByName(Name: ELFThreadDataSectionName);
849
850 // Handle thread BSS section if there is one.
851 if (auto *ThreadBSSSection = G.findSectionByName(Name: ELFThreadBSSSectionName)) {
852 // If there's already a thread data section in this graph then merge the
853 // thread BSS section content into it, otherwise just treat the thread
854 // BSS section as the thread data section.
855 if (ThreadDataSection)
856 G.mergeSections(DstSection&: *ThreadDataSection, SrcSection&: *ThreadBSSSection);
857 else
858 ThreadDataSection = ThreadBSSSection;
859 }
860
861 // Having merged thread BSS (if present) and thread data (if present),
862 // record the resulting section range.
863 if (ThreadDataSection) {
864 jitlink::SectionRange R(*ThreadDataSection);
865 if (!R.empty())
866 POSR.ThreadDataSection = R.getRange();
867 }
868
869 if (POSR.EHFrameSection.Start || POSR.ThreadDataSection.Start) {
870 if (auto Err = MP.registerPerObjectSections(G, POSR, IsBootstrapping))
871 return Err;
872 }
873
874 return Error::success();
875 });
876}
877
878Error ELFNixPlatform::ELFNixPlatformPlugin::preserveInitSections(
879 jitlink::LinkGraph &G, MaterializationResponsibility &MR) {
880
881 if (const auto &InitSymName = MR.getInitializerSymbol()) {
882
883 jitlink::Symbol *InitSym = nullptr;
884
885 for (auto &InitSection : G.sections()) {
886 // Skip non-init sections.
887 if (!isELFInitializerSection(SecName: InitSection.getName()) ||
888 InitSection.empty())
889 continue;
890
891 // Create the init symbol if it has not been created already and attach it
892 // to the first block.
893 if (!InitSym) {
894 auto &B = **InitSection.blocks().begin();
895 InitSym = &G.addDefinedSymbol(
896 Content&: B, Offset: 0, Name: *InitSymName, Size: B.getSize(), L: jitlink::Linkage::Strong,
897 S: jitlink::Scope::SideEffectsOnly, IsCallable: false, IsLive: true);
898 }
899
900 // Add keep-alive edges to anonymous symbols in all other init blocks.
901 for (auto *B : InitSection.blocks()) {
902 if (B == &InitSym->getBlock())
903 continue;
904
905 auto &S = G.addAnonymousSymbol(Content&: *B, Offset: 0, Size: B->getSize(), IsCallable: false, IsLive: true);
906 InitSym->getBlock().addEdge(K: jitlink::Edge::KeepAlive, Offset: 0, Target&: S, Addend: 0);
907 }
908 }
909
910 // Also preserve fini sections (.fini_array, .fini, .dtors)
911 for (auto &FiniSection : G.sections()) {
912 // Skip non-fini sections.
913 if (!isELFFinalizerSection(SecName: FiniSection.getName()) || FiniSection.empty())
914 continue;
915
916 // Create the init symbol if it has not been created already and attach it
917 // to the first fini block.
918 if (!InitSym) {
919 auto &B = **FiniSection.blocks().begin();
920 InitSym = &G.addDefinedSymbol(
921 Content&: B, Offset: 0, Name: *InitSymName, Size: B.getSize(), L: jitlink::Linkage::Strong,
922 S: jitlink::Scope::SideEffectsOnly, IsCallable: false, IsLive: true);
923 }
924
925 // Add keep-alive edges to anonymous symbols in all fini blocks.
926 for (auto *B : FiniSection.blocks()) {
927 if (B == &InitSym->getBlock())
928 continue;
929
930 auto &S = G.addAnonymousSymbol(Content&: *B, Offset: 0, Size: B->getSize(), IsCallable: false, IsLive: true);
931 InitSym->getBlock().addEdge(K: jitlink::Edge::KeepAlive, Offset: 0, Target&: S, Addend: 0);
932 }
933 }
934 }
935
936 return Error::success();
937}
938
939Error ELFNixPlatform::ELFNixPlatformPlugin::registerInitSections(
940 jitlink::LinkGraph &G, JITDylib &JD, bool IsBootstrapping) {
941 SmallVector<ExecutorAddrRange> ELFNixPlatformSecs;
942 LLVM_DEBUG(dbgs() << "ELFNixPlatform::registerInitSections\n");
943
944 SmallVector<jitlink::Section *> OrderedInitSections;
945 for (auto &Sec : G.sections())
946 if (isELFInitializerSection(SecName: Sec.getName()))
947 OrderedInitSections.push_back(Elt: &Sec);
948
949 // Helper to get section type and priority for sorting.
950 // Returns: {type_order, priority, has_priority}
951 // type_order: 0 = .init_array, 1 = .init, 2 = .ctors
952 auto getInitSectionInfo =
953 [](const jitlink::Section *Sec) -> std::tuple<int, uint64_t, bool> {
954 StringRef Name = Sec->getName();
955 if (Name.starts_with(Prefix: ".init_array")) {
956 StringRef PrioStr = Name;
957 uint64_t Prio = 0;
958 bool HasPrio = PrioStr.consume_front(Prefix: ".init_array.") &&
959 !PrioStr.getAsInteger(Radix: 10, Result&: Prio);
960 return {0, Prio, HasPrio};
961 }
962 if (Name.starts_with(Prefix: ".init"))
963 return {1, 0, false};
964 if (Name.starts_with(Prefix: ".ctors")) {
965 StringRef PrioStr = Name;
966 uint64_t Prio = 0;
967 bool HasPrio =
968 PrioStr.consume_front(Prefix: ".ctors.") && !PrioStr.getAsInteger(Radix: 10, Result&: Prio);
969 return {2, Prio, HasPrio};
970 }
971 return {3, 0, false};
972 };
973
974 // Sort init sections:
975 // 1. .init_array sections first (ascending priority - lower runs first)
976 // 2. .init sections next
977 // 3. .ctors sections last (descending priority - higher runs first, legacy
978 // behavior)
979 llvm::sort(C&: OrderedInitSections,
980 Comp: [&](const jitlink::Section *LHS, const jitlink::Section *RHS) {
981 auto [LType, LPrio, LHasPrio] = getInitSectionInfo(LHS);
982 auto [RType, RPrio, RHasPrio] = getInitSectionInfo(RHS);
983
984 if (LType != RType)
985 return LType < RType;
986
987 // Same type - sort by priority
988 if (LType == 0) {
989 // .init_array: ascending priority (lower priority number runs
990 // first)
991 if (LHasPrio && RHasPrio)
992 return LPrio < RPrio;
993 if (LHasPrio)
994 return true;
995 if (RHasPrio)
996 return false;
997 } else if (LType == 2) {
998 // .ctors: descending priority (higher priority number runs
999 // first)
1000 if (LHasPrio && RHasPrio)
1001 return LPrio > RPrio;
1002 if (LHasPrio)
1003 return true;
1004 if (RHasPrio)
1005 return false;
1006 }
1007 return LHS->getName() < RHS->getName();
1008 });
1009
1010 for (auto &Sec : OrderedInitSections)
1011 ELFNixPlatformSecs.push_back(Elt: jitlink::SectionRange(*Sec).getRange());
1012
1013 // Dump the scraped inits.
1014 LLVM_DEBUG({
1015 dbgs() << "ELFNixPlatform: Scraped " << G.getName() << " init sections:\n";
1016 for (auto &Sec : G.sections()) {
1017 jitlink::SectionRange R(Sec);
1018 dbgs() << " " << Sec.getName() << ": " << R.getRange() << "\n";
1019 }
1020 });
1021
1022 ExecutorAddr HeaderAddr;
1023 {
1024 std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
1025 auto I = MP.JITDylibToHandleAddr.find(Val: &JD);
1026 assert(I != MP.JITDylibToHandleAddr.end() && "No header registered for JD");
1027 assert(I->second && "Null header registered for JD");
1028 HeaderAddr = I->second;
1029 }
1030
1031 using SPSRegisterInitSectionsArgs =
1032 SPSArgList<SPSExecutorAddr, SPSSequence<SPSExecutorAddrRange>>;
1033
1034 if (LLVM_UNLIKELY(IsBootstrapping)) {
1035 MP.Bootstrap.load()->addArgumentsToRTFnMap(
1036 func1: &MP.RegisterInitSections, func2: &MP.DeregisterInitSections,
1037 arg1: getArgDataBufferType<SPSRegisterInitSectionsArgs>(Args: HeaderAddr,
1038 Args: ELFNixPlatformSecs),
1039 arg2: getArgDataBufferType<SPSRegisterInitSectionsArgs>(Args: HeaderAddr,
1040 Args: ELFNixPlatformSecs));
1041 return Error::success();
1042 }
1043
1044 G.allocActions().push_back(
1045 x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSRegisterInitSectionsArgs>(
1046 FnAddr: MP.RegisterInitSections.Addr, Args: HeaderAddr, Args: ELFNixPlatformSecs)),
1047 .Dealloc: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSRegisterInitSectionsArgs>(
1048 FnAddr: MP.DeregisterInitSections.Addr, Args: HeaderAddr, Args: ELFNixPlatformSecs))});
1049
1050 return Error::success();
1051}
1052
1053Error ELFNixPlatform::ELFNixPlatformPlugin::registerFiniSections(
1054 jitlink::LinkGraph &G, JITDylib &JD, bool IsBootstrapping) {
1055 SmallVector<ExecutorAddrRange> ELFNixFiniSecs;
1056 LLVM_DEBUG(dbgs() << "ELFNixPlatform::registerFiniSections\n");
1057
1058 SmallVector<jitlink::Section *> OrderedFiniSections;
1059 for (auto &Sec : G.sections())
1060 if (isELFFinalizerSection(SecName: Sec.getName()))
1061 OrderedFiniSections.push_back(Elt: &Sec);
1062
1063 // Helper to get section type and priority for sorting.
1064 // Returns: {type_order, priority, has_priority}
1065 // type_order: 0 = .dtors, 1 = .fini, 2 = .fini_array
1066 auto getFiniSectionInfo =
1067 [](const jitlink::Section *Sec) -> std::tuple<int, uint64_t, bool> {
1068 StringRef Name = Sec->getName();
1069 if (Name.starts_with(Prefix: ".dtors")) {
1070 StringRef PrioStr = Name;
1071 uint64_t Prio = 0;
1072 bool HasPrio =
1073 PrioStr.consume_front(Prefix: ".dtors.") && !PrioStr.getAsInteger(Radix: 10, Result&: Prio);
1074 return {0, Prio, HasPrio};
1075 }
1076 if (Name.starts_with(Prefix: ".fini"))
1077 if (!Name.starts_with(Prefix: ".fini_array"))
1078 return {1, 0, false};
1079 if (Name.starts_with(Prefix: ".fini_array")) {
1080 StringRef PrioStr = Name;
1081 uint64_t Prio = 0;
1082 bool HasPrio = PrioStr.consume_front(Prefix: ".fini_array.") &&
1083 !PrioStr.getAsInteger(Radix: 10, Result&: Prio);
1084 return {2, Prio, HasPrio};
1085 }
1086 return {3, 0, false};
1087 };
1088
1089 // Sort fini sections:
1090 // 1. .dtors sections first (ascending priority, as they appear)
1091 // 2. .fini sections next
1092 // 3. .fini_array sections last (descending priority - higher runs first)
1093 llvm::sort(C&: OrderedFiniSections,
1094 Comp: [&](const jitlink::Section *LHS, const jitlink::Section *RHS) {
1095 auto [LType, LPrio, LHasPrio] = getFiniSectionInfo(LHS);
1096 auto [RType, RPrio, RHasPrio] = getFiniSectionInfo(RHS);
1097
1098 if (LType != RType)
1099 return LType < RType;
1100
1101 // Same type - sort by priority
1102 if (LType == 0) {
1103 // .dtors: no-priority first, then ascending priority (lower
1104 // runs first)
1105 if (LHasPrio && RHasPrio)
1106 return LPrio < RPrio;
1107 if (LHasPrio)
1108 return false;
1109 if (RHasPrio)
1110 return true;
1111 } else if (LType == 2) {
1112 // .fini_array: no-priority first, then descending priority
1113 // (higher runs first)
1114 if (LHasPrio && RHasPrio)
1115 return LPrio > RPrio;
1116 if (LHasPrio)
1117 return false;
1118 if (RHasPrio)
1119 return true;
1120 }
1121 return LHS->getName() < RHS->getName();
1122 });
1123
1124 for (auto *Sec : OrderedFiniSections)
1125 ELFNixFiniSecs.push_back(Elt: jitlink::SectionRange(*Sec).getRange());
1126
1127 if (ELFNixFiniSecs.empty())
1128 return Error::success();
1129
1130 // Dump the scraped finis.
1131 LLVM_DEBUG({
1132 dbgs() << "ELFNixPlatform: Scraped " << G.getName() << " fini sections:\n";
1133 for (auto *Sec : OrderedFiniSections) {
1134 jitlink::SectionRange R(*Sec);
1135 dbgs() << " " << Sec->getName() << ": " << R.getRange() << "\n";
1136 }
1137 });
1138
1139 ExecutorAddr HeaderAddr;
1140 {
1141 std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
1142 auto I = MP.JITDylibToHandleAddr.find(Val: &JD);
1143 assert(I != MP.JITDylibToHandleAddr.end() && "No header registered for JD");
1144 assert(I->second && "Null header registered for JD");
1145 HeaderAddr = I->second;
1146 }
1147
1148 using SPSRegisterFiniSectionsArgs =
1149 SPSArgList<SPSExecutorAddr, SPSSequence<SPSExecutorAddrRange>>;
1150
1151 if (LLVM_UNLIKELY(IsBootstrapping)) {
1152 MP.Bootstrap.load()->addArgumentsToRTFnMap(
1153 func1: &MP.RegisterFiniSections, func2: &MP.DeregisterFiniSections,
1154 arg1: getArgDataBufferType<SPSRegisterFiniSectionsArgs>(Args: HeaderAddr,
1155 Args: ELFNixFiniSecs),
1156 arg2: getArgDataBufferType<SPSRegisterFiniSectionsArgs>(Args: HeaderAddr,
1157 Args: ELFNixFiniSecs));
1158 return Error::success();
1159 }
1160
1161 G.allocActions().push_back(
1162 x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSRegisterFiniSectionsArgs>(
1163 FnAddr: MP.RegisterFiniSections.Addr, Args: HeaderAddr, Args: ELFNixFiniSecs)),
1164 .Dealloc: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSRegisterFiniSectionsArgs>(
1165 FnAddr: MP.DeregisterFiniSections.Addr, Args: HeaderAddr, Args: ELFNixFiniSecs))});
1166
1167 return Error::success();
1168}
1169
1170Error ELFNixPlatform::ELFNixPlatformPlugin::fixTLVSectionsAndEdges(
1171 jitlink::LinkGraph &G, JITDylib &JD) {
1172 auto TLSGetAddrSymbolName = G.intern(SymbolName: "__tls_get_addr");
1173 auto TLSDescResolveSymbolName = G.intern(SymbolName: "__tlsdesc_resolver");
1174 auto TLSGetOffsetSymbolName = G.intern(SymbolName: "__tls_get_offset");
1175 for (auto *Sym : G.external_symbols()) {
1176 if (Sym->getName() == TLSGetAddrSymbolName) {
1177 auto TLSGetAddr =
1178 MP.getExecutionSession().intern(SymName: "___orc_rt_elfnix_tls_get_addr");
1179 Sym->setName(std::move(TLSGetAddr));
1180 } else if (Sym->getName() == TLSDescResolveSymbolName) {
1181 auto TLSGetAddr =
1182 MP.getExecutionSession().intern(SymName: "___orc_rt_elfnix_tlsdesc_resolver");
1183 Sym->setName(std::move(TLSGetAddr));
1184 } else if (Sym->getName() == TLSGetOffsetSymbolName) {
1185 auto TLSGetAddr =
1186 MP.getExecutionSession().intern(SymName: "___orc_rt_elfnix_tls_get_offset");
1187 Sym->setName(std::move(TLSGetAddr));
1188 }
1189 }
1190
1191 auto *TLSInfoEntrySection = G.findSectionByName(Name: "$__TLSINFO");
1192
1193 if (TLSInfoEntrySection) {
1194 std::optional<uint64_t> Key;
1195 {
1196 std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
1197 auto I = MP.JITDylibToPThreadKey.find(Val: &JD);
1198 if (I != MP.JITDylibToPThreadKey.end())
1199 Key = I->second;
1200 }
1201 if (!Key) {
1202 if (auto KeyOrErr = MP.createPThreadKey())
1203 Key = *KeyOrErr;
1204 else
1205 return KeyOrErr.takeError();
1206 }
1207
1208 uint64_t PlatformKeyBits =
1209 support::endian::byte_swap(value: *Key, endian: G.getEndianness());
1210
1211 for (auto *B : TLSInfoEntrySection->blocks()) {
1212 // FIXME: The TLS descriptor byte length may different with different
1213 // ISA
1214 assert(B->getSize() == (G.getPointerSize() * 2) &&
1215 "TLS descriptor must be 2 words length");
1216 auto TLSInfoEntryContent = B->getMutableContent(G);
1217 memcpy(dest: TLSInfoEntryContent.data(), src: &PlatformKeyBits, n: G.getPointerSize());
1218 }
1219 }
1220
1221 return Error::success();
1222}
1223
1224} // End namespace orc.
1225} // End namespace llvm.
1226