1//===----- ELFNixPlatform.cpp - Utilities for executing ELFNix in Orc -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/Orc/ELFNixPlatform.h"
10#include "llvm/ExecutionEngine/Orc/Mangling.h"
11
12#include "llvm/ExecutionEngine/JITLink/aarch64.h"
13#include "llvm/ExecutionEngine/JITLink/loongarch.h"
14#include "llvm/ExecutionEngine/JITLink/ppc64.h"
15#include "llvm/ExecutionEngine/JITLink/systemz.h"
16#include "llvm/ExecutionEngine/JITLink/x86_64.h"
17#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
18#include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h"
19#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
20#include "llvm/Support/Debug.h"
21#include <optional>
22
23#define DEBUG_TYPE "orc"
24
25using namespace llvm;
26using namespace llvm::orc;
27using namespace llvm::orc::shared;
28
29namespace {
30
31template <typename SPSSerializer, typename... ArgTs>
32shared::WrapperFunctionCall::ArgDataBufferType
33getArgDataBufferType(const ArgTs &...Args) {
34 shared::WrapperFunctionCall::ArgDataBufferType ArgData;
35 ArgData.resize(SPSSerializer::size(Args...));
36 SPSOutputBuffer OB(ArgData.empty() ? nullptr : ArgData.data(),
37 ArgData.size());
38 if (SPSSerializer::serialize(OB, Args...))
39 return ArgData;
40 return {};
41}
42
43std::unique_ptr<jitlink::LinkGraph> createPlatformGraph(ELFNixPlatform &MOP,
44 std::string Name) {
45 auto &ES = MOP.getExecutionSession();
46 return std::make_unique<jitlink::LinkGraph>(
47 args: std::move(Name), args: ES.getSymbolStringPool(), args: ES.getTargetTriple(),
48 args: SubtargetFeatures(), args&: jitlink::getGenericEdgeKindName);
49}
50
51// Creates a Bootstrap-Complete LinkGraph to run deferred actions.
52class ELFNixPlatformCompleteBootstrapMaterializationUnit
53 : public MaterializationUnit {
54public:
55 ELFNixPlatformCompleteBootstrapMaterializationUnit(
56 ELFNixPlatform &MOP, StringRef PlatformJDName,
57 SymbolStringPtr CompleteBootstrapSymbol, DeferredRuntimeFnMap DeferredAAs,
58 ExecutorAddr ELFNixHeaderAddr, ExecutorAddr PlatformBootstrap,
59 ExecutorAddr PlatformShutdown, ExecutorAddr RegisterJITDylib,
60 ExecutorAddr DeregisterJITDylib)
61 : MaterializationUnit(
62 {{{CompleteBootstrapSymbol, JITSymbolFlags::None}}, nullptr}),
63 MOP(MOP), PlatformJDName(PlatformJDName),
64 CompleteBootstrapSymbol(std::move(CompleteBootstrapSymbol)),
65 DeferredAAsMap(std::move(DeferredAAs)),
66 ELFNixHeaderAddr(ELFNixHeaderAddr),
67 PlatformBootstrap(PlatformBootstrap),
68 PlatformShutdown(PlatformShutdown), RegisterJITDylib(RegisterJITDylib),
69 DeregisterJITDylib(DeregisterJITDylib) {}
70
71 StringRef getName() const override {
72 return "ELFNixPlatformCompleteBootstrap";
73 }
74
75 void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
76 using namespace jitlink;
77 auto G = createPlatformGraph(MOP, Name: "<OrcRTCompleteBootstrap>");
78 auto &PlaceholderSection =
79 G->createSection(Name: "__orc_rt_cplt_bs", Prot: MemProt::Read);
80 auto &PlaceholderBlock =
81 G->createZeroFillBlock(Parent&: PlaceholderSection, Size: 1, Address: ExecutorAddr(), Alignment: 1, AlignmentOffset: 0);
82 G->addDefinedSymbol(Content&: PlaceholderBlock, Offset: 0, Name: *CompleteBootstrapSymbol, Size: 1,
83 L: Linkage::Strong, S: Scope::Hidden, IsCallable: false, IsLive: true);
84
85 // 1. Bootstrap the platform support code.
86 G->allocActions().push_back(
87 x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
88 FnAddr: PlatformBootstrap, Args: ELFNixHeaderAddr)),
89 .Dealloc: cantFail(
90 ValOrErr: WrapperFunctionCall::Create<SPSArgList<>>(FnAddr: PlatformShutdown))});
91
92 // 2. Register the platform JITDylib.
93 G->allocActions().push_back(
94 x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<
95 SPSArgList<SPSString, SPSExecutorAddr>>(
96 FnAddr: RegisterJITDylib, Args: PlatformJDName, Args: ELFNixHeaderAddr)),
97 .Dealloc: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
98 FnAddr: DeregisterJITDylib, Args: ELFNixHeaderAddr))});
99
100 // 4. Add the deferred actions to the graph.
101 for (auto &[Fn, CallDatas] : DeferredAAsMap) {
102 for (auto &CallData : CallDatas) {
103 G->allocActions().push_back(
104 x: {.Finalize: WrapperFunctionCall(Fn.first->Addr, std::move(CallData.first)),
105 .Dealloc: WrapperFunctionCall(Fn.second->Addr, std::move(CallData.second))});
106 }
107 }
108
109 MOP.getObjectLinkingLayer().emit(R: std::move(R), G: std::move(G));
110 }
111
112 void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
113
114private:
115 ELFNixPlatform &MOP;
116 StringRef PlatformJDName;
117 SymbolStringPtr CompleteBootstrapSymbol;
118 DeferredRuntimeFnMap DeferredAAsMap;
119 ExecutorAddr ELFNixHeaderAddr;
120 ExecutorAddr PlatformBootstrap;
121 ExecutorAddr PlatformShutdown;
122 ExecutorAddr RegisterJITDylib;
123 ExecutorAddr DeregisterJITDylib;
124};
125
126class DSOHandleMaterializationUnit : public MaterializationUnit {
127public:
128 DSOHandleMaterializationUnit(ELFNixPlatform &ENP,
129 const SymbolStringPtr &DSOHandleSymbol)
130 : MaterializationUnit(
131 createDSOHandleSectionInterface(ENP, DSOHandleSymbol)),
132 ENP(ENP) {}
133
134 StringRef getName() const override { return "DSOHandleMU"; }
135
136 void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
137
138 auto &ES = ENP.getExecutionSession();
139
140 jitlink::Edge::Kind EdgeKind;
141
142 switch (ES.getTargetTriple().getArch()) {
143 case Triple::x86_64:
144 EdgeKind = jitlink::x86_64::Pointer64;
145 break;
146 case Triple::aarch64:
147 EdgeKind = jitlink::aarch64::Pointer64;
148 break;
149 case Triple::ppc64:
150 EdgeKind = jitlink::ppc64::Pointer64;
151 break;
152 case Triple::ppc64le:
153 EdgeKind = jitlink::ppc64::Pointer64;
154 break;
155 case Triple::loongarch64:
156 EdgeKind = jitlink::loongarch::Pointer64;
157 break;
158 case Triple::systemz:
159 EdgeKind = jitlink::systemz::Pointer64;
160 break;
161 default:
162 llvm_unreachable("Unrecognized architecture");
163 }
164
165 // void *__dso_handle = &__dso_handle;
166 auto G = std::make_unique<jitlink::LinkGraph>(
167 args: "<DSOHandleMU>", args: ES.getSymbolStringPool(), args: ES.getTargetTriple(),
168 args: SubtargetFeatures(), args&: jitlink::getGenericEdgeKindName);
169 auto &DSOHandleSection =
170 G->createSection(Name: ".data.__dso_handle", Prot: MemProt::Read);
171 auto &DSOHandleBlock = G->createContentBlock(
172 Parent&: DSOHandleSection, Content: getDSOHandleContent(PointerSize: G->getPointerSize()),
173 Address: orc::ExecutorAddr(), Alignment: 8, AlignmentOffset: 0);
174 auto &DSOHandleSymbol = G->addDefinedSymbol(
175 Content&: DSOHandleBlock, Offset: 0, Name: *R->getInitializerSymbol(), Size: DSOHandleBlock.getSize(),
176 L: jitlink::Linkage::Strong, S: jitlink::Scope::Default, IsCallable: false, IsLive: true);
177 DSOHandleBlock.addEdge(K: EdgeKind, Offset: 0, Target&: DSOHandleSymbol, Addend: 0);
178
179 ENP.getObjectLinkingLayer().emit(R: std::move(R), G: std::move(G));
180 }
181
182 void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
183
184private:
185 static MaterializationUnit::Interface
186 createDSOHandleSectionInterface(ELFNixPlatform &ENP,
187 const SymbolStringPtr &DSOHandleSymbol) {
188 SymbolFlagsMap SymbolFlags;
189 SymbolFlags[DSOHandleSymbol] = JITSymbolFlags::Exported;
190 return MaterializationUnit::Interface(std::move(SymbolFlags),
191 DSOHandleSymbol);
192 }
193
194 ArrayRef<char> getDSOHandleContent(size_t PointerSize) {
195 static const char Content[8] = {0};
196 assert(PointerSize <= sizeof Content);
197 return {Content, PointerSize};
198 }
199
200 ELFNixPlatform &ENP;
201};
202
203} // end anonymous namespace
204
205namespace llvm {
206namespace orc {
207
208Expected<std::unique_ptr<ELFNixPlatform>>
209ELFNixPlatform::Create(ObjectLinkingLayer &ObjLinkingLayer,
210 JITDylib &PlatformJD,
211 std::unique_ptr<DefinitionGenerator> OrcRuntime,
212 std::optional<SymbolAliasMap> RuntimeAliases) {
213
214 auto &ES = ObjLinkingLayer.getExecutionSession();
215
216 // If the target is not supported then bail out immediately.
217 if (!supportedTarget(TT: ES.getTargetTriple()))
218 return make_error<StringError>(Args: "Unsupported ELFNixPlatform triple: " +
219 ES.getTargetTriple().str(),
220 Args: inconvertibleErrorCode());
221
222 // Create default aliases if the caller didn't supply any.
223 if (!RuntimeAliases) {
224 auto StandardRuntimeAliases = standardPlatformAliases(ES, PlatformJD);
225 if (!StandardRuntimeAliases)
226 return StandardRuntimeAliases.takeError();
227 RuntimeAliases = std::move(*StandardRuntimeAliases);
228 }
229
230 // Define the aliases.
231 if (auto Err = PlatformJD.define(MU: symbolAliases(Aliases: std::move(*RuntimeAliases))))
232 return std::move(Err);
233
234 {
235 // Add JIT dispatch reexports from bootstrap JITDylib.
236 MangleAndInterner Mangle(ES);
237 auto Exports = buildSimpleReexportsAliasMap(
238 SourceJD&: ES.getBootstrapJITDylib(),
239 Symbols: {{Mangle(rt::DispatchName), Mangle(rt::DispatchCtxName)}});
240 if (!Exports)
241 return Exports.takeError();
242 if (auto Err =
243 PlatformJD.define(MU: reexports(SourceJD&: ES.getBootstrapJITDylib(), Aliases: *Exports)))
244 return Err;
245 }
246
247 // Create the instance.
248 Error Err = Error::success();
249 auto P = std::unique_ptr<ELFNixPlatform>(new ELFNixPlatform(
250 ObjLinkingLayer, PlatformJD, std::move(OrcRuntime), Err));
251 if (Err)
252 return std::move(Err);
253 return std::move(P);
254}
255
256Expected<std::unique_ptr<ELFNixPlatform>>
257ELFNixPlatform::Create(ObjectLinkingLayer &ObjLinkingLayer,
258 JITDylib &PlatformJD, const char *OrcRuntimePath,
259 std::optional<SymbolAliasMap> RuntimeAliases) {
260
261 // Create a generator for the ORC runtime archive.
262 auto OrcRuntimeArchiveGenerator =
263 StaticLibraryDefinitionGenerator::Load(L&: ObjLinkingLayer, FileName: OrcRuntimePath);
264 if (!OrcRuntimeArchiveGenerator)
265 return OrcRuntimeArchiveGenerator.takeError();
266
267 return Create(ObjLinkingLayer, PlatformJD,
268 OrcRuntime: std::move(*OrcRuntimeArchiveGenerator),
269 RuntimeAliases: std::move(RuntimeAliases));
270}
271
272Error ELFNixPlatform::setupJITDylib(JITDylib &JD) {
273 if (auto Err = JD.define(MU: std::make_unique<DSOHandleMaterializationUnit>(
274 args&: *this, args&: DSOHandleSymbol)))
275 return Err;
276
277 return ES.lookup(SearchOrder: {&JD}, Symbol: DSOHandleSymbol).takeError();
278}
279
280Error ELFNixPlatform::teardownJITDylib(JITDylib &JD) {
281 std::lock_guard<std::mutex> Lock(PlatformMutex);
282 auto I = JITDylibToHandleAddr.find(Val: &JD);
283 if (I != JITDylibToHandleAddr.end()) {
284 assert(HandleAddrToJITDylib.count(I->second) &&
285 "HandleAddrToJITDylib missing entry");
286 HandleAddrToJITDylib.erase(Val: I->second);
287 JITDylibToHandleAddr.erase(I);
288 }
289 return Error::success();
290}
291
292Error ELFNixPlatform::notifyAdding(ResourceTracker &RT,
293 const MaterializationUnit &MU) {
294
295 auto &JD = RT.getJITDylib();
296 const auto &InitSym = MU.getInitializerSymbol();
297 if (!InitSym)
298 return Error::success();
299
300 RegisteredInitSymbols[&JD].add(Name: InitSym,
301 Flags: SymbolLookupFlags::WeaklyReferencedSymbol);
302 LLVM_DEBUG({
303 dbgs() << "ELFNixPlatform: Registered init symbol " << *InitSym
304 << " for MU " << MU.getName() << "\n";
305 });
306 return Error::success();
307}
308
309Error ELFNixPlatform::notifyRemoving(ResourceTracker &RT) {
310 llvm_unreachable("Not supported yet");
311}
312
313static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases,
314 ArrayRef<std::pair<const char *, const char *>> AL) {
315 for (auto &KV : AL) {
316 auto AliasName = ES.intern(SymName: KV.first);
317 assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map");
318 Aliases[std::move(AliasName)] = {ES.intern(SymName: KV.second),
319 JITSymbolFlags::Exported};
320 }
321}
322
323Expected<SymbolAliasMap>
324ELFNixPlatform::standardPlatformAliases(ExecutionSession &ES,
325 JITDylib &PlatformJD) {
326 SymbolAliasMap Aliases;
327 addAliases(ES, Aliases, AL: requiredCXXAliases());
328 addAliases(ES, Aliases, AL: standardRuntimeUtilityAliases());
329 addAliases(ES, Aliases, AL: standardLazyCompilationAliases());
330 return Aliases;
331}
332
333ArrayRef<std::pair<const char *, const char *>>
334ELFNixPlatform::requiredCXXAliases() {
335 static const std::pair<const char *, const char *> RequiredCXXAliases[] = {
336 {"__cxa_atexit", "__orc_rt_elfnix_cxa_atexit"},
337 {"atexit", "__orc_rt_elfnix_atexit"}};
338
339 return ArrayRef<std::pair<const char *, const char *>>(RequiredCXXAliases);
340}
341
342ArrayRef<std::pair<const char *, const char *>>
343ELFNixPlatform::standardRuntimeUtilityAliases() {
344 static const std::pair<const char *, const char *>
345 StandardRuntimeUtilityAliases[] = {
346 {"__orc_rt_run_program", "__orc_rt_elfnix_run_program"},
347 {"__orc_rt_jit_dlerror", "__orc_rt_elfnix_jit_dlerror"},
348 {"__orc_rt_jit_dlopen", "__orc_rt_elfnix_jit_dlopen"},
349 {"__orc_rt_jit_dlupdate", "__orc_rt_elfnix_jit_dlupdate"},
350 {"__orc_rt_jit_dlclose", "__orc_rt_elfnix_jit_dlclose"},
351 {"__orc_rt_jit_dlsym", "__orc_rt_elfnix_jit_dlsym"},
352 {"__orc_rt_log_error", "__orc_rt_log_error_to_stderr"}};
353
354 return ArrayRef<std::pair<const char *, const char *>>(
355 StandardRuntimeUtilityAliases);
356}
357
358ArrayRef<std::pair<const char *, const char *>>
359ELFNixPlatform::standardLazyCompilationAliases() {
360 static const std::pair<const char *, const char *>
361 StandardLazyCompilationAliases[] = {
362 {"__orc_rt_reenter", "__orc_rt_sysv_reenter"}};
363
364 return ArrayRef<std::pair<const char *, const char *>>(
365 StandardLazyCompilationAliases);
366}
367
368bool ELFNixPlatform::supportedTarget(const Triple &TT) {
369 switch (TT.getArch()) {
370 case Triple::x86_64:
371 case Triple::aarch64:
372 // FIXME: jitlink for ppc64 hasn't been well tested, leave it unsupported
373 // right now.
374 case Triple::ppc64le:
375 case Triple::loongarch64:
376 case Triple::systemz:
377 return true;
378 default:
379 return false;
380 }
381}
382
383ELFNixPlatform::ELFNixPlatform(
384 ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD,
385 std::unique_ptr<DefinitionGenerator> OrcRuntimeGenerator, Error &Err)
386 : ES(ObjLinkingLayer.getExecutionSession()), PlatformJD(PlatformJD),
387 ObjLinkingLayer(ObjLinkingLayer),
388 DSOHandleSymbol(ES.intern(SymName: "__dso_handle")) {
389 ErrorAsOutParameter _(Err);
390 ObjLinkingLayer.addPlugin(P: std::make_unique<ELFNixPlatformPlugin>(args&: *this));
391
392 PlatformJD.addGenerator(DefGenerator: std::move(OrcRuntimeGenerator));
393
394 BootstrapInfo BI;
395 Bootstrap = &BI;
396
397 // PlatformJD hasn't been 'set-up' by the platform yet (since we're creating
398 // the platform now), so set it up.
399 if (auto E2 = setupJITDylib(PlatformJD)) {
400 Err = std::move(E2);
401 return;
402 }
403
404 // Step (2) Request runtime registration functions to trigger
405 // materialization..
406 if ((Err = ES.lookup(
407 SearchOrder: makeJITDylibSearchOrder(JDs: &PlatformJD),
408 Symbols: SymbolLookupSet(
409 {PlatformBootstrap.Name, PlatformShutdown.Name,
410 RegisterJITDylib.Name, DeregisterJITDylib.Name,
411 RegisterInitSections.Name, DeregisterInitSections.Name,
412 RegisterFiniSections.Name, DeregisterFiniSections.Name,
413 RegisterObjectSections.Name,
414 DeregisterObjectSections.Name, CreatePThreadKey.Name}))
415 .takeError()))
416 return;
417
418 // Step (3) Wait for any incidental linker work to complete.
419 {
420 std::unique_lock<std::mutex> Lock(BI.Mutex);
421 BI.CV.wait(lock&: Lock, p: [&]() { return BI.ActiveGraphs == 0; });
422 Bootstrap = nullptr;
423 }
424
425 // Step (4) Add complete-bootstrap materialization unit and request.
426 auto BootstrapCompleteSymbol =
427 ES.intern(SymName: "__orc_rt_elfnix_complete_bootstrap");
428 if ((Err = PlatformJD.define(
429 MU: std::make_unique<ELFNixPlatformCompleteBootstrapMaterializationUnit>(
430 args&: *this, args: PlatformJD.getName(), args&: BootstrapCompleteSymbol,
431 args: std::move(BI.DeferredRTFnMap), args&: BI.ELFNixHeaderAddr,
432 args&: PlatformBootstrap.Addr, args&: PlatformShutdown.Addr,
433 args&: RegisterJITDylib.Addr, args&: DeregisterJITDylib.Addr))))
434 return;
435 if ((Err = ES.lookup(SearchOrder: makeJITDylibSearchOrder(
436 JDs: &PlatformJD, Flags: JITDylibLookupFlags::MatchAllSymbols),
437 Symbol: std::move(BootstrapCompleteSymbol))
438 .takeError()))
439 return;
440
441 // Associate wrapper function tags with JIT-side function implementations.
442 if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) {
443 Err = std::move(E2);
444 return;
445 }
446}
447
448Error ELFNixPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) {
449 ExecutionSession::JITDispatchHandlerAssociationMap WFs;
450
451 using RecordInitializersSPSSig =
452 SPSExpected<SPSELFNixJITDylibDepInfoMap>(SPSExecutorAddr);
453 WFs[ES.intern(SymName: "__orc_rt_elfnix_push_initializers_tag")] =
454 ES.wrapAsyncWithSPS<RecordInitializersSPSSig>(
455 Instance: this, Method: &ELFNixPlatform::rt_recordInitializers);
456
457 using LookupSymbolSPSSig =
458 SPSExpected<SPSExecutorAddr>(SPSExecutorAddr, SPSString);
459 WFs[ES.intern(SymName: "__orc_rt_elfnix_symbol_lookup_tag")] =
460 ES.wrapAsyncWithSPS<LookupSymbolSPSSig>(Instance: this,
461 Method: &ELFNixPlatform::rt_lookupSymbol);
462
463 return ES.registerJITDispatchHandlers(JD&: PlatformJD, WFs: std::move(WFs));
464}
465
466void ELFNixPlatform::pushInitializersLoop(
467 PushInitializersSendResultFn SendResult, JITDylibSP JD) {
468 DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols;
469 DenseMap<JITDylib *, SmallVector<JITDylib *>> JDDepMap;
470 SmallVector<JITDylib *, 16> Worklist({JD.get()});
471
472 ES.runSessionLocked(F: [&]() {
473 while (!Worklist.empty()) {
474 // FIXME: Check for defunct dylibs.
475
476 auto DepJD = Worklist.back();
477 Worklist.pop_back();
478
479 // If we've already visited this JITDylib on this iteration then continue.
480 auto [It, Inserted] = JDDepMap.try_emplace(Key: DepJD);
481 if (!Inserted)
482 continue;
483
484 // Add dep info.
485 auto &DM = It->second;
486 DepJD->withLinkOrderDo(F: [&](const JITDylibSearchOrder &O) {
487 for (auto &KV : O) {
488 if (KV.first == DepJD)
489 continue;
490 DM.push_back(Elt: KV.first);
491 Worklist.push_back(Elt: KV.first);
492 }
493 });
494
495 // Add any registered init symbols.
496 auto RISItr = RegisteredInitSymbols.find(Val: DepJD);
497 if (RISItr != RegisteredInitSymbols.end()) {
498 NewInitSymbols[DepJD] = std::move(RISItr->second);
499 RegisteredInitSymbols.erase(I: RISItr);
500 }
501 }
502 });
503
504 // If there are no further init symbols to look up then send the link order
505 // (as a list of header addresses) to the caller.
506 if (NewInitSymbols.empty()) {
507
508 // To make the list intelligible to the runtime we need to convert all
509 // JITDylib pointers to their header addresses. Only include JITDylibs
510 // that appear in the JITDylibToHandleAddr map (i.e. those that have been
511 // through setupJITDylib) -- bare JITDylibs aren't managed by the platform.
512 DenseMap<JITDylib *, ExecutorAddr> HeaderAddrs;
513 HeaderAddrs.reserve(NumEntries: JDDepMap.size());
514 {
515 std::lock_guard<std::mutex> Lock(PlatformMutex);
516 for (auto &KV : JDDepMap) {
517 auto I = JITDylibToHandleAddr.find(Val: KV.first);
518 if (I != JITDylibToHandleAddr.end())
519 HeaderAddrs[KV.first] = I->second;
520 }
521 }
522
523 // Build the dep info map to return.
524 ELFNixJITDylibDepInfoMap DIM;
525 DIM.reserve(n: JDDepMap.size());
526 for (auto &KV : JDDepMap) {
527 auto HI = HeaderAddrs.find(Val: KV.first);
528 // Skip unmanaged JITDylibs.
529 if (HI == HeaderAddrs.end())
530 continue;
531 auto H = HI->second;
532 ELFNixJITDylibDepInfo DepInfo;
533 for (auto &Dep : KV.second) {
534 auto HJ = HeaderAddrs.find(Val: Dep);
535 if (HJ != HeaderAddrs.end())
536 DepInfo.push_back(x: HJ->second);
537 }
538 DIM.push_back(x: std::make_pair(x&: H, y: std::move(DepInfo)));
539 }
540 SendResult(DIM);
541 return;
542 }
543
544 // Otherwise issue a lookup and re-run this phase when it completes.
545 lookupInitSymbolsAsync(
546 OnComplete: [this, SendResult = std::move(SendResult), JD](Error Err) mutable {
547 if (Err)
548 SendResult(std::move(Err));
549 else
550 pushInitializersLoop(SendResult: std::move(SendResult), JD);
551 },
552 ES, InitSyms: std::move(NewInitSymbols));
553}
554
555void ELFNixPlatform::rt_recordInitializers(
556 PushInitializersSendResultFn SendResult, ExecutorAddr JDHeaderAddr) {
557 JITDylibSP JD;
558 {
559 std::lock_guard<std::mutex> Lock(PlatformMutex);
560 auto I = HandleAddrToJITDylib.find(Val: JDHeaderAddr);
561 if (I != HandleAddrToJITDylib.end())
562 JD = I->second;
563 }
564
565 LLVM_DEBUG({
566 dbgs() << "ELFNixPlatform::rt_recordInitializers(" << JDHeaderAddr << ") ";
567 if (JD)
568 dbgs() << "pushing initializers for " << JD->getName() << "\n";
569 else
570 dbgs() << "No JITDylib for header address.\n";
571 });
572
573 if (!JD) {
574 SendResult(make_error<StringError>(Args: "No JITDylib with header addr " +
575 formatv(Fmt: "{0:x}", Vals&: JDHeaderAddr),
576 Args: inconvertibleErrorCode()));
577 return;
578 }
579
580 pushInitializersLoop(SendResult: std::move(SendResult), JD);
581}
582
583void ELFNixPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult,
584 ExecutorAddr Handle,
585 StringRef SymbolName) {
586 LLVM_DEBUG({
587 dbgs() << "ELFNixPlatform::rt_lookupSymbol(\"" << Handle << "\")\n";
588 });
589
590 JITDylib *JD = nullptr;
591
592 {
593 std::lock_guard<std::mutex> Lock(PlatformMutex);
594 auto I = HandleAddrToJITDylib.find(Val: Handle);
595 if (I != HandleAddrToJITDylib.end())
596 JD = I->second;
597 }
598
599 if (!JD) {
600 LLVM_DEBUG(dbgs() << " No JITDylib for handle " << Handle << "\n");
601 SendResult(make_error<StringError>(Args: "No JITDylib associated with handle " +
602 formatv(Fmt: "{0:x}", Vals&: Handle),
603 Args: inconvertibleErrorCode()));
604 return;
605 }
606
607 // Use functor class to work around XL build compiler issue on AIX.
608 class RtLookupNotifyComplete {
609 public:
610 RtLookupNotifyComplete(SendSymbolAddressFn &&SendResult)
611 : SendResult(std::move(SendResult)) {}
612 void operator()(Expected<SymbolMap> Result) {
613 if (Result) {
614 assert(Result->size() == 1 && "Unexpected result map count");
615 SendResult(Result->begin()->second.getAddress());
616 } else {
617 SendResult(Result.takeError());
618 }
619 }
620
621 private:
622 SendSymbolAddressFn SendResult;
623 };
624
625 ES.lookup(
626 K: LookupKind::DLSym, SearchOrder: {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}},
627 Symbols: SymbolLookupSet(ES.intern(SymName: SymbolName)), RequiredState: SymbolState::Ready,
628 NotifyComplete: RtLookupNotifyComplete(std::move(SendResult)), RegisterDependencies: NoDependenciesToRegister);
629}
630
631Error ELFNixPlatform::ELFNixPlatformPlugin::bootstrapPipelineStart(
632 jitlink::LinkGraph &G) {
633 // Increment the active graphs count in BootstrapInfo.
634 std::lock_guard<std::mutex> Lock(MP.Bootstrap.load()->Mutex);
635 ++MP.Bootstrap.load()->ActiveGraphs;
636 return Error::success();
637}
638
639Error ELFNixPlatform::ELFNixPlatformPlugin::
640 bootstrapPipelineRecordRuntimeFunctions(jitlink::LinkGraph &G) {
641 // Record bootstrap function names.
642 std::pair<StringRef, ExecutorAddr *> RuntimeSymbols[] = {
643 {*MP.DSOHandleSymbol, &MP.Bootstrap.load()->ELFNixHeaderAddr},
644 {*MP.PlatformBootstrap.Name, &MP.PlatformBootstrap.Addr},
645 {*MP.PlatformShutdown.Name, &MP.PlatformShutdown.Addr},
646 {*MP.RegisterJITDylib.Name, &MP.RegisterJITDylib.Addr},
647 {*MP.DeregisterJITDylib.Name, &MP.DeregisterJITDylib.Addr},
648 {*MP.RegisterObjectSections.Name, &MP.RegisterObjectSections.Addr},
649 {*MP.DeregisterObjectSections.Name, &MP.DeregisterObjectSections.Addr},
650 {*MP.RegisterInitSections.Name, &MP.RegisterInitSections.Addr},
651 {*MP.DeregisterInitSections.Name, &MP.DeregisterInitSections.Addr},
652 {*MP.RegisterFiniSections.Name, &MP.RegisterFiniSections.Addr},
653 {*MP.DeregisterFiniSections.Name, &MP.DeregisterFiniSections.Addr},
654 {*MP.CreatePThreadKey.Name, &MP.CreatePThreadKey.Addr}};
655
656 bool RegisterELFNixHeader = false;
657
658 for (auto *Sym : G.defined_symbols()) {
659 for (auto &RTSym : RuntimeSymbols) {
660 if (Sym->hasName() && *Sym->getName() == RTSym.first) {
661 if (*RTSym.second)
662 return make_error<StringError>(
663 Args: "Duplicate " + RTSym.first +
664 " detected during ELFNixPlatform bootstrap",
665 Args: inconvertibleErrorCode());
666
667 if (*Sym->getName() == *MP.DSOHandleSymbol)
668 RegisterELFNixHeader = true;
669
670 *RTSym.second = Sym->getAddress();
671 }
672 }
673 }
674
675 if (RegisterELFNixHeader) {
676 // If this graph defines the elfnix header symbol then create the internal
677 // mapping between it and PlatformJD.
678 std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
679 MP.JITDylibToHandleAddr[&MP.PlatformJD] =
680 MP.Bootstrap.load()->ELFNixHeaderAddr;
681 MP.HandleAddrToJITDylib[MP.Bootstrap.load()->ELFNixHeaderAddr] =
682 &MP.PlatformJD;
683 }
684
685 return Error::success();
686}
687
688Error ELFNixPlatform::ELFNixPlatformPlugin::bootstrapPipelineEnd(
689 jitlink::LinkGraph &G) {
690 std::lock_guard<std::mutex> Lock(MP.Bootstrap.load()->Mutex);
691 assert(MP.Bootstrap && "DeferredAAs reset before bootstrap completed");
692 --MP.Bootstrap.load()->ActiveGraphs;
693 // Notify Bootstrap->CV while holding the mutex because the mutex is
694 // also keeping Bootstrap->CV alive.
695 if (MP.Bootstrap.load()->ActiveGraphs == 0)
696 MP.Bootstrap.load()->CV.notify_all();
697 return Error::success();
698}
699
700Error ELFNixPlatform::registerPerObjectSections(
701 jitlink::LinkGraph &G, const ELFPerObjectSectionsToRegister &POSR,
702 bool IsBootstrapping) {
703 using SPSRegisterPerObjSectionsArgs =
704 SPSArgList<SPSELFPerObjectSectionsToRegister>;
705
706 if (LLVM_UNLIKELY(IsBootstrapping)) {
707 Bootstrap.load()->addArgumentsToRTFnMap(
708 func1: &RegisterObjectSections, func2: &DeregisterObjectSections,
709 arg1: getArgDataBufferType<SPSRegisterPerObjSectionsArgs>(Args: POSR),
710 arg2: getArgDataBufferType<SPSRegisterPerObjSectionsArgs>(Args: POSR));
711 return Error::success();
712 }
713
714 G.allocActions().push_back(
715 x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSRegisterPerObjSectionsArgs>(
716 FnAddr: RegisterObjectSections.Addr, Args: POSR)),
717 .Dealloc: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSRegisterPerObjSectionsArgs>(
718 FnAddr: DeregisterObjectSections.Addr, Args: POSR))});
719
720 return Error::success();
721}
722
723Expected<uint64_t> ELFNixPlatform::createPThreadKey() {
724 if (!CreatePThreadKey.Addr)
725 return make_error<StringError>(
726 Args: "Attempting to create pthread key in target, but runtime support has "
727 "not been loaded yet",
728 Args: inconvertibleErrorCode());
729
730 Expected<uint64_t> Result(0);
731 if (auto Err = ES.callSPSWrapper<SPSExpected<uint64_t>(void)>(
732 WrapperFnAddr: CreatePThreadKey.Addr, WrapperCallArgs&: Result))
733 return std::move(Err);
734 return Result;
735}
736
737void ELFNixPlatform::ELFNixPlatformPlugin::modifyPassConfig(
738 MaterializationResponsibility &MR, jitlink::LinkGraph &LG,
739 jitlink::PassConfiguration &Config) {
740 using namespace jitlink;
741
742 bool InBootstrapPhase =
743 &MR.getTargetJITDylib() == &MP.PlatformJD && MP.Bootstrap;
744
745 // If we're in the bootstrap phase then increment the active graphs.
746 if (InBootstrapPhase) {
747 Config.PrePrunePasses.push_back(
748 x: [this](LinkGraph &G) { return bootstrapPipelineStart(G); });
749 Config.PostAllocationPasses.push_back(x: [this](LinkGraph &G) {
750 return bootstrapPipelineRecordRuntimeFunctions(G);
751 });
752 }
753
754 // If the initializer symbol is the __dso_handle symbol then just add
755 // the DSO handle support passes.
756 if (auto InitSymbol = MR.getInitializerSymbol()) {
757 if (InitSymbol == MP.DSOHandleSymbol && !InBootstrapPhase) {
758 addDSOHandleSupportPasses(MR, Config);
759 // The DSOHandle materialization unit doesn't require any other
760 // support, so we can bail out early.
761 return;
762 }
763
764 /// Preserve init sections.
765 Config.PrePrunePasses.push_back(
766 x: [this, &MR](jitlink::LinkGraph &G) -> Error {
767 if (auto Err = preserveInitSections(G, MR))
768 return Err;
769 return Error::success();
770 });
771 }
772
773 // Add passes for eh-frame and TLV support.
774 addEHAndTLVSupportPasses(MR, Config, IsBootstrapping: InBootstrapPhase);
775
776 // If the object contains initializers then add passes to record them.
777 Config.PostFixupPasses.push_back(x: [this, &JD = MR.getTargetJITDylib(),
778 InBootstrapPhase](jitlink::LinkGraph &G) {
779 return registerInitSections(G, JD, IsBootstrapping: InBootstrapPhase);
780 });
781
782 // If the object contains finalizers then add passes to record them.
783 Config.PostFixupPasses.push_back(x: [this, &JD = MR.getTargetJITDylib(),
784 InBootstrapPhase](jitlink::LinkGraph &G) {
785 return registerFiniSections(G, JD, IsBootstrapping: InBootstrapPhase);
786 });
787
788 // If we're in the bootstrap phase then steal allocation actions and then
789 // decrement the active graphs.
790 if (InBootstrapPhase)
791 Config.PostFixupPasses.push_back(
792 x: [this](LinkGraph &G) { return bootstrapPipelineEnd(G); });
793}
794
795void ELFNixPlatform::ELFNixPlatformPlugin::addDSOHandleSupportPasses(
796 MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) {
797
798 Config.PostAllocationPasses.push_back(x: [this, &JD = MR.getTargetJITDylib()](
799 jitlink::LinkGraph &G) -> Error {
800 auto I = llvm::find_if(Range: G.defined_symbols(), P: [this](jitlink::Symbol *Sym) {
801 return Sym->getName() == MP.DSOHandleSymbol;
802 });
803 assert(I != G.defined_symbols().end() && "Missing DSO handle symbol");
804 {
805 std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
806 auto HandleAddr = (*I)->getAddress();
807 MP.HandleAddrToJITDylib[HandleAddr] = &JD;
808 MP.JITDylibToHandleAddr[&JD] = HandleAddr;
809
810 G.allocActions().push_back(
811 x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<
812 SPSArgList<SPSString, SPSExecutorAddr>>(
813 FnAddr: MP.RegisterJITDylib.Addr, Args: JD.getName(), Args: HandleAddr)),
814 .Dealloc: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
815 FnAddr: MP.DeregisterJITDylib.Addr, Args: HandleAddr))});
816 }
817 return Error::success();
818 });
819}
820
821void ELFNixPlatform::ELFNixPlatformPlugin::addEHAndTLVSupportPasses(
822 MaterializationResponsibility &MR, jitlink::PassConfiguration &Config,
823 bool IsBootstrapping) {
824
825 // Insert TLV lowering at the start of the PostPrunePasses, since we want
826 // it to run before GOT/PLT lowering.
827
828 // TODO: Check that before the fixTLVSectionsAndEdges pass, the GOT/PLT build
829 // pass has done. Because the TLS descriptor need to be allocate in GOT.
830 Config.PostPrunePasses.push_back(
831 x: [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
832 return fixTLVSectionsAndEdges(G, JD);
833 });
834
835 // Add a pass to register the final addresses of the eh-frame and TLV sections
836 // with the runtime.
837 Config.PostFixupPasses.push_back(x: [this, IsBootstrapping](
838 jitlink::LinkGraph &G) -> Error {
839 ELFPerObjectSectionsToRegister POSR;
840
841 if (auto *EHFrameSection = G.findSectionByName(Name: ELFEHFrameSectionName)) {
842 jitlink::SectionRange R(*EHFrameSection);
843 if (!R.empty())
844 POSR.EHFrameSection = R.getRange();
845 }
846
847 // Get a pointer to the thread data section if there is one. It will be used
848 // below.
849 jitlink::Section *ThreadDataSection =
850 G.findSectionByName(Name: ELFThreadDataSectionName);
851
852 // Handle thread BSS section if there is one.
853 if (auto *ThreadBSSSection = G.findSectionByName(Name: ELFThreadBSSSectionName)) {
854 // If there's already a thread data section in this graph then merge the
855 // thread BSS section content into it, otherwise just treat the thread
856 // BSS section as the thread data section.
857 if (ThreadDataSection)
858 G.mergeSections(DstSection&: *ThreadDataSection, SrcSection&: *ThreadBSSSection);
859 else
860 ThreadDataSection = ThreadBSSSection;
861 }
862
863 // Having merged thread BSS (if present) and thread data (if present),
864 // record the resulting section range.
865 if (ThreadDataSection) {
866 jitlink::SectionRange R(*ThreadDataSection);
867 if (!R.empty())
868 POSR.ThreadDataSection = R.getRange();
869 }
870
871 if (POSR.EHFrameSection.Start || POSR.ThreadDataSection.Start) {
872 if (auto Err = MP.registerPerObjectSections(G, POSR, IsBootstrapping))
873 return Err;
874 }
875
876 return Error::success();
877 });
878}
879
880Error ELFNixPlatform::ELFNixPlatformPlugin::preserveInitSections(
881 jitlink::LinkGraph &G, MaterializationResponsibility &MR) {
882
883 if (const auto &InitSymName = MR.getInitializerSymbol()) {
884
885 jitlink::Symbol *InitSym = nullptr;
886
887 for (auto &InitSection : G.sections()) {
888 // Skip non-init sections.
889 if (!isELFInitializerSection(SecName: InitSection.getName()) ||
890 InitSection.empty())
891 continue;
892
893 // Create the init symbol if it has not been created already and attach it
894 // to the first block.
895 if (!InitSym) {
896 auto &B = **InitSection.blocks().begin();
897 InitSym = &G.addDefinedSymbol(
898 Content&: B, Offset: 0, Name: *InitSymName, Size: B.getSize(), L: jitlink::Linkage::Strong,
899 S: jitlink::Scope::SideEffectsOnly, IsCallable: false, IsLive: true);
900 }
901
902 // Add keep-alive edges to anonymous symbols in all other init blocks.
903 for (auto *B : InitSection.blocks()) {
904 if (B == &InitSym->getBlock())
905 continue;
906
907 auto &S = G.addAnonymousSymbol(Content&: *B, Offset: 0, Size: B->getSize(), IsCallable: false, IsLive: true);
908 InitSym->getBlock().addEdge(K: jitlink::Edge::KeepAlive, Offset: 0, Target&: S, Addend: 0);
909 }
910 }
911
912 // Also preserve fini sections (.fini_array, .fini, .dtors)
913 for (auto &FiniSection : G.sections()) {
914 // Skip non-fini sections.
915 if (!isELFFinalizerSection(SecName: FiniSection.getName()) || FiniSection.empty())
916 continue;
917
918 // Create the init symbol if it has not been created already and attach it
919 // to the first fini block.
920 if (!InitSym) {
921 auto &B = **FiniSection.blocks().begin();
922 InitSym = &G.addDefinedSymbol(
923 Content&: B, Offset: 0, Name: *InitSymName, Size: B.getSize(), L: jitlink::Linkage::Strong,
924 S: jitlink::Scope::SideEffectsOnly, IsCallable: false, IsLive: true);
925 }
926
927 // Add keep-alive edges to anonymous symbols in all fini blocks.
928 for (auto *B : FiniSection.blocks()) {
929 if (B == &InitSym->getBlock())
930 continue;
931
932 auto &S = G.addAnonymousSymbol(Content&: *B, Offset: 0, Size: B->getSize(), IsCallable: false, IsLive: true);
933 InitSym->getBlock().addEdge(K: jitlink::Edge::KeepAlive, Offset: 0, Target&: S, Addend: 0);
934 }
935 }
936 }
937
938 return Error::success();
939}
940
941Error ELFNixPlatform::ELFNixPlatformPlugin::registerInitSections(
942 jitlink::LinkGraph &G, JITDylib &JD, bool IsBootstrapping) {
943 SmallVector<ExecutorAddrRange> ELFNixPlatformSecs;
944 LLVM_DEBUG(dbgs() << "ELFNixPlatform::registerInitSections\n");
945
946 SmallVector<jitlink::Section *> OrderedInitSections;
947 for (auto &Sec : G.sections())
948 if (isELFInitializerSection(SecName: Sec.getName()))
949 OrderedInitSections.push_back(Elt: &Sec);
950
951 // Helper to get section type and priority for sorting.
952 // Returns: {type_order, priority, has_priority}
953 // type_order: 0 = .init_array, 1 = .init, 2 = .ctors
954 auto getInitSectionInfo =
955 [](const jitlink::Section *Sec) -> std::tuple<int, uint64_t, bool> {
956 StringRef Name = Sec->getName();
957 if (Name.starts_with(Prefix: ".init_array")) {
958 StringRef PrioStr = Name;
959 uint64_t Prio = 0;
960 bool HasPrio = PrioStr.consume_front(Prefix: ".init_array.") &&
961 !PrioStr.getAsInteger(Radix: 10, Result&: Prio);
962 return {0, Prio, HasPrio};
963 }
964 if (Name.starts_with(Prefix: ".init"))
965 return {1, 0, false};
966 if (Name.starts_with(Prefix: ".ctors")) {
967 StringRef PrioStr = Name;
968 uint64_t Prio = 0;
969 bool HasPrio =
970 PrioStr.consume_front(Prefix: ".ctors.") && !PrioStr.getAsInteger(Radix: 10, Result&: Prio);
971 return {2, Prio, HasPrio};
972 }
973 return {3, 0, false};
974 };
975
976 // Sort init sections:
977 // 1. .init_array sections first (ascending priority - lower runs first)
978 // 2. .init sections next
979 // 3. .ctors sections last (descending priority - higher runs first, legacy
980 // behavior)
981 llvm::sort(C&: OrderedInitSections,
982 Comp: [&](const jitlink::Section *LHS, const jitlink::Section *RHS) {
983 auto [LType, LPrio, LHasPrio] = getInitSectionInfo(LHS);
984 auto [RType, RPrio, RHasPrio] = getInitSectionInfo(RHS);
985
986 if (LType != RType)
987 return LType < RType;
988
989 // Same type - sort by priority
990 if (LType == 0) {
991 // .init_array: ascending priority (lower priority number runs
992 // first)
993 if (LHasPrio && RHasPrio)
994 return LPrio < RPrio;
995 if (LHasPrio)
996 return true;
997 if (RHasPrio)
998 return false;
999 } else if (LType == 2) {
1000 // .ctors: descending priority (higher priority number runs
1001 // first)
1002 if (LHasPrio && RHasPrio)
1003 return LPrio > RPrio;
1004 if (LHasPrio)
1005 return true;
1006 if (RHasPrio)
1007 return false;
1008 }
1009 return LHS->getName() < RHS->getName();
1010 });
1011
1012 for (auto &Sec : OrderedInitSections)
1013 ELFNixPlatformSecs.push_back(Elt: jitlink::SectionRange(*Sec).getRange());
1014
1015 // Dump the scraped inits.
1016 LLVM_DEBUG({
1017 dbgs() << "ELFNixPlatform: Scraped " << G.getName() << " init sections:\n";
1018 for (auto &Sec : G.sections()) {
1019 jitlink::SectionRange R(Sec);
1020 dbgs() << " " << Sec.getName() << ": " << R.getRange() << "\n";
1021 }
1022 });
1023
1024 ExecutorAddr HeaderAddr;
1025 {
1026 std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
1027 auto I = MP.JITDylibToHandleAddr.find(Val: &JD);
1028 assert(I != MP.JITDylibToHandleAddr.end() && "No header registered for JD");
1029 assert(I->second && "Null header registered for JD");
1030 HeaderAddr = I->second;
1031 }
1032
1033 using SPSRegisterInitSectionsArgs =
1034 SPSArgList<SPSExecutorAddr, SPSSequence<SPSExecutorAddrRange>>;
1035
1036 if (LLVM_UNLIKELY(IsBootstrapping)) {
1037 MP.Bootstrap.load()->addArgumentsToRTFnMap(
1038 func1: &MP.RegisterInitSections, func2: &MP.DeregisterInitSections,
1039 arg1: getArgDataBufferType<SPSRegisterInitSectionsArgs>(Args: HeaderAddr,
1040 Args: ELFNixPlatformSecs),
1041 arg2: getArgDataBufferType<SPSRegisterInitSectionsArgs>(Args: HeaderAddr,
1042 Args: ELFNixPlatformSecs));
1043 return Error::success();
1044 }
1045
1046 G.allocActions().push_back(
1047 x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSRegisterInitSectionsArgs>(
1048 FnAddr: MP.RegisterInitSections.Addr, Args: HeaderAddr, Args: ELFNixPlatformSecs)),
1049 .Dealloc: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSRegisterInitSectionsArgs>(
1050 FnAddr: MP.DeregisterInitSections.Addr, Args: HeaderAddr, Args: ELFNixPlatformSecs))});
1051
1052 return Error::success();
1053}
1054
1055Error ELFNixPlatform::ELFNixPlatformPlugin::registerFiniSections(
1056 jitlink::LinkGraph &G, JITDylib &JD, bool IsBootstrapping) {
1057 SmallVector<ExecutorAddrRange> ELFNixFiniSecs;
1058 LLVM_DEBUG(dbgs() << "ELFNixPlatform::registerFiniSections\n");
1059
1060 SmallVector<jitlink::Section *> OrderedFiniSections;
1061 for (auto &Sec : G.sections())
1062 if (isELFFinalizerSection(SecName: Sec.getName()))
1063 OrderedFiniSections.push_back(Elt: &Sec);
1064
1065 // Helper to get section type and priority for sorting.
1066 // Returns: {type_order, priority, has_priority}
1067 // type_order: 0 = .dtors, 1 = .fini, 2 = .fini_array
1068 auto getFiniSectionInfo =
1069 [](const jitlink::Section *Sec) -> std::tuple<int, uint64_t, bool> {
1070 StringRef Name = Sec->getName();
1071 if (Name.starts_with(Prefix: ".dtors")) {
1072 StringRef PrioStr = Name;
1073 uint64_t Prio = 0;
1074 bool HasPrio =
1075 PrioStr.consume_front(Prefix: ".dtors.") && !PrioStr.getAsInteger(Radix: 10, Result&: Prio);
1076 return {0, Prio, HasPrio};
1077 }
1078 if (Name.starts_with(Prefix: ".fini"))
1079 if (!Name.starts_with(Prefix: ".fini_array"))
1080 return {1, 0, false};
1081 if (Name.starts_with(Prefix: ".fini_array")) {
1082 StringRef PrioStr = Name;
1083 uint64_t Prio = 0;
1084 bool HasPrio = PrioStr.consume_front(Prefix: ".fini_array.") &&
1085 !PrioStr.getAsInteger(Radix: 10, Result&: Prio);
1086 return {2, Prio, HasPrio};
1087 }
1088 return {3, 0, false};
1089 };
1090
1091 // Sort fini sections:
1092 // 1. .dtors sections first (ascending priority, as they appear)
1093 // 2. .fini sections next
1094 // 3. .fini_array sections last (descending priority - higher runs first)
1095 llvm::sort(C&: OrderedFiniSections,
1096 Comp: [&](const jitlink::Section *LHS, const jitlink::Section *RHS) {
1097 auto [LType, LPrio, LHasPrio] = getFiniSectionInfo(LHS);
1098 auto [RType, RPrio, RHasPrio] = getFiniSectionInfo(RHS);
1099
1100 if (LType != RType)
1101 return LType < RType;
1102
1103 // Same type - sort by priority
1104 if (LType == 0) {
1105 // .dtors: no-priority first, then ascending priority (lower
1106 // runs first)
1107 if (LHasPrio && RHasPrio)
1108 return LPrio < RPrio;
1109 if (LHasPrio)
1110 return false;
1111 if (RHasPrio)
1112 return true;
1113 } else if (LType == 2) {
1114 // .fini_array: no-priority first, then descending priority
1115 // (higher runs first)
1116 if (LHasPrio && RHasPrio)
1117 return LPrio > RPrio;
1118 if (LHasPrio)
1119 return false;
1120 if (RHasPrio)
1121 return true;
1122 }
1123 return LHS->getName() < RHS->getName();
1124 });
1125
1126 for (auto *Sec : OrderedFiniSections)
1127 ELFNixFiniSecs.push_back(Elt: jitlink::SectionRange(*Sec).getRange());
1128
1129 if (ELFNixFiniSecs.empty())
1130 return Error::success();
1131
1132 // Dump the scraped finis.
1133 LLVM_DEBUG({
1134 dbgs() << "ELFNixPlatform: Scraped " << G.getName() << " fini sections:\n";
1135 for (auto *Sec : OrderedFiniSections) {
1136 jitlink::SectionRange R(*Sec);
1137 dbgs() << " " << Sec->getName() << ": " << R.getRange() << "\n";
1138 }
1139 });
1140
1141 ExecutorAddr HeaderAddr;
1142 {
1143 std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
1144 auto I = MP.JITDylibToHandleAddr.find(Val: &JD);
1145 assert(I != MP.JITDylibToHandleAddr.end() && "No header registered for JD");
1146 assert(I->second && "Null header registered for JD");
1147 HeaderAddr = I->second;
1148 }
1149
1150 using SPSRegisterFiniSectionsArgs =
1151 SPSArgList<SPSExecutorAddr, SPSSequence<SPSExecutorAddrRange>>;
1152
1153 if (LLVM_UNLIKELY(IsBootstrapping)) {
1154 MP.Bootstrap.load()->addArgumentsToRTFnMap(
1155 func1: &MP.RegisterFiniSections, func2: &MP.DeregisterFiniSections,
1156 arg1: getArgDataBufferType<SPSRegisterFiniSectionsArgs>(Args: HeaderAddr,
1157 Args: ELFNixFiniSecs),
1158 arg2: getArgDataBufferType<SPSRegisterFiniSectionsArgs>(Args: HeaderAddr,
1159 Args: ELFNixFiniSecs));
1160 return Error::success();
1161 }
1162
1163 G.allocActions().push_back(
1164 x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSRegisterFiniSectionsArgs>(
1165 FnAddr: MP.RegisterFiniSections.Addr, Args: HeaderAddr, Args: ELFNixFiniSecs)),
1166 .Dealloc: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSRegisterFiniSectionsArgs>(
1167 FnAddr: MP.DeregisterFiniSections.Addr, Args: HeaderAddr, Args: ELFNixFiniSecs))});
1168
1169 return Error::success();
1170}
1171
1172Error ELFNixPlatform::ELFNixPlatformPlugin::fixTLVSectionsAndEdges(
1173 jitlink::LinkGraph &G, JITDylib &JD) {
1174 auto TLSGetAddrSymbolName = G.intern(SymbolName: "__tls_get_addr");
1175 auto TLSDescResolveSymbolName = G.intern(SymbolName: "__tlsdesc_resolver");
1176 auto TLSGetOffsetSymbolName = G.intern(SymbolName: "__tls_get_offset");
1177 for (auto *Sym : G.external_symbols()) {
1178 if (Sym->getName() == TLSGetAddrSymbolName) {
1179 auto TLSGetAddr =
1180 MP.getExecutionSession().intern(SymName: "___orc_rt_elfnix_tls_get_addr");
1181 Sym->setName(std::move(TLSGetAddr));
1182 } else if (Sym->getName() == TLSDescResolveSymbolName) {
1183 auto TLSGetAddr =
1184 MP.getExecutionSession().intern(SymName: "___orc_rt_elfnix_tlsdesc_resolver");
1185 Sym->setName(std::move(TLSGetAddr));
1186 } else if (Sym->getName() == TLSGetOffsetSymbolName) {
1187 auto TLSGetAddr =
1188 MP.getExecutionSession().intern(SymName: "___orc_rt_elfnix_tls_get_offset");
1189 Sym->setName(std::move(TLSGetAddr));
1190 }
1191 }
1192
1193 auto *TLSInfoEntrySection = G.findSectionByName(Name: "$__TLSINFO");
1194
1195 if (TLSInfoEntrySection) {
1196 std::optional<uint64_t> Key;
1197 {
1198 std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
1199 auto I = MP.JITDylibToPThreadKey.find(Val: &JD);
1200 if (I != MP.JITDylibToPThreadKey.end())
1201 Key = I->second;
1202 }
1203 if (!Key) {
1204 if (auto KeyOrErr = MP.createPThreadKey())
1205 Key = *KeyOrErr;
1206 else
1207 return KeyOrErr.takeError();
1208 }
1209
1210 uint64_t PlatformKeyBits =
1211 support::endian::byte_swap(value: *Key, endian: G.getEndianness());
1212
1213 for (auto *B : TLSInfoEntrySection->blocks()) {
1214 // FIXME: The TLS descriptor byte length may different with different
1215 // ISA
1216 assert(B->getSize() == (G.getPointerSize() * 2) &&
1217 "TLS descriptor must be 2 words length");
1218 auto TLSInfoEntryContent = B->getMutableContent(G);
1219 memcpy(dest: TLSInfoEntryContent.data(), src: &PlatformKeyBits, n: G.getPointerSize());
1220 }
1221 }
1222
1223 return Error::success();
1224}
1225
1226} // End namespace orc.
1227} // End namespace llvm.
1228