1//===----- ELFNixPlatform.cpp - Utilities for executing ELFNix in Orc -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/Orc/ELFNixPlatform.h"
10
11#include "llvm/ExecutionEngine/JITLink/aarch64.h"
12#include "llvm/ExecutionEngine/JITLink/loongarch.h"
13#include "llvm/ExecutionEngine/JITLink/ppc64.h"
14#include "llvm/ExecutionEngine/JITLink/systemz.h"
15#include "llvm/ExecutionEngine/JITLink/x86_64.h"
16#include "llvm/ExecutionEngine/Orc/AbsoluteSymbols.h"
17#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
18#include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h"
19#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
20#include "llvm/Support/Debug.h"
21#include <optional>
22
23#define DEBUG_TYPE "orc"
24
25using namespace llvm;
26using namespace llvm::orc;
27using namespace llvm::orc::shared;
28
29namespace {
30
31template <typename SPSSerializer, typename... ArgTs>
32shared::WrapperFunctionCall::ArgDataBufferType
33getArgDataBufferType(const ArgTs &...Args) {
34 shared::WrapperFunctionCall::ArgDataBufferType ArgData;
35 ArgData.resize(SPSSerializer::size(Args...));
36 SPSOutputBuffer OB(ArgData.empty() ? nullptr : ArgData.data(),
37 ArgData.size());
38 if (SPSSerializer::serialize(OB, Args...))
39 return ArgData;
40 return {};
41}
42
43std::unique_ptr<jitlink::LinkGraph> createPlatformGraph(ELFNixPlatform &MOP,
44 std::string Name) {
45 auto &ES = MOP.getExecutionSession();
46 return std::make_unique<jitlink::LinkGraph>(
47 args: std::move(Name), args: ES.getSymbolStringPool(), args: ES.getTargetTriple(),
48 args: SubtargetFeatures(), args&: jitlink::getGenericEdgeKindName);
49}
50
51// Creates a Bootstrap-Complete LinkGraph to run deferred actions.
52class ELFNixPlatformCompleteBootstrapMaterializationUnit
53 : public MaterializationUnit {
54public:
55 ELFNixPlatformCompleteBootstrapMaterializationUnit(
56 ELFNixPlatform &MOP, StringRef PlatformJDName,
57 SymbolStringPtr CompleteBootstrapSymbol, DeferredRuntimeFnMap DeferredAAs,
58 ExecutorAddr ELFNixHeaderAddr, ExecutorAddr PlatformBootstrap,
59 ExecutorAddr PlatformShutdown, ExecutorAddr RegisterJITDylib,
60 ExecutorAddr DeregisterJITDylib)
61 : MaterializationUnit(
62 {{{CompleteBootstrapSymbol, JITSymbolFlags::None}}, nullptr}),
63 MOP(MOP), PlatformJDName(PlatformJDName),
64 CompleteBootstrapSymbol(std::move(CompleteBootstrapSymbol)),
65 DeferredAAsMap(std::move(DeferredAAs)),
66 ELFNixHeaderAddr(ELFNixHeaderAddr),
67 PlatformBootstrap(PlatformBootstrap),
68 PlatformShutdown(PlatformShutdown), RegisterJITDylib(RegisterJITDylib),
69 DeregisterJITDylib(DeregisterJITDylib) {}
70
71 StringRef getName() const override {
72 return "ELFNixPlatformCompleteBootstrap";
73 }
74
75 void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
76 using namespace jitlink;
77 auto G = createPlatformGraph(MOP, Name: "<OrcRTCompleteBootstrap>");
78 auto &PlaceholderSection =
79 G->createSection(Name: "__orc_rt_cplt_bs", Prot: MemProt::Read);
80 auto &PlaceholderBlock =
81 G->createZeroFillBlock(Parent&: PlaceholderSection, Size: 1, Address: ExecutorAddr(), Alignment: 1, AlignmentOffset: 0);
82 G->addDefinedSymbol(Content&: PlaceholderBlock, Offset: 0, Name: *CompleteBootstrapSymbol, Size: 1,
83 L: Linkage::Strong, S: Scope::Hidden, IsCallable: false, IsLive: true);
84
85 // 1. Bootstrap the platform support code.
86 G->allocActions().push_back(
87 x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
88 FnAddr: PlatformBootstrap, Args: ELFNixHeaderAddr)),
89 .Dealloc: cantFail(
90 ValOrErr: WrapperFunctionCall::Create<SPSArgList<>>(FnAddr: PlatformShutdown))});
91
92 // 2. Register the platform JITDylib.
93 G->allocActions().push_back(
94 x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<
95 SPSArgList<SPSString, SPSExecutorAddr>>(
96 FnAddr: RegisterJITDylib, Args: PlatformJDName, Args: ELFNixHeaderAddr)),
97 .Dealloc: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
98 FnAddr: DeregisterJITDylib, Args: ELFNixHeaderAddr))});
99
100 // 4. Add the deferred actions to the graph.
101 for (auto &[Fn, CallDatas] : DeferredAAsMap) {
102 for (auto &CallData : CallDatas) {
103 G->allocActions().push_back(
104 x: {.Finalize: WrapperFunctionCall(Fn.first->Addr, std::move(CallData.first)),
105 .Dealloc: WrapperFunctionCall(Fn.second->Addr, std::move(CallData.second))});
106 }
107 }
108
109 MOP.getObjectLinkingLayer().emit(R: std::move(R), G: std::move(G));
110 }
111
112 void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
113
114private:
115 ELFNixPlatform &MOP;
116 StringRef PlatformJDName;
117 SymbolStringPtr CompleteBootstrapSymbol;
118 DeferredRuntimeFnMap DeferredAAsMap;
119 ExecutorAddr ELFNixHeaderAddr;
120 ExecutorAddr PlatformBootstrap;
121 ExecutorAddr PlatformShutdown;
122 ExecutorAddr RegisterJITDylib;
123 ExecutorAddr DeregisterJITDylib;
124};
125
126class DSOHandleMaterializationUnit : public MaterializationUnit {
127public:
128 DSOHandleMaterializationUnit(ELFNixPlatform &ENP,
129 const SymbolStringPtr &DSOHandleSymbol)
130 : MaterializationUnit(
131 createDSOHandleSectionInterface(ENP, DSOHandleSymbol)),
132 ENP(ENP) {}
133
134 StringRef getName() const override { return "DSOHandleMU"; }
135
136 void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
137
138 auto &ES = ENP.getExecutionSession();
139
140 jitlink::Edge::Kind EdgeKind;
141
142 switch (ES.getTargetTriple().getArch()) {
143 case Triple::x86_64:
144 EdgeKind = jitlink::x86_64::Pointer64;
145 break;
146 case Triple::aarch64:
147 EdgeKind = jitlink::aarch64::Pointer64;
148 break;
149 case Triple::ppc64:
150 EdgeKind = jitlink::ppc64::Pointer64;
151 break;
152 case Triple::ppc64le:
153 EdgeKind = jitlink::ppc64::Pointer64;
154 break;
155 case Triple::loongarch64:
156 EdgeKind = jitlink::loongarch::Pointer64;
157 break;
158 case Triple::systemz:
159 EdgeKind = jitlink::systemz::Pointer64;
160 break;
161 default:
162 llvm_unreachable("Unrecognized architecture");
163 }
164
165 // void *__dso_handle = &__dso_handle;
166 auto G = std::make_unique<jitlink::LinkGraph>(
167 args: "<DSOHandleMU>", args: ES.getSymbolStringPool(), args: ES.getTargetTriple(),
168 args: SubtargetFeatures(), args&: jitlink::getGenericEdgeKindName);
169 auto &DSOHandleSection =
170 G->createSection(Name: ".data.__dso_handle", Prot: MemProt::Read);
171 auto &DSOHandleBlock = G->createContentBlock(
172 Parent&: DSOHandleSection, Content: getDSOHandleContent(PointerSize: G->getPointerSize()),
173 Address: orc::ExecutorAddr(), Alignment: 8, AlignmentOffset: 0);
174 auto &DSOHandleSymbol = G->addDefinedSymbol(
175 Content&: DSOHandleBlock, Offset: 0, Name: *R->getInitializerSymbol(), Size: DSOHandleBlock.getSize(),
176 L: jitlink::Linkage::Strong, S: jitlink::Scope::Default, IsCallable: false, IsLive: true);
177 DSOHandleBlock.addEdge(K: EdgeKind, Offset: 0, Target&: DSOHandleSymbol, Addend: 0);
178
179 ENP.getObjectLinkingLayer().emit(R: std::move(R), G: std::move(G));
180 }
181
182 void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
183
184private:
185 static MaterializationUnit::Interface
186 createDSOHandleSectionInterface(ELFNixPlatform &ENP,
187 const SymbolStringPtr &DSOHandleSymbol) {
188 SymbolFlagsMap SymbolFlags;
189 SymbolFlags[DSOHandleSymbol] = JITSymbolFlags::Exported;
190 return MaterializationUnit::Interface(std::move(SymbolFlags),
191 DSOHandleSymbol);
192 }
193
194 ArrayRef<char> getDSOHandleContent(size_t PointerSize) {
195 static const char Content[8] = {0};
196 assert(PointerSize <= sizeof Content);
197 return {Content, PointerSize};
198 }
199
200 ELFNixPlatform &ENP;
201};
202
203} // end anonymous namespace
204
205namespace llvm {
206namespace orc {
207
208Expected<std::unique_ptr<ELFNixPlatform>>
209ELFNixPlatform::Create(ObjectLinkingLayer &ObjLinkingLayer,
210 JITDylib &PlatformJD,
211 std::unique_ptr<DefinitionGenerator> OrcRuntime,
212 std::optional<SymbolAliasMap> RuntimeAliases) {
213
214 auto &ES = ObjLinkingLayer.getExecutionSession();
215
216 // If the target is not supported then bail out immediately.
217 if (!supportedTarget(TT: ES.getTargetTriple()))
218 return make_error<StringError>(Args: "Unsupported ELFNixPlatform triple: " +
219 ES.getTargetTriple().str(),
220 Args: inconvertibleErrorCode());
221
222 // Create default aliases if the caller didn't supply any.
223 if (!RuntimeAliases) {
224 auto StandardRuntimeAliases = standardPlatformAliases(ES, PlatformJD);
225 if (!StandardRuntimeAliases)
226 return StandardRuntimeAliases.takeError();
227 RuntimeAliases = std::move(*StandardRuntimeAliases);
228 }
229
230 // Define the aliases.
231 if (auto Err = PlatformJD.define(MU: symbolAliases(Aliases: std::move(*RuntimeAliases))))
232 return std::move(Err);
233
234 {
235 // Add JIT dispatch reexports from bootstrap JITDylib.
236 auto Exports = buildSimpleReexportsAliasMap(
237 SourceJD&: ES.getBootstrapJITDylib(),
238 Symbols: {{ES.intern(SymName: rt::DispatchName), ES.intern(SymName: rt::DispatchCtxName)}});
239 if (!Exports)
240 return Exports.takeError();
241 if (auto Err =
242 PlatformJD.define(MU: reexports(SourceJD&: ES.getBootstrapJITDylib(), Aliases: *Exports)))
243 return Err;
244 }
245
246 // Create the instance.
247 Error Err = Error::success();
248 auto P = std::unique_ptr<ELFNixPlatform>(new ELFNixPlatform(
249 ObjLinkingLayer, PlatformJD, std::move(OrcRuntime), Err));
250 if (Err)
251 return std::move(Err);
252 return std::move(P);
253}
254
255Expected<std::unique_ptr<ELFNixPlatform>>
256ELFNixPlatform::Create(ObjectLinkingLayer &ObjLinkingLayer,
257 JITDylib &PlatformJD, const char *OrcRuntimePath,
258 std::optional<SymbolAliasMap> RuntimeAliases) {
259
260 // Create a generator for the ORC runtime archive.
261 auto OrcRuntimeArchiveGenerator =
262 StaticLibraryDefinitionGenerator::Load(L&: ObjLinkingLayer, FileName: OrcRuntimePath);
263 if (!OrcRuntimeArchiveGenerator)
264 return OrcRuntimeArchiveGenerator.takeError();
265
266 return Create(ObjLinkingLayer, PlatformJD,
267 OrcRuntime: std::move(*OrcRuntimeArchiveGenerator),
268 RuntimeAliases: std::move(RuntimeAliases));
269}
270
271Error ELFNixPlatform::setupJITDylib(JITDylib &JD) {
272 if (auto Err = JD.define(MU: std::make_unique<DSOHandleMaterializationUnit>(
273 args&: *this, args&: DSOHandleSymbol)))
274 return Err;
275
276 return ES.lookup(SearchOrder: {&JD}, Symbol: DSOHandleSymbol).takeError();
277}
278
279Error ELFNixPlatform::teardownJITDylib(JITDylib &JD) {
280 std::lock_guard<std::mutex> Lock(PlatformMutex);
281 auto I = JITDylibToHandleAddr.find(Val: &JD);
282 if (I != JITDylibToHandleAddr.end()) {
283 assert(HandleAddrToJITDylib.count(I->second) &&
284 "HandleAddrToJITDylib missing entry");
285 HandleAddrToJITDylib.erase(Val: I->second);
286 JITDylibToHandleAddr.erase(I);
287 }
288 return Error::success();
289}
290
291Error ELFNixPlatform::notifyAdding(ResourceTracker &RT,
292 const MaterializationUnit &MU) {
293
294 auto &JD = RT.getJITDylib();
295 const auto &InitSym = MU.getInitializerSymbol();
296 if (!InitSym)
297 return Error::success();
298
299 RegisteredInitSymbols[&JD].add(Name: InitSym,
300 Flags: SymbolLookupFlags::WeaklyReferencedSymbol);
301 LLVM_DEBUG({
302 dbgs() << "ELFNixPlatform: Registered init symbol " << *InitSym
303 << " for MU " << MU.getName() << "\n";
304 });
305 return Error::success();
306}
307
308Error ELFNixPlatform::notifyRemoving(ResourceTracker &RT) {
309 llvm_unreachable("Not supported yet");
310}
311
312static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases,
313 ArrayRef<std::pair<const char *, const char *>> AL) {
314 for (auto &KV : AL) {
315 auto AliasName = ES.intern(SymName: KV.first);
316 assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map");
317 Aliases[std::move(AliasName)] = {ES.intern(SymName: KV.second),
318 JITSymbolFlags::Exported};
319 }
320}
321
322Expected<SymbolAliasMap>
323ELFNixPlatform::standardPlatformAliases(ExecutionSession &ES,
324 JITDylib &PlatformJD) {
325 SymbolAliasMap Aliases;
326 addAliases(ES, Aliases, AL: requiredCXXAliases());
327 addAliases(ES, Aliases, AL: standardRuntimeUtilityAliases());
328 addAliases(ES, Aliases, AL: standardLazyCompilationAliases());
329 return Aliases;
330}
331
332ArrayRef<std::pair<const char *, const char *>>
333ELFNixPlatform::requiredCXXAliases() {
334 static const std::pair<const char *, const char *> RequiredCXXAliases[] = {
335 {"__cxa_atexit", "__orc_rt_elfnix_cxa_atexit"},
336 {"atexit", "__orc_rt_elfnix_atexit"}};
337
338 return ArrayRef<std::pair<const char *, const char *>>(RequiredCXXAliases);
339}
340
341ArrayRef<std::pair<const char *, const char *>>
342ELFNixPlatform::standardRuntimeUtilityAliases() {
343 static const std::pair<const char *, const char *>
344 StandardRuntimeUtilityAliases[] = {
345 {"__orc_rt_run_program", "__orc_rt_elfnix_run_program"},
346 {"__orc_rt_jit_dlerror", "__orc_rt_elfnix_jit_dlerror"},
347 {"__orc_rt_jit_dlopen", "__orc_rt_elfnix_jit_dlopen"},
348 {"__orc_rt_jit_dlupdate", "__orc_rt_elfnix_jit_dlupdate"},
349 {"__orc_rt_jit_dlclose", "__orc_rt_elfnix_jit_dlclose"},
350 {"__orc_rt_jit_dlsym", "__orc_rt_elfnix_jit_dlsym"},
351 {"__orc_rt_log_error", "__orc_rt_log_error_to_stderr"}};
352
353 return ArrayRef<std::pair<const char *, const char *>>(
354 StandardRuntimeUtilityAliases);
355}
356
357ArrayRef<std::pair<const char *, const char *>>
358ELFNixPlatform::standardLazyCompilationAliases() {
359 static const std::pair<const char *, const char *>
360 StandardLazyCompilationAliases[] = {
361 {"__orc_rt_reenter", "__orc_rt_sysv_reenter"}};
362
363 return ArrayRef<std::pair<const char *, const char *>>(
364 StandardLazyCompilationAliases);
365}
366
367bool ELFNixPlatform::supportedTarget(const Triple &TT) {
368 switch (TT.getArch()) {
369 case Triple::x86_64:
370 case Triple::aarch64:
371 // FIXME: jitlink for ppc64 hasn't been well tested, leave it unsupported
372 // right now.
373 case Triple::ppc64le:
374 case Triple::loongarch64:
375 case Triple::systemz:
376 return true;
377 default:
378 return false;
379 }
380}
381
382ELFNixPlatform::ELFNixPlatform(
383 ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD,
384 std::unique_ptr<DefinitionGenerator> OrcRuntimeGenerator, Error &Err)
385 : ES(ObjLinkingLayer.getExecutionSession()), PlatformJD(PlatformJD),
386 ObjLinkingLayer(ObjLinkingLayer),
387 DSOHandleSymbol(ES.intern(SymName: "__dso_handle")) {
388 ErrorAsOutParameter _(Err);
389 ObjLinkingLayer.addPlugin(P: std::make_unique<ELFNixPlatformPlugin>(args&: *this));
390
391 PlatformJD.addGenerator(DefGenerator: std::move(OrcRuntimeGenerator));
392
393 BootstrapInfo BI;
394 Bootstrap = &BI;
395
396 // PlatformJD hasn't been 'set-up' by the platform yet (since we're creating
397 // the platform now), so set it up.
398 if (auto E2 = setupJITDylib(PlatformJD)) {
399 Err = std::move(E2);
400 return;
401 }
402
403 // Step (2) Request runtime registration functions to trigger
404 // materialization..
405 if ((Err = ES.lookup(
406 SearchOrder: makeJITDylibSearchOrder(JDs: &PlatformJD),
407 Symbols: SymbolLookupSet(
408 {PlatformBootstrap.Name, PlatformShutdown.Name,
409 RegisterJITDylib.Name, DeregisterJITDylib.Name,
410 RegisterInitSections.Name, DeregisterInitSections.Name,
411 RegisterFiniSections.Name, DeregisterFiniSections.Name,
412 RegisterObjectSections.Name,
413 DeregisterObjectSections.Name, CreatePThreadKey.Name}))
414 .takeError()))
415 return;
416
417 // Step (3) Wait for any incidental linker work to complete.
418 {
419 std::unique_lock<std::mutex> Lock(BI.Mutex);
420 BI.CV.wait(lock&: Lock, p: [&]() { return BI.ActiveGraphs == 0; });
421 Bootstrap = nullptr;
422 }
423
424 // Step (4) Add complete-bootstrap materialization unit and request.
425 auto BootstrapCompleteSymbol =
426 ES.intern(SymName: "__orc_rt_elfnix_complete_bootstrap");
427 if ((Err = PlatformJD.define(
428 MU: std::make_unique<ELFNixPlatformCompleteBootstrapMaterializationUnit>(
429 args&: *this, args: PlatformJD.getName(), args&: BootstrapCompleteSymbol,
430 args: std::move(BI.DeferredRTFnMap), args&: BI.ELFNixHeaderAddr,
431 args&: PlatformBootstrap.Addr, args&: PlatformShutdown.Addr,
432 args&: RegisterJITDylib.Addr, args&: DeregisterJITDylib.Addr))))
433 return;
434 if ((Err = ES.lookup(SearchOrder: makeJITDylibSearchOrder(
435 JDs: &PlatformJD, Flags: JITDylibLookupFlags::MatchAllSymbols),
436 Symbol: std::move(BootstrapCompleteSymbol))
437 .takeError()))
438 return;
439
440 // Associate wrapper function tags with JIT-side function implementations.
441 if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) {
442 Err = std::move(E2);
443 return;
444 }
445}
446
447Error ELFNixPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) {
448 ExecutionSession::JITDispatchHandlerAssociationMap WFs;
449
450 using RecordInitializersSPSSig =
451 SPSExpected<SPSELFNixJITDylibDepInfoMap>(SPSExecutorAddr);
452 WFs[ES.intern(SymName: "__orc_rt_elfnix_push_initializers_tag")] =
453 ES.wrapAsyncWithSPS<RecordInitializersSPSSig>(
454 Instance: this, Method: &ELFNixPlatform::rt_recordInitializers);
455
456 using LookupSymbolSPSSig =
457 SPSExpected<SPSExecutorAddr>(SPSExecutorAddr, SPSString);
458 WFs[ES.intern(SymName: "__orc_rt_elfnix_symbol_lookup_tag")] =
459 ES.wrapAsyncWithSPS<LookupSymbolSPSSig>(Instance: this,
460 Method: &ELFNixPlatform::rt_lookupSymbol);
461
462 return ES.registerJITDispatchHandlers(JD&: PlatformJD, WFs: std::move(WFs));
463}
464
465void ELFNixPlatform::pushInitializersLoop(
466 PushInitializersSendResultFn SendResult, JITDylibSP JD) {
467 DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols;
468 DenseMap<JITDylib *, SmallVector<JITDylib *>> JDDepMap;
469 SmallVector<JITDylib *, 16> Worklist({JD.get()});
470
471 ES.runSessionLocked(F: [&]() {
472 while (!Worklist.empty()) {
473 // FIXME: Check for defunct dylibs.
474
475 auto DepJD = Worklist.back();
476 Worklist.pop_back();
477
478 // If we've already visited this JITDylib on this iteration then continue.
479 auto [It, Inserted] = JDDepMap.try_emplace(Key: DepJD);
480 if (!Inserted)
481 continue;
482
483 // Add dep info.
484 auto &DM = It->second;
485 DepJD->withLinkOrderDo(F: [&](const JITDylibSearchOrder &O) {
486 for (auto &KV : O) {
487 if (KV.first == DepJD)
488 continue;
489 DM.push_back(Elt: KV.first);
490 Worklist.push_back(Elt: KV.first);
491 }
492 });
493
494 // Add any registered init symbols.
495 auto RISItr = RegisteredInitSymbols.find(Val: DepJD);
496 if (RISItr != RegisteredInitSymbols.end()) {
497 NewInitSymbols[DepJD] = std::move(RISItr->second);
498 RegisteredInitSymbols.erase(I: RISItr);
499 }
500 }
501 });
502
503 // If there are no further init symbols to look up then send the link order
504 // (as a list of header addresses) to the caller.
505 if (NewInitSymbols.empty()) {
506
507 // To make the list intelligible to the runtime we need to convert all
508 // JITDylib pointers to their header addresses. Only include JITDylibs
509 // that appear in the JITDylibToHandleAddr map (i.e. those that have been
510 // through setupJITDylib) -- bare JITDylibs aren't managed by the platform.
511 DenseMap<JITDylib *, ExecutorAddr> HeaderAddrs;
512 HeaderAddrs.reserve(NumEntries: JDDepMap.size());
513 {
514 std::lock_guard<std::mutex> Lock(PlatformMutex);
515 for (auto &KV : JDDepMap) {
516 auto I = JITDylibToHandleAddr.find(Val: KV.first);
517 if (I != JITDylibToHandleAddr.end())
518 HeaderAddrs[KV.first] = I->second;
519 }
520 }
521
522 // Build the dep info map to return.
523 ELFNixJITDylibDepInfoMap DIM;
524 DIM.reserve(n: JDDepMap.size());
525 for (auto &KV : JDDepMap) {
526 auto HI = HeaderAddrs.find(Val: KV.first);
527 // Skip unmanaged JITDylibs.
528 if (HI == HeaderAddrs.end())
529 continue;
530 auto H = HI->second;
531 ELFNixJITDylibDepInfo DepInfo;
532 for (auto &Dep : KV.second) {
533 auto HJ = HeaderAddrs.find(Val: Dep);
534 if (HJ != HeaderAddrs.end())
535 DepInfo.push_back(x: HJ->second);
536 }
537 DIM.push_back(x: std::make_pair(x&: H, y: std::move(DepInfo)));
538 }
539 SendResult(DIM);
540 return;
541 }
542
543 // Otherwise issue a lookup and re-run this phase when it completes.
544 lookupInitSymbolsAsync(
545 OnComplete: [this, SendResult = std::move(SendResult), JD](Error Err) mutable {
546 if (Err)
547 SendResult(std::move(Err));
548 else
549 pushInitializersLoop(SendResult: std::move(SendResult), JD);
550 },
551 ES, InitSyms: std::move(NewInitSymbols));
552}
553
554void ELFNixPlatform::rt_recordInitializers(
555 PushInitializersSendResultFn SendResult, ExecutorAddr JDHeaderAddr) {
556 JITDylibSP JD;
557 {
558 std::lock_guard<std::mutex> Lock(PlatformMutex);
559 auto I = HandleAddrToJITDylib.find(Val: JDHeaderAddr);
560 if (I != HandleAddrToJITDylib.end())
561 JD = I->second;
562 }
563
564 LLVM_DEBUG({
565 dbgs() << "ELFNixPlatform::rt_recordInitializers(" << JDHeaderAddr << ") ";
566 if (JD)
567 dbgs() << "pushing initializers for " << JD->getName() << "\n";
568 else
569 dbgs() << "No JITDylib for header address.\n";
570 });
571
572 if (!JD) {
573 SendResult(make_error<StringError>(Args: "No JITDylib with header addr " +
574 formatv(Fmt: "{0:x}", Vals&: JDHeaderAddr),
575 Args: inconvertibleErrorCode()));
576 return;
577 }
578
579 pushInitializersLoop(SendResult: std::move(SendResult), JD);
580}
581
582void ELFNixPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult,
583 ExecutorAddr Handle,
584 StringRef SymbolName) {
585 LLVM_DEBUG({
586 dbgs() << "ELFNixPlatform::rt_lookupSymbol(\"" << Handle << "\")\n";
587 });
588
589 JITDylib *JD = nullptr;
590
591 {
592 std::lock_guard<std::mutex> Lock(PlatformMutex);
593 auto I = HandleAddrToJITDylib.find(Val: Handle);
594 if (I != HandleAddrToJITDylib.end())
595 JD = I->second;
596 }
597
598 if (!JD) {
599 LLVM_DEBUG(dbgs() << " No JITDylib for handle " << Handle << "\n");
600 SendResult(make_error<StringError>(Args: "No JITDylib associated with handle " +
601 formatv(Fmt: "{0:x}", Vals&: Handle),
602 Args: inconvertibleErrorCode()));
603 return;
604 }
605
606 // Use functor class to work around XL build compiler issue on AIX.
607 class RtLookupNotifyComplete {
608 public:
609 RtLookupNotifyComplete(SendSymbolAddressFn &&SendResult)
610 : SendResult(std::move(SendResult)) {}
611 void operator()(Expected<SymbolMap> Result) {
612 if (Result) {
613 assert(Result->size() == 1 && "Unexpected result map count");
614 SendResult(Result->begin()->second.getAddress());
615 } else {
616 SendResult(Result.takeError());
617 }
618 }
619
620 private:
621 SendSymbolAddressFn SendResult;
622 };
623
624 ES.lookup(
625 K: LookupKind::DLSym, SearchOrder: {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}},
626 Symbols: SymbolLookupSet(ES.intern(SymName: SymbolName)), RequiredState: SymbolState::Ready,
627 NotifyComplete: RtLookupNotifyComplete(std::move(SendResult)), RegisterDependencies: NoDependenciesToRegister);
628}
629
630Error ELFNixPlatform::ELFNixPlatformPlugin::bootstrapPipelineStart(
631 jitlink::LinkGraph &G) {
632 // Increment the active graphs count in BootstrapInfo.
633 std::lock_guard<std::mutex> Lock(MP.Bootstrap.load()->Mutex);
634 ++MP.Bootstrap.load()->ActiveGraphs;
635 return Error::success();
636}
637
638Error ELFNixPlatform::ELFNixPlatformPlugin::
639 bootstrapPipelineRecordRuntimeFunctions(jitlink::LinkGraph &G) {
640 // Record bootstrap function names.
641 std::pair<StringRef, ExecutorAddr *> RuntimeSymbols[] = {
642 {*MP.DSOHandleSymbol, &MP.Bootstrap.load()->ELFNixHeaderAddr},
643 {*MP.PlatformBootstrap.Name, &MP.PlatformBootstrap.Addr},
644 {*MP.PlatformShutdown.Name, &MP.PlatformShutdown.Addr},
645 {*MP.RegisterJITDylib.Name, &MP.RegisterJITDylib.Addr},
646 {*MP.DeregisterJITDylib.Name, &MP.DeregisterJITDylib.Addr},
647 {*MP.RegisterObjectSections.Name, &MP.RegisterObjectSections.Addr},
648 {*MP.DeregisterObjectSections.Name, &MP.DeregisterObjectSections.Addr},
649 {*MP.RegisterInitSections.Name, &MP.RegisterInitSections.Addr},
650 {*MP.DeregisterInitSections.Name, &MP.DeregisterInitSections.Addr},
651 {*MP.RegisterFiniSections.Name, &MP.RegisterFiniSections.Addr},
652 {*MP.DeregisterFiniSections.Name, &MP.DeregisterFiniSections.Addr},
653 {*MP.CreatePThreadKey.Name, &MP.CreatePThreadKey.Addr}};
654
655 bool RegisterELFNixHeader = false;
656
657 for (auto *Sym : G.defined_symbols()) {
658 for (auto &RTSym : RuntimeSymbols) {
659 if (Sym->hasName() && *Sym->getName() == RTSym.first) {
660 if (*RTSym.second)
661 return make_error<StringError>(
662 Args: "Duplicate " + RTSym.first +
663 " detected during ELFNixPlatform bootstrap",
664 Args: inconvertibleErrorCode());
665
666 if (*Sym->getName() == *MP.DSOHandleSymbol)
667 RegisterELFNixHeader = true;
668
669 *RTSym.second = Sym->getAddress();
670 }
671 }
672 }
673
674 if (RegisterELFNixHeader) {
675 // If this graph defines the elfnix header symbol then create the internal
676 // mapping between it and PlatformJD.
677 std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
678 MP.JITDylibToHandleAddr[&MP.PlatformJD] =
679 MP.Bootstrap.load()->ELFNixHeaderAddr;
680 MP.HandleAddrToJITDylib[MP.Bootstrap.load()->ELFNixHeaderAddr] =
681 &MP.PlatformJD;
682 }
683
684 return Error::success();
685}
686
687Error ELFNixPlatform::ELFNixPlatformPlugin::bootstrapPipelineEnd(
688 jitlink::LinkGraph &G) {
689 std::lock_guard<std::mutex> Lock(MP.Bootstrap.load()->Mutex);
690 assert(MP.Bootstrap && "DeferredAAs reset before bootstrap completed");
691 --MP.Bootstrap.load()->ActiveGraphs;
692 // Notify Bootstrap->CV while holding the mutex because the mutex is
693 // also keeping Bootstrap->CV alive.
694 if (MP.Bootstrap.load()->ActiveGraphs == 0)
695 MP.Bootstrap.load()->CV.notify_all();
696 return Error::success();
697}
698
699Error ELFNixPlatform::registerPerObjectSections(
700 jitlink::LinkGraph &G, const ELFPerObjectSectionsToRegister &POSR,
701 bool IsBootstrapping) {
702 using SPSRegisterPerObjSectionsArgs =
703 SPSArgList<SPSELFPerObjectSectionsToRegister>;
704
705 if (LLVM_UNLIKELY(IsBootstrapping)) {
706 Bootstrap.load()->addArgumentsToRTFnMap(
707 func1: &RegisterObjectSections, func2: &DeregisterObjectSections,
708 arg1: getArgDataBufferType<SPSRegisterPerObjSectionsArgs>(Args: POSR),
709 arg2: getArgDataBufferType<SPSRegisterPerObjSectionsArgs>(Args: POSR));
710 return Error::success();
711 }
712
713 G.allocActions().push_back(
714 x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSRegisterPerObjSectionsArgs>(
715 FnAddr: RegisterObjectSections.Addr, Args: POSR)),
716 .Dealloc: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSRegisterPerObjSectionsArgs>(
717 FnAddr: DeregisterObjectSections.Addr, Args: POSR))});
718
719 return Error::success();
720}
721
722Expected<uint64_t> ELFNixPlatform::createPThreadKey() {
723 if (!CreatePThreadKey.Addr)
724 return make_error<StringError>(
725 Args: "Attempting to create pthread key in target, but runtime support has "
726 "not been loaded yet",
727 Args: inconvertibleErrorCode());
728
729 Expected<uint64_t> Result(0);
730 if (auto Err = ES.callSPSWrapper<SPSExpected<uint64_t>(void)>(
731 WrapperFnAddr: CreatePThreadKey.Addr, WrapperCallArgs&: Result))
732 return std::move(Err);
733 return Result;
734}
735
736void ELFNixPlatform::ELFNixPlatformPlugin::modifyPassConfig(
737 MaterializationResponsibility &MR, jitlink::LinkGraph &LG,
738 jitlink::PassConfiguration &Config) {
739 using namespace jitlink;
740
741 bool InBootstrapPhase =
742 &MR.getTargetJITDylib() == &MP.PlatformJD && MP.Bootstrap;
743
744 // If we're in the bootstrap phase then increment the active graphs.
745 if (InBootstrapPhase) {
746 Config.PrePrunePasses.push_back(
747 x: [this](LinkGraph &G) { return bootstrapPipelineStart(G); });
748 Config.PostAllocationPasses.push_back(x: [this](LinkGraph &G) {
749 return bootstrapPipelineRecordRuntimeFunctions(G);
750 });
751 }
752
753 // If the initializer symbol is the __dso_handle symbol then just add
754 // the DSO handle support passes.
755 if (auto InitSymbol = MR.getInitializerSymbol()) {
756 if (InitSymbol == MP.DSOHandleSymbol && !InBootstrapPhase) {
757 addDSOHandleSupportPasses(MR, Config);
758 // The DSOHandle materialization unit doesn't require any other
759 // support, so we can bail out early.
760 return;
761 }
762
763 /// Preserve init sections.
764 Config.PrePrunePasses.push_back(
765 x: [this, &MR](jitlink::LinkGraph &G) -> Error {
766 if (auto Err = preserveInitSections(G, MR))
767 return Err;
768 return Error::success();
769 });
770 }
771
772 // Add passes for eh-frame and TLV support.
773 addEHAndTLVSupportPasses(MR, Config, IsBootstrapping: InBootstrapPhase);
774
775 // If the object contains initializers then add passes to record them.
776 Config.PostFixupPasses.push_back(x: [this, &JD = MR.getTargetJITDylib(),
777 InBootstrapPhase](jitlink::LinkGraph &G) {
778 return registerInitSections(G, JD, IsBootstrapping: InBootstrapPhase);
779 });
780
781 // If the object contains finalizers then add passes to record them.
782 Config.PostFixupPasses.push_back(x: [this, &JD = MR.getTargetJITDylib(),
783 InBootstrapPhase](jitlink::LinkGraph &G) {
784 return registerFiniSections(G, JD, IsBootstrapping: InBootstrapPhase);
785 });
786
787 // If we're in the bootstrap phase then steal allocation actions and then
788 // decrement the active graphs.
789 if (InBootstrapPhase)
790 Config.PostFixupPasses.push_back(
791 x: [this](LinkGraph &G) { return bootstrapPipelineEnd(G); });
792}
793
794void ELFNixPlatform::ELFNixPlatformPlugin::addDSOHandleSupportPasses(
795 MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) {
796
797 Config.PostAllocationPasses.push_back(x: [this, &JD = MR.getTargetJITDylib()](
798 jitlink::LinkGraph &G) -> Error {
799 auto I = llvm::find_if(Range: G.defined_symbols(), P: [this](jitlink::Symbol *Sym) {
800 return Sym->getName() == MP.DSOHandleSymbol;
801 });
802 assert(I != G.defined_symbols().end() && "Missing DSO handle symbol");
803 {
804 std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
805 auto HandleAddr = (*I)->getAddress();
806 MP.HandleAddrToJITDylib[HandleAddr] = &JD;
807 MP.JITDylibToHandleAddr[&JD] = HandleAddr;
808
809 G.allocActions().push_back(
810 x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<
811 SPSArgList<SPSString, SPSExecutorAddr>>(
812 FnAddr: MP.RegisterJITDylib.Addr, Args: JD.getName(), Args: HandleAddr)),
813 .Dealloc: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
814 FnAddr: MP.DeregisterJITDylib.Addr, Args: HandleAddr))});
815 }
816 return Error::success();
817 });
818}
819
820void ELFNixPlatform::ELFNixPlatformPlugin::addEHAndTLVSupportPasses(
821 MaterializationResponsibility &MR, jitlink::PassConfiguration &Config,
822 bool IsBootstrapping) {
823
824 // Insert TLV lowering at the start of the PostPrunePasses, since we want
825 // it to run before GOT/PLT lowering.
826
827 // TODO: Check that before the fixTLVSectionsAndEdges pass, the GOT/PLT build
828 // pass has done. Because the TLS descriptor need to be allocate in GOT.
829 Config.PostPrunePasses.push_back(
830 x: [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
831 return fixTLVSectionsAndEdges(G, JD);
832 });
833
834 // Add a pass to register the final addresses of the eh-frame and TLV sections
835 // with the runtime.
836 Config.PostFixupPasses.push_back(x: [this, IsBootstrapping](
837 jitlink::LinkGraph &G) -> Error {
838 ELFPerObjectSectionsToRegister POSR;
839
840 if (auto *EHFrameSection = G.findSectionByName(Name: ELFEHFrameSectionName)) {
841 jitlink::SectionRange R(*EHFrameSection);
842 if (!R.empty())
843 POSR.EHFrameSection = R.getRange();
844 }
845
846 // Get a pointer to the thread data section if there is one. It will be used
847 // below.
848 jitlink::Section *ThreadDataSection =
849 G.findSectionByName(Name: ELFThreadDataSectionName);
850
851 // Handle thread BSS section if there is one.
852 if (auto *ThreadBSSSection = G.findSectionByName(Name: ELFThreadBSSSectionName)) {
853 // If there's already a thread data section in this graph then merge the
854 // thread BSS section content into it, otherwise just treat the thread
855 // BSS section as the thread data section.
856 if (ThreadDataSection)
857 G.mergeSections(DstSection&: *ThreadDataSection, SrcSection&: *ThreadBSSSection);
858 else
859 ThreadDataSection = ThreadBSSSection;
860 }
861
862 // Having merged thread BSS (if present) and thread data (if present),
863 // record the resulting section range.
864 if (ThreadDataSection) {
865 jitlink::SectionRange R(*ThreadDataSection);
866 if (!R.empty())
867 POSR.ThreadDataSection = R.getRange();
868 }
869
870 if (POSR.EHFrameSection.Start || POSR.ThreadDataSection.Start) {
871 if (auto Err = MP.registerPerObjectSections(G, POSR, IsBootstrapping))
872 return Err;
873 }
874
875 return Error::success();
876 });
877}
878
879Error ELFNixPlatform::ELFNixPlatformPlugin::preserveInitSections(
880 jitlink::LinkGraph &G, MaterializationResponsibility &MR) {
881
882 if (const auto &InitSymName = MR.getInitializerSymbol()) {
883
884 jitlink::Symbol *InitSym = nullptr;
885
886 for (auto &InitSection : G.sections()) {
887 // Skip non-init sections.
888 if (!isELFInitializerSection(SecName: InitSection.getName()) ||
889 InitSection.empty())
890 continue;
891
892 // Create the init symbol if it has not been created already and attach it
893 // to the first block.
894 if (!InitSym) {
895 auto &B = **InitSection.blocks().begin();
896 InitSym = &G.addDefinedSymbol(
897 Content&: B, Offset: 0, Name: *InitSymName, Size: B.getSize(), L: jitlink::Linkage::Strong,
898 S: jitlink::Scope::SideEffectsOnly, IsCallable: false, IsLive: true);
899 }
900
901 // Add keep-alive edges to anonymous symbols in all other init blocks.
902 for (auto *B : InitSection.blocks()) {
903 if (B == &InitSym->getBlock())
904 continue;
905
906 auto &S = G.addAnonymousSymbol(Content&: *B, Offset: 0, Size: B->getSize(), IsCallable: false, IsLive: true);
907 InitSym->getBlock().addEdge(K: jitlink::Edge::KeepAlive, Offset: 0, Target&: S, Addend: 0);
908 }
909 }
910
911 // Also preserve fini sections (.fini_array, .fini, .dtors)
912 for (auto &FiniSection : G.sections()) {
913 // Skip non-fini sections.
914 if (!isELFFinalizerSection(SecName: FiniSection.getName()) || FiniSection.empty())
915 continue;
916
917 // Create the init symbol if it has not been created already and attach it
918 // to the first fini block.
919 if (!InitSym) {
920 auto &B = **FiniSection.blocks().begin();
921 InitSym = &G.addDefinedSymbol(
922 Content&: B, Offset: 0, Name: *InitSymName, Size: B.getSize(), L: jitlink::Linkage::Strong,
923 S: jitlink::Scope::SideEffectsOnly, IsCallable: false, IsLive: true);
924 }
925
926 // Add keep-alive edges to anonymous symbols in all fini blocks.
927 for (auto *B : FiniSection.blocks()) {
928 if (B == &InitSym->getBlock())
929 continue;
930
931 auto &S = G.addAnonymousSymbol(Content&: *B, Offset: 0, Size: B->getSize(), IsCallable: false, IsLive: true);
932 InitSym->getBlock().addEdge(K: jitlink::Edge::KeepAlive, Offset: 0, Target&: S, Addend: 0);
933 }
934 }
935 }
936
937 return Error::success();
938}
939
940Error ELFNixPlatform::ELFNixPlatformPlugin::registerInitSections(
941 jitlink::LinkGraph &G, JITDylib &JD, bool IsBootstrapping) {
942 SmallVector<ExecutorAddrRange> ELFNixPlatformSecs;
943 LLVM_DEBUG(dbgs() << "ELFNixPlatform::registerInitSections\n");
944
945 SmallVector<jitlink::Section *> OrderedInitSections;
946 for (auto &Sec : G.sections())
947 if (isELFInitializerSection(SecName: Sec.getName()))
948 OrderedInitSections.push_back(Elt: &Sec);
949
950 // Helper to get section type and priority for sorting.
951 // Returns: {type_order, priority, has_priority}
952 // type_order: 0 = .init_array, 1 = .init, 2 = .ctors
953 auto getInitSectionInfo =
954 [](const jitlink::Section *Sec) -> std::tuple<int, uint64_t, bool> {
955 StringRef Name = Sec->getName();
956 if (Name.starts_with(Prefix: ".init_array")) {
957 StringRef PrioStr = Name;
958 uint64_t Prio = 0;
959 bool HasPrio = PrioStr.consume_front(Prefix: ".init_array.") &&
960 !PrioStr.getAsInteger(Radix: 10, Result&: Prio);
961 return {0, Prio, HasPrio};
962 }
963 if (Name.starts_with(Prefix: ".init"))
964 return {1, 0, false};
965 if (Name.starts_with(Prefix: ".ctors")) {
966 StringRef PrioStr = Name;
967 uint64_t Prio = 0;
968 bool HasPrio =
969 PrioStr.consume_front(Prefix: ".ctors.") && !PrioStr.getAsInteger(Radix: 10, Result&: Prio);
970 return {2, Prio, HasPrio};
971 }
972 return {3, 0, false};
973 };
974
975 // Sort init sections:
976 // 1. .init_array sections first (ascending priority - lower runs first)
977 // 2. .init sections next
978 // 3. .ctors sections last (descending priority - higher runs first, legacy
979 // behavior)
980 llvm::sort(C&: OrderedInitSections,
981 Comp: [&](const jitlink::Section *LHS, const jitlink::Section *RHS) {
982 auto [LType, LPrio, LHasPrio] = getInitSectionInfo(LHS);
983 auto [RType, RPrio, RHasPrio] = getInitSectionInfo(RHS);
984
985 if (LType != RType)
986 return LType < RType;
987
988 // Same type - sort by priority
989 if (LType == 0) {
990 // .init_array: ascending priority (lower priority number runs
991 // first)
992 if (LHasPrio && RHasPrio)
993 return LPrio < RPrio;
994 if (LHasPrio)
995 return true;
996 if (RHasPrio)
997 return false;
998 } else if (LType == 2) {
999 // .ctors: descending priority (higher priority number runs
1000 // first)
1001 if (LHasPrio && RHasPrio)
1002 return LPrio > RPrio;
1003 if (LHasPrio)
1004 return true;
1005 if (RHasPrio)
1006 return false;
1007 }
1008 return LHS->getName() < RHS->getName();
1009 });
1010
1011 for (auto &Sec : OrderedInitSections)
1012 ELFNixPlatformSecs.push_back(Elt: jitlink::SectionRange(*Sec).getRange());
1013
1014 // Dump the scraped inits.
1015 LLVM_DEBUG({
1016 dbgs() << "ELFNixPlatform: Scraped " << G.getName() << " init sections:\n";
1017 for (auto &Sec : G.sections()) {
1018 jitlink::SectionRange R(Sec);
1019 dbgs() << " " << Sec.getName() << ": " << R.getRange() << "\n";
1020 }
1021 });
1022
1023 ExecutorAddr HeaderAddr;
1024 {
1025 std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
1026 auto I = MP.JITDylibToHandleAddr.find(Val: &JD);
1027 assert(I != MP.JITDylibToHandleAddr.end() && "No header registered for JD");
1028 assert(I->second && "Null header registered for JD");
1029 HeaderAddr = I->second;
1030 }
1031
1032 using SPSRegisterInitSectionsArgs =
1033 SPSArgList<SPSExecutorAddr, SPSSequence<SPSExecutorAddrRange>>;
1034
1035 if (LLVM_UNLIKELY(IsBootstrapping)) {
1036 MP.Bootstrap.load()->addArgumentsToRTFnMap(
1037 func1: &MP.RegisterInitSections, func2: &MP.DeregisterInitSections,
1038 arg1: getArgDataBufferType<SPSRegisterInitSectionsArgs>(Args: HeaderAddr,
1039 Args: ELFNixPlatformSecs),
1040 arg2: getArgDataBufferType<SPSRegisterInitSectionsArgs>(Args: HeaderAddr,
1041 Args: ELFNixPlatformSecs));
1042 return Error::success();
1043 }
1044
1045 G.allocActions().push_back(
1046 x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSRegisterInitSectionsArgs>(
1047 FnAddr: MP.RegisterInitSections.Addr, Args: HeaderAddr, Args: ELFNixPlatformSecs)),
1048 .Dealloc: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSRegisterInitSectionsArgs>(
1049 FnAddr: MP.DeregisterInitSections.Addr, Args: HeaderAddr, Args: ELFNixPlatformSecs))});
1050
1051 return Error::success();
1052}
1053
1054Error ELFNixPlatform::ELFNixPlatformPlugin::registerFiniSections(
1055 jitlink::LinkGraph &G, JITDylib &JD, bool IsBootstrapping) {
1056 SmallVector<ExecutorAddrRange> ELFNixFiniSecs;
1057 LLVM_DEBUG(dbgs() << "ELFNixPlatform::registerFiniSections\n");
1058
1059 SmallVector<jitlink::Section *> OrderedFiniSections;
1060 for (auto &Sec : G.sections())
1061 if (isELFFinalizerSection(SecName: Sec.getName()))
1062 OrderedFiniSections.push_back(Elt: &Sec);
1063
1064 // Helper to get section type and priority for sorting.
1065 // Returns: {type_order, priority, has_priority}
1066 // type_order: 0 = .dtors, 1 = .fini, 2 = .fini_array
1067 auto getFiniSectionInfo =
1068 [](const jitlink::Section *Sec) -> std::tuple<int, uint64_t, bool> {
1069 StringRef Name = Sec->getName();
1070 if (Name.starts_with(Prefix: ".dtors")) {
1071 StringRef PrioStr = Name;
1072 uint64_t Prio = 0;
1073 bool HasPrio =
1074 PrioStr.consume_front(Prefix: ".dtors.") && !PrioStr.getAsInteger(Radix: 10, Result&: Prio);
1075 return {0, Prio, HasPrio};
1076 }
1077 if (Name.starts_with(Prefix: ".fini"))
1078 if (!Name.starts_with(Prefix: ".fini_array"))
1079 return {1, 0, false};
1080 if (Name.starts_with(Prefix: ".fini_array")) {
1081 StringRef PrioStr = Name;
1082 uint64_t Prio = 0;
1083 bool HasPrio = PrioStr.consume_front(Prefix: ".fini_array.") &&
1084 !PrioStr.getAsInteger(Radix: 10, Result&: Prio);
1085 return {2, Prio, HasPrio};
1086 }
1087 return {3, 0, false};
1088 };
1089
1090 // Sort fini sections:
1091 // 1. .dtors sections first (ascending priority, as they appear)
1092 // 2. .fini sections next
1093 // 3. .fini_array sections last (descending priority - higher runs first)
1094 llvm::sort(C&: OrderedFiniSections,
1095 Comp: [&](const jitlink::Section *LHS, const jitlink::Section *RHS) {
1096 auto [LType, LPrio, LHasPrio] = getFiniSectionInfo(LHS);
1097 auto [RType, RPrio, RHasPrio] = getFiniSectionInfo(RHS);
1098
1099 if (LType != RType)
1100 return LType < RType;
1101
1102 // Same type - sort by priority
1103 if (LType == 0) {
1104 // .dtors: no-priority first, then ascending priority (lower
1105 // runs first)
1106 if (LHasPrio && RHasPrio)
1107 return LPrio < RPrio;
1108 if (LHasPrio)
1109 return false;
1110 if (RHasPrio)
1111 return true;
1112 } else if (LType == 2) {
1113 // .fini_array: no-priority first, then descending priority
1114 // (higher runs first)
1115 if (LHasPrio && RHasPrio)
1116 return LPrio > RPrio;
1117 if (LHasPrio)
1118 return false;
1119 if (RHasPrio)
1120 return true;
1121 }
1122 return LHS->getName() < RHS->getName();
1123 });
1124
1125 for (auto *Sec : OrderedFiniSections)
1126 ELFNixFiniSecs.push_back(Elt: jitlink::SectionRange(*Sec).getRange());
1127
1128 if (ELFNixFiniSecs.empty())
1129 return Error::success();
1130
1131 // Dump the scraped finis.
1132 LLVM_DEBUG({
1133 dbgs() << "ELFNixPlatform: Scraped " << G.getName() << " fini sections:\n";
1134 for (auto *Sec : OrderedFiniSections) {
1135 jitlink::SectionRange R(*Sec);
1136 dbgs() << " " << Sec->getName() << ": " << R.getRange() << "\n";
1137 }
1138 });
1139
1140 ExecutorAddr HeaderAddr;
1141 {
1142 std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
1143 auto I = MP.JITDylibToHandleAddr.find(Val: &JD);
1144 assert(I != MP.JITDylibToHandleAddr.end() && "No header registered for JD");
1145 assert(I->second && "Null header registered for JD");
1146 HeaderAddr = I->second;
1147 }
1148
1149 using SPSRegisterFiniSectionsArgs =
1150 SPSArgList<SPSExecutorAddr, SPSSequence<SPSExecutorAddrRange>>;
1151
1152 if (LLVM_UNLIKELY(IsBootstrapping)) {
1153 MP.Bootstrap.load()->addArgumentsToRTFnMap(
1154 func1: &MP.RegisterFiniSections, func2: &MP.DeregisterFiniSections,
1155 arg1: getArgDataBufferType<SPSRegisterFiniSectionsArgs>(Args: HeaderAddr,
1156 Args: ELFNixFiniSecs),
1157 arg2: getArgDataBufferType<SPSRegisterFiniSectionsArgs>(Args: HeaderAddr,
1158 Args: ELFNixFiniSecs));
1159 return Error::success();
1160 }
1161
1162 G.allocActions().push_back(
1163 x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSRegisterFiniSectionsArgs>(
1164 FnAddr: MP.RegisterFiniSections.Addr, Args: HeaderAddr, Args: ELFNixFiniSecs)),
1165 .Dealloc: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSRegisterFiniSectionsArgs>(
1166 FnAddr: MP.DeregisterFiniSections.Addr, Args: HeaderAddr, Args: ELFNixFiniSecs))});
1167
1168 return Error::success();
1169}
1170
1171Error ELFNixPlatform::ELFNixPlatformPlugin::fixTLVSectionsAndEdges(
1172 jitlink::LinkGraph &G, JITDylib &JD) {
1173 auto TLSGetAddrSymbolName = G.intern(SymbolName: "__tls_get_addr");
1174 auto TLSDescResolveSymbolName = G.intern(SymbolName: "__tlsdesc_resolver");
1175 auto TLSGetOffsetSymbolName = G.intern(SymbolName: "__tls_get_offset");
1176 for (auto *Sym : G.external_symbols()) {
1177 if (Sym->getName() == TLSGetAddrSymbolName) {
1178 auto TLSGetAddr =
1179 MP.getExecutionSession().intern(SymName: "___orc_rt_elfnix_tls_get_addr");
1180 Sym->setName(std::move(TLSGetAddr));
1181 } else if (Sym->getName() == TLSDescResolveSymbolName) {
1182 auto TLSGetAddr =
1183 MP.getExecutionSession().intern(SymName: "___orc_rt_elfnix_tlsdesc_resolver");
1184 Sym->setName(std::move(TLSGetAddr));
1185 } else if (Sym->getName() == TLSGetOffsetSymbolName) {
1186 auto TLSGetAddr =
1187 MP.getExecutionSession().intern(SymName: "___orc_rt_elfnix_tls_get_offset");
1188 Sym->setName(std::move(TLSGetAddr));
1189 }
1190 }
1191
1192 auto *TLSInfoEntrySection = G.findSectionByName(Name: "$__TLSINFO");
1193
1194 if (TLSInfoEntrySection) {
1195 std::optional<uint64_t> Key;
1196 {
1197 std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
1198 auto I = MP.JITDylibToPThreadKey.find(Val: &JD);
1199 if (I != MP.JITDylibToPThreadKey.end())
1200 Key = I->second;
1201 }
1202 if (!Key) {
1203 if (auto KeyOrErr = MP.createPThreadKey())
1204 Key = *KeyOrErr;
1205 else
1206 return KeyOrErr.takeError();
1207 }
1208
1209 uint64_t PlatformKeyBits =
1210 support::endian::byte_swap(value: *Key, endian: G.getEndianness());
1211
1212 for (auto *B : TLSInfoEntrySection->blocks()) {
1213 // FIXME: The TLS descriptor byte length may different with different
1214 // ISA
1215 assert(B->getSize() == (G.getPointerSize() * 2) &&
1216 "TLS descriptor must be 2 words length");
1217 auto TLSInfoEntryContent = B->getMutableContent(G);
1218 memcpy(dest: TLSInfoEntryContent.data(), src: &PlatformKeyBits, n: G.getPointerSize());
1219 }
1220 }
1221
1222 return Error::success();
1223}
1224
1225} // End namespace orc.
1226} // End namespace llvm.
1227