1//===----- ELFNixPlatform.cpp - Utilities for executing ELFNix in Orc -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/Orc/ELFNixPlatform.h"
10#include "llvm/ExecutionEngine/Orc/Mangling.h"
11
12#include "llvm/ExecutionEngine/JITLink/aarch64.h"
13#include "llvm/ExecutionEngine/JITLink/loongarch.h"
14#include "llvm/ExecutionEngine/JITLink/mips.h"
15#include "llvm/ExecutionEngine/JITLink/ppc64.h"
16#include "llvm/ExecutionEngine/JITLink/systemz.h"
17#include "llvm/ExecutionEngine/JITLink/x86_64.h"
18#include "llvm/ExecutionEngine/Orc/BindCallControllerHandler.h"
19#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
20#include "llvm/ExecutionEngine/Orc/SPSHandlerSpec.h"
21#include "llvm/ExecutionEngine/Orc/SPSProxySpec.h"
22#include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h"
23#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
24#include "llvm/Support/Debug.h"
25#include "llvm/Support/Endian.h"
26#include <optional>
27
28#define DEBUG_TYPE "orc"
29
30using namespace llvm;
31using namespace llvm::orc;
32using namespace llvm::orc::shared;
33
34// Controller-interface descriptors for the ELFNix platform runtime's SPS
35// wrapper calls.
36namespace llvm::orc::elfnix_sps_ci {
37struct CreatePThreadKey {
38 static constexpr SymbolNameSpec Name =
39 SymbolNameSpec::c(Name: "__orc_rt_elfnix_create_pthread_key");
40 using SPSSig = SPSExpected<uint64_t>();
41};
42} // namespace llvm::orc::elfnix_sps_ci
43
44// Specs for the call-controller handlers that the ELFNix platform
45// runtime calls.
46namespace llvm::orc::elfnix_sps_handlers {
47struct RecordInitializers
48 : sps::HandlerSpec<SPSExpected<SPSELFNixJITDylibDepInfoMap>(
49 SPSExecutorAddr)> {
50 static constexpr SymbolNameSpec Name =
51 SymbolNameSpec::c(Name: "__orc_rt_elfnix_push_initializers_tag");
52};
53struct LookupSymbol : sps::HandlerSpec<SPSExpected<SPSExecutorAddr>(
54 SPSExecutorAddr, SPSString)> {
55 static constexpr SymbolNameSpec Name =
56 SymbolNameSpec::c(Name: "__orc_rt_elfnix_symbol_lookup_tag");
57};
58} // namespace llvm::orc::elfnix_sps_handlers
59
60namespace {
61
62template <typename SPSSerializer, typename... ArgTs>
63shared::WrapperFunctionCall::ArgDataBufferType
64getArgDataBufferType(const ArgTs &...Args) {
65 shared::WrapperFunctionCall::ArgDataBufferType ArgData;
66 ArgData.resize(SPSSerializer::size(Args...));
67 SPSOutputBuffer OB(ArgData.empty() ? nullptr : ArgData.data(),
68 ArgData.size());
69 if (SPSSerializer::serialize(OB, Args...))
70 return ArgData;
71 return {};
72}
73
74std::unique_ptr<jitlink::LinkGraph> createPlatformGraph(ELFNixPlatform &MOP,
75 std::string Name) {
76 auto &ES = MOP.getExecutionSession();
77 return std::make_unique<jitlink::LinkGraph>(
78 args: std::move(Name), args: ES.getSymbolStringPool(), args: ES.getTargetTriple(),
79 args: ES.getTargetTriple().getArchPointerBitWidth() / 8, args: SubtargetFeatures(),
80 args&: jitlink::getGenericEdgeKindName);
81}
82
83// Creates a Bootstrap-Complete LinkGraph to run deferred actions.
84class ELFNixPlatformCompleteBootstrapMaterializationUnit
85 : public MaterializationUnit {
86public:
87 ELFNixPlatformCompleteBootstrapMaterializationUnit(
88 ELFNixPlatform &MOP, StringRef PlatformJDName,
89 SymbolStringPtr CompleteBootstrapSymbol, DeferredRuntimeFnMap DeferredAAs,
90 ExecutorAddr ELFNixHeaderAddr, ExecutorAddr PlatformBootstrap,
91 ExecutorAddr PlatformShutdown, ExecutorAddr RegisterJITDylib,
92 ExecutorAddr DeregisterJITDylib)
93 : MaterializationUnit(
94 {{{CompleteBootstrapSymbol, JITSymbolFlags::None}}, nullptr}),
95 MOP(MOP), PlatformJDName(PlatformJDName),
96 CompleteBootstrapSymbol(std::move(CompleteBootstrapSymbol)),
97 DeferredAAsMap(std::move(DeferredAAs)),
98 ELFNixHeaderAddr(ELFNixHeaderAddr),
99 PlatformBootstrap(PlatformBootstrap),
100 PlatformShutdown(PlatformShutdown), RegisterJITDylib(RegisterJITDylib),
101 DeregisterJITDylib(DeregisterJITDylib) {}
102
103 StringRef getName() const override {
104 return "ELFNixPlatformCompleteBootstrap";
105 }
106
107 void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
108 using namespace jitlink;
109 auto G = createPlatformGraph(MOP, Name: "<OrcRTCompleteBootstrap>");
110 auto &PlaceholderSection =
111 G->createSection(Name: "__orc_rt_cplt_bs", Prot: MemProt::Read);
112 auto &PlaceholderBlock =
113 G->createZeroFillBlock(Parent&: PlaceholderSection, Size: 1, Address: ExecutorAddr(), Alignment: 1, AlignmentOffset: 0);
114 G->addDefinedSymbol(Content&: PlaceholderBlock, Offset: 0, Name: *CompleteBootstrapSymbol, Size: 1,
115 L: Linkage::Strong, S: Scope::Hidden, IsCallable: false, IsLive: true);
116
117 // 1. Bootstrap the platform support code.
118 G->allocActions().push_back(
119 x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
120 FnAddr: PlatformBootstrap, Args: ELFNixHeaderAddr)),
121 .Dealloc: cantFail(
122 ValOrErr: WrapperFunctionCall::Create<SPSArgList<>>(FnAddr: PlatformShutdown))});
123
124 // 2. Register the platform JITDylib.
125 G->allocActions().push_back(
126 x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<
127 SPSArgList<SPSString, SPSExecutorAddr>>(
128 FnAddr: RegisterJITDylib, Args: PlatformJDName, Args: ELFNixHeaderAddr)),
129 .Dealloc: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
130 FnAddr: DeregisterJITDylib, Args: ELFNixHeaderAddr))});
131
132 // 4. Add the deferred actions to the graph.
133 for (auto &[Fn, CallDatas] : DeferredAAsMap) {
134 for (auto &CallData : CallDatas) {
135 G->allocActions().push_back(
136 x: {.Finalize: WrapperFunctionCall(Fn.first->Addr, std::move(CallData.first)),
137 .Dealloc: WrapperFunctionCall(Fn.second->Addr, std::move(CallData.second))});
138 }
139 }
140
141 MOP.getObjectLinkingLayer().emit(R: std::move(R), G: std::move(G));
142 }
143
144 void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
145
146private:
147 ELFNixPlatform &MOP;
148 StringRef PlatformJDName;
149 SymbolStringPtr CompleteBootstrapSymbol;
150 DeferredRuntimeFnMap DeferredAAsMap;
151 ExecutorAddr ELFNixHeaderAddr;
152 ExecutorAddr PlatformBootstrap;
153 ExecutorAddr PlatformShutdown;
154 ExecutorAddr RegisterJITDylib;
155 ExecutorAddr DeregisterJITDylib;
156};
157
158class DSOHandleMaterializationUnit : public MaterializationUnit {
159public:
160 DSOHandleMaterializationUnit(ELFNixPlatform &ENP,
161 const SymbolStringPtr &DSOHandleSymbol)
162 : MaterializationUnit(
163 createDSOHandleSectionInterface(ENP, DSOHandleSymbol)),
164 ENP(ENP) {}
165
166 StringRef getName() const override { return "DSOHandleMU"; }
167
168 void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
169
170 auto &ES = ENP.getExecutionSession();
171
172 jitlink::Edge::Kind EdgeKind;
173 unsigned PointerSize = ES.getTargetTriple().getArchPointerBitWidth() / 8;
174
175 switch (ES.getTargetTriple().getArch()) {
176 case Triple::x86_64:
177 EdgeKind = jitlink::x86_64::Pointer64;
178 break;
179 case Triple::aarch64:
180 EdgeKind = jitlink::aarch64::Pointer64;
181 break;
182 case Triple::ppc64:
183 EdgeKind = jitlink::ppc64::Pointer64;
184 break;
185 case Triple::ppc64le:
186 EdgeKind = jitlink::ppc64::Pointer64;
187 break;
188 case Triple::loongarch64:
189 EdgeKind = jitlink::loongarch::Pointer64;
190 break;
191 case Triple::mips:
192 case Triple::mipsel:
193 EdgeKind = jitlink::mips::Pointer32;
194 break;
195 case Triple::mips64:
196 case Triple::mips64el:
197 if (ES.getTargetTriple().isABIN32()) {
198 EdgeKind = jitlink::mips::Pointer32;
199 PointerSize = 4;
200 } else
201 EdgeKind = jitlink::mips::Pointer64;
202 break;
203 case Triple::systemz:
204 EdgeKind = jitlink::systemz::Pointer64;
205 break;
206 default:
207 llvm_unreachable("Unrecognized architecture");
208 }
209
210 // void *__dso_handle = &__dso_handle;
211 auto G = std::make_unique<jitlink::LinkGraph>(
212 args: "<DSOHandleMU>", args: ES.getSymbolStringPool(), args: ES.getTargetTriple(),
213 args&: PointerSize, args: SubtargetFeatures(), args&: jitlink::getGenericEdgeKindName);
214 auto &DSOHandleSection =
215 G->createSection(Name: ".data.__dso_handle", Prot: MemProt::Read);
216 auto &DSOHandleBlock = G->createContentBlock(
217 Parent&: DSOHandleSection, Content: getDSOHandleContent(PointerSize: G->getPointerSize()),
218 Address: orc::ExecutorAddr(), Alignment: 8, AlignmentOffset: 0);
219 auto &DSOHandleSymbol = G->addDefinedSymbol(
220 Content&: DSOHandleBlock, Offset: 0, Name: *R->getInitializerSymbol(), Size: DSOHandleBlock.getSize(),
221 L: jitlink::Linkage::Strong, S: jitlink::Scope::Default, IsCallable: false, IsLive: true);
222 DSOHandleBlock.addEdge(K: EdgeKind, Offset: 0, Target&: DSOHandleSymbol, Addend: 0);
223
224 ENP.getObjectLinkingLayer().emit(R: std::move(R), G: std::move(G));
225 }
226
227 void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
228
229private:
230 static MaterializationUnit::Interface
231 createDSOHandleSectionInterface(ELFNixPlatform &ENP,
232 const SymbolStringPtr &DSOHandleSymbol) {
233 SymbolFlagsMap SymbolFlags;
234 SymbolFlags[DSOHandleSymbol] = JITSymbolFlags::Exported;
235 return MaterializationUnit::Interface(std::move(SymbolFlags),
236 DSOHandleSymbol);
237 }
238
239 ArrayRef<char> getDSOHandleContent(size_t PointerSize) {
240 static const char Content[8] = {0};
241 assert(PointerSize <= sizeof Content);
242 return {Content, PointerSize};
243 }
244
245 ELFNixPlatform &ENP;
246};
247
248} // end anonymous namespace
249
250namespace llvm {
251namespace orc {
252
253Expected<std::unique_ptr<ELFNixPlatform>>
254ELFNixPlatform::Create(ObjectLinkingLayer &ObjLinkingLayer,
255 JITDylib &PlatformJD,
256 std::unique_ptr<DefinitionGenerator> OrcRuntime,
257 std::optional<SymbolAliasMap> RuntimeAliases) {
258
259 auto &ES = ObjLinkingLayer.getExecutionSession();
260
261 // If the target is not supported then bail out immediately.
262 if (!supportedTarget(TT: ES.getTargetTriple()))
263 return make_error<StringError>(Args: "Unsupported ELFNixPlatform triple: " +
264 ES.getTargetTriple().str(),
265 Args: inconvertibleErrorCode());
266
267 // Create default aliases if the caller didn't supply any.
268 if (!RuntimeAliases) {
269 auto StandardRuntimeAliases = standardPlatformAliases(ES, PlatformJD);
270 if (!StandardRuntimeAliases)
271 return StandardRuntimeAliases.takeError();
272 RuntimeAliases = std::move(*StandardRuntimeAliases);
273 }
274
275 // Define the aliases.
276 if (auto Err = PlatformJD.define(MU: symbolAliases(Aliases: std::move(*RuntimeAliases))))
277 return std::move(Err);
278
279 {
280 // Add JIT dispatch reexports from bootstrap JITDylib.
281 MangleAndInterner Mangle(ES);
282 auto Exports = buildSimpleReexportsAliasMap(
283 SourceJD&: ES.getBootstrapJITDylib(),
284 Symbols: {{Mangle(rt::DispatchName), Mangle(rt::DispatchCtxName)}});
285 if (!Exports)
286 return Exports.takeError();
287 if (auto Err =
288 PlatformJD.define(MU: reexports(SourceJD&: ES.getBootstrapJITDylib(), Aliases: *Exports)))
289 return Err;
290 }
291
292 // Create the instance.
293 Error Err = Error::success();
294 auto P = std::unique_ptr<ELFNixPlatform>(new ELFNixPlatform(
295 ObjLinkingLayer, PlatformJD, std::move(OrcRuntime), Err));
296 if (Err)
297 return std::move(Err);
298 return std::move(P);
299}
300
301Expected<std::unique_ptr<ELFNixPlatform>>
302ELFNixPlatform::Create(ObjectLinkingLayer &ObjLinkingLayer,
303 JITDylib &PlatformJD, const char *OrcRuntimePath,
304 std::optional<SymbolAliasMap> RuntimeAliases) {
305
306 // Create a generator for the ORC runtime archive.
307 auto OrcRuntimeArchiveGenerator =
308 StaticLibraryDefinitionGenerator::Load(L&: ObjLinkingLayer, FileName: OrcRuntimePath);
309 if (!OrcRuntimeArchiveGenerator)
310 return OrcRuntimeArchiveGenerator.takeError();
311
312 return Create(ObjLinkingLayer, PlatformJD,
313 OrcRuntime: std::move(*OrcRuntimeArchiveGenerator),
314 RuntimeAliases: std::move(RuntimeAliases));
315}
316
317Error ELFNixPlatform::setupJITDylib(JITDylib &JD) {
318 if (auto Err = JD.define(MU: std::make_unique<DSOHandleMaterializationUnit>(
319 args&: *this, args&: DSOHandleSymbol)))
320 return Err;
321
322 return ES.lookup(SearchOrder: {&JD}, Symbol: DSOHandleSymbol).takeError();
323}
324
325Error ELFNixPlatform::teardownJITDylib(JITDylib &JD) {
326 std::lock_guard<std::mutex> Lock(PlatformMutex);
327 auto I = JITDylibToHandleAddr.find(Val: &JD);
328 if (I != JITDylibToHandleAddr.end()) {
329 assert(HandleAddrToJITDylib.count(I->second) &&
330 "HandleAddrToJITDylib missing entry");
331 HandleAddrToJITDylib.erase(Val: I->second);
332 JITDylibToHandleAddr.erase(I);
333 }
334 return Error::success();
335}
336
337Error ELFNixPlatform::notifyAdding(ResourceTracker &RT,
338 const MaterializationUnit &MU) {
339
340 auto &JD = RT.getJITDylib();
341 const auto &InitSym = MU.getInitializerSymbol();
342 if (!InitSym)
343 return Error::success();
344
345 RegisteredInitSymbols[&JD].add(Name: InitSym,
346 Flags: SymbolLookupFlags::WeaklyReferencedSymbol);
347 LLVM_DEBUG({
348 dbgs() << "ELFNixPlatform: Registered init symbol " << *InitSym
349 << " for MU " << MU.getName() << "\n";
350 });
351 return Error::success();
352}
353
354Error ELFNixPlatform::notifyRemoving(ResourceTracker &RT) {
355 llvm_unreachable("Not supported yet");
356}
357
358static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases,
359 ArrayRef<std::pair<const char *, const char *>> AL) {
360 for (auto &KV : AL) {
361 auto AliasName = ES.intern(SymName: KV.first);
362 assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map");
363 Aliases[std::move(AliasName)] = {ES.intern(SymName: KV.second),
364 JITSymbolFlags::Exported};
365 }
366}
367
368Expected<SymbolAliasMap>
369ELFNixPlatform::standardPlatformAliases(ExecutionSession &ES,
370 JITDylib &PlatformJD) {
371 SymbolAliasMap Aliases;
372 addAliases(ES, Aliases, AL: requiredCXXAliases());
373 addAliases(ES, Aliases, AL: standardRuntimeUtilityAliases());
374 addAliases(ES, Aliases, AL: standardLazyCompilationAliases());
375 return Aliases;
376}
377
378ArrayRef<std::pair<const char *, const char *>>
379ELFNixPlatform::requiredCXXAliases() {
380 static const std::pair<const char *, const char *> RequiredCXXAliases[] = {
381 {"__cxa_atexit", "__orc_rt_elfnix_cxa_atexit"},
382 {"atexit", "__orc_rt_elfnix_atexit"}};
383
384 return ArrayRef<std::pair<const char *, const char *>>(RequiredCXXAliases);
385}
386
387ArrayRef<std::pair<const char *, const char *>>
388ELFNixPlatform::standardRuntimeUtilityAliases() {
389 static const std::pair<const char *, const char *>
390 StandardRuntimeUtilityAliases[] = {
391 {"__orc_rt_run_program", "__orc_rt_elfnix_run_program"},
392 {"__orc_rt_jit_dlerror", "__orc_rt_elfnix_jit_dlerror"},
393 {"__orc_rt_jit_dlopen", "__orc_rt_elfnix_jit_dlopen"},
394 {"__orc_rt_jit_dlupdate", "__orc_rt_elfnix_jit_dlupdate"},
395 {"__orc_rt_jit_dlclose", "__orc_rt_elfnix_jit_dlclose"},
396 {"__orc_rt_jit_dlsym", "__orc_rt_elfnix_jit_dlsym"},
397 {"__orc_rt_log_error", "__orc_rt_log_error_to_stderr"}};
398
399 return ArrayRef<std::pair<const char *, const char *>>(
400 StandardRuntimeUtilityAliases);
401}
402
403ArrayRef<std::pair<const char *, const char *>>
404ELFNixPlatform::standardLazyCompilationAliases() {
405 static const std::pair<const char *, const char *>
406 StandardLazyCompilationAliases[] = {
407 {"__orc_rt_reenter", "__orc_rt_sysv_reenter"}};
408
409 return ArrayRef<std::pair<const char *, const char *>>(
410 StandardLazyCompilationAliases);
411}
412
413bool ELFNixPlatform::supportedTarget(const Triple &TT) {
414 switch (TT.getArch()) {
415 case Triple::x86_64:
416 case Triple::aarch64:
417 // FIXME: jitlink for ppc64 hasn't been well tested, leave it unsupported
418 // right now.
419 case Triple::ppc64le:
420 case Triple::loongarch64:
421 case Triple::systemz:
422 case Triple::mips:
423 case Triple::mipsel:
424 case Triple::mips64:
425 case Triple::mips64el:
426 return true;
427 default:
428 return false;
429 }
430}
431
432ELFNixPlatform::ELFNixPlatform(
433 ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD,
434 std::unique_ptr<DefinitionGenerator> OrcRuntimeGenerator, Error &Err)
435 : ES(ObjLinkingLayer.getExecutionSession()), PlatformJD(PlatformJD),
436 ObjLinkingLayer(ObjLinkingLayer),
437 DSOHandleSymbol(ES.intern(SymName: "__dso_handle")) {
438 ErrorAsOutParameter _(Err);
439 ObjLinkingLayer.addPlugin(P: std::make_unique<ELFNixPlatformPlugin>(args&: *this));
440
441 PlatformJD.addGenerator(DefGenerator: std::move(OrcRuntimeGenerator));
442
443 BootstrapInfo BI;
444 Bootstrap = &BI;
445
446 // PlatformJD hasn't been 'set-up' by the platform yet (since we're creating
447 // the platform now), so set it up.
448 if (auto E2 = setupJITDylib(PlatformJD)) {
449 Err = std::move(E2);
450 return;
451 }
452
453 // Step (2) Request runtime registration functions to trigger
454 // materialization..
455 if ((Err = ES.lookup(
456 SearchOrder: makeJITDylibSearchOrder(JDs: &PlatformJD),
457 Symbols: SymbolLookupSet(
458 {PlatformBootstrap.Name, PlatformShutdown.Name,
459 RegisterJITDylib.Name, DeregisterJITDylib.Name,
460 RegisterInitSections.Name, DeregisterInitSections.Name,
461 RegisterFiniSections.Name, DeregisterFiniSections.Name,
462 RegisterObjectSections.Name,
463 DeregisterObjectSections.Name, CreatePThreadKey.Name}))
464 .takeError()))
465 return;
466
467 // Step (3) Wait for any incidental linker work to complete.
468 {
469 std::unique_lock<std::mutex> Lock(BI.Mutex);
470 BI.CV.wait(lock&: Lock, p: [&]() { return BI.ActiveGraphs == 0; });
471 Bootstrap = nullptr;
472 }
473
474 // Step (4) Add complete-bootstrap materialization unit and request.
475 auto BootstrapCompleteSymbol =
476 ES.intern(SymName: "__orc_rt_elfnix_complete_bootstrap");
477 if ((Err = PlatformJD.define(
478 MU: std::make_unique<ELFNixPlatformCompleteBootstrapMaterializationUnit>(
479 args&: *this, args: PlatformJD.getName(), args&: BootstrapCompleteSymbol,
480 args: std::move(BI.DeferredRTFnMap), args&: BI.ELFNixHeaderAddr,
481 args&: PlatformBootstrap.Addr, args&: PlatformShutdown.Addr,
482 args&: RegisterJITDylib.Addr, args&: DeregisterJITDylib.Addr))))
483 return;
484 if ((Err = ES.lookup(SearchOrder: makeJITDylibSearchOrder(
485 JDs: &PlatformJD, Flags: JITDylibLookupFlags::MatchAllSymbols),
486 Symbol: std::move(BootstrapCompleteSymbol))
487 .takeError()))
488 return;
489
490 // Associate wrapper function tags with JIT-side function implementations.
491 if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) {
492 Err = std::move(E2);
493 return;
494 }
495}
496
497Error ELFNixPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) {
498 return ES.registerCallControllerHandlers(
499 JD&: PlatformJD,
500 Hs: bindCallControllerHandler<elfnix_sps_handlers::RecordInitializers>(
501 Instance: this, Method: &ELFNixPlatform::rt_recordInitializers),
502 Hs: bindCallControllerHandler<elfnix_sps_handlers::LookupSymbol>(
503 Instance: this, Method: &ELFNixPlatform::rt_lookupSymbol));
504}
505
506void ELFNixPlatform::pushInitializersLoop(
507 PushInitializersSendResultFn SendResult, JITDylibSP JD) {
508 DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols;
509 DenseMap<JITDylib *, SmallVector<JITDylib *>> JDDepMap;
510 SmallVector<JITDylib *, 16> Worklist({JD.get()});
511
512 ES.runSessionLocked(F: [&]() {
513 while (!Worklist.empty()) {
514 // FIXME: Check for defunct dylibs.
515
516 auto DepJD = Worklist.back();
517 Worklist.pop_back();
518
519 // If we've already visited this JITDylib on this iteration then continue.
520 auto [It, Inserted] = JDDepMap.try_emplace(Key: DepJD);
521 if (!Inserted)
522 continue;
523
524 // Add dep info.
525 auto &DM = It->second;
526 DepJD->withLinkOrderDo(F: [&](const JITDylibSearchOrder &O) {
527 for (auto &KV : O) {
528 if (KV.first == DepJD)
529 continue;
530 DM.push_back(Elt: KV.first);
531 Worklist.push_back(Elt: KV.first);
532 }
533 });
534
535 // Add any registered init symbols.
536 auto RISItr = RegisteredInitSymbols.find(Val: DepJD);
537 if (RISItr != RegisteredInitSymbols.end()) {
538 NewInitSymbols[DepJD] = std::move(RISItr->second);
539 RegisteredInitSymbols.erase(I: RISItr);
540 }
541 }
542 });
543
544 // If there are no further init symbols to look up then send the link order
545 // (as a list of header addresses) to the caller.
546 if (NewInitSymbols.empty()) {
547
548 // To make the list intelligible to the runtime we need to convert all
549 // JITDylib pointers to their header addresses. Only include JITDylibs
550 // that appear in the JITDylibToHandleAddr map (i.e. those that have been
551 // through setupJITDylib) -- bare JITDylibs aren't managed by the platform.
552 DenseMap<JITDylib *, ExecutorAddr> HeaderAddrs;
553 HeaderAddrs.reserve(NumEntries: JDDepMap.size());
554 {
555 std::lock_guard<std::mutex> Lock(PlatformMutex);
556 for (auto &KV : JDDepMap) {
557 auto I = JITDylibToHandleAddr.find(Val: KV.first);
558 if (I != JITDylibToHandleAddr.end())
559 HeaderAddrs[KV.first] = I->second;
560 }
561 }
562
563 // Build the dep info map to return.
564 ELFNixJITDylibDepInfoMap DIM;
565 DIM.reserve(n: JDDepMap.size());
566 for (auto &KV : JDDepMap) {
567 auto HI = HeaderAddrs.find(Val: KV.first);
568 // Skip unmanaged JITDylibs.
569 if (HI == HeaderAddrs.end())
570 continue;
571 auto H = HI->second;
572 ELFNixJITDylibDepInfo DepInfo;
573 for (auto &Dep : KV.second) {
574 auto HJ = HeaderAddrs.find(Val: Dep);
575 if (HJ != HeaderAddrs.end())
576 DepInfo.push_back(x: HJ->second);
577 }
578 DIM.push_back(x: std::make_pair(x&: H, y: std::move(DepInfo)));
579 }
580 SendResult(DIM);
581 return;
582 }
583
584 // Otherwise issue a lookup and re-run this phase when it completes.
585 lookupInitSymbolsAsync(
586 OnComplete: [this, SendResult = std::move(SendResult), JD](Error Err) mutable {
587 if (Err)
588 SendResult(std::move(Err));
589 else
590 pushInitializersLoop(SendResult: std::move(SendResult), JD);
591 },
592 ES, InitSyms: std::move(NewInitSymbols));
593}
594
595void ELFNixPlatform::rt_recordInitializers(
596 PushInitializersSendResultFn SendResult, ExecutorAddr JDHeaderAddr) {
597 JITDylibSP JD;
598 {
599 std::lock_guard<std::mutex> Lock(PlatformMutex);
600 auto I = HandleAddrToJITDylib.find(Val: JDHeaderAddr);
601 if (I != HandleAddrToJITDylib.end())
602 JD = I->second;
603 }
604
605 LLVM_DEBUG({
606 dbgs() << "ELFNixPlatform::rt_recordInitializers(" << JDHeaderAddr << ") ";
607 if (JD)
608 dbgs() << "pushing initializers for " << JD->getName() << "\n";
609 else
610 dbgs() << "No JITDylib for header address.\n";
611 });
612
613 if (!JD) {
614 SendResult(make_error<StringError>(Args: "No JITDylib with header addr " +
615 formatv(Fmt: "{0:x}", Vals&: JDHeaderAddr),
616 Args: inconvertibleErrorCode()));
617 return;
618 }
619
620 pushInitializersLoop(SendResult: std::move(SendResult), JD);
621}
622
623void ELFNixPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult,
624 ExecutorAddr Handle,
625 StringRef SymbolName) {
626 LLVM_DEBUG({
627 dbgs() << "ELFNixPlatform::rt_lookupSymbol(\"" << Handle << "\")\n";
628 });
629
630 JITDylib *JD = nullptr;
631
632 {
633 std::lock_guard<std::mutex> Lock(PlatformMutex);
634 auto I = HandleAddrToJITDylib.find(Val: Handle);
635 if (I != HandleAddrToJITDylib.end())
636 JD = I->second;
637 }
638
639 if (!JD) {
640 LLVM_DEBUG(dbgs() << " No JITDylib for handle " << Handle << "\n");
641 SendResult(make_error<StringError>(Args: "No JITDylib associated with handle " +
642 formatv(Fmt: "{0:x}", Vals&: Handle),
643 Args: inconvertibleErrorCode()));
644 return;
645 }
646
647 // Use functor class to work around XL build compiler issue on AIX.
648 class RtLookupNotifyComplete {
649 public:
650 RtLookupNotifyComplete(SendSymbolAddressFn &&SendResult)
651 : SendResult(std::move(SendResult)) {}
652 void operator()(Expected<SymbolMap> Result) {
653 if (Result) {
654 assert(Result->size() == 1 && "Unexpected result map count");
655 SendResult(Result->begin()->second.getAddress());
656 } else {
657 SendResult(Result.takeError());
658 }
659 }
660
661 private:
662 SendSymbolAddressFn SendResult;
663 };
664
665 ES.lookup(
666 K: LookupKind::DLSym, SearchOrder: {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}},
667 Symbols: SymbolLookupSet(ES.intern(SymName: SymbolName)), RequiredState: SymbolState::Ready,
668 NotifyComplete: RtLookupNotifyComplete(std::move(SendResult)), RegisterDependencies: NoDependenciesToRegister);
669}
670
671Error ELFNixPlatform::ELFNixPlatformPlugin::bootstrapPipelineStart(
672 jitlink::LinkGraph &G) {
673 // Increment the active graphs count in BootstrapInfo.
674 std::lock_guard<std::mutex> Lock(MP.Bootstrap.load()->Mutex);
675 ++MP.Bootstrap.load()->ActiveGraphs;
676 return Error::success();
677}
678
679Error ELFNixPlatform::ELFNixPlatformPlugin::
680 bootstrapPipelineRecordRuntimeFunctions(jitlink::LinkGraph &G) {
681 // Record bootstrap function names.
682 std::pair<StringRef, ExecutorAddr *> RuntimeSymbols[] = {
683 {*MP.DSOHandleSymbol, &MP.Bootstrap.load()->ELFNixHeaderAddr},
684 {*MP.PlatformBootstrap.Name, &MP.PlatformBootstrap.Addr},
685 {*MP.PlatformShutdown.Name, &MP.PlatformShutdown.Addr},
686 {*MP.RegisterJITDylib.Name, &MP.RegisterJITDylib.Addr},
687 {*MP.DeregisterJITDylib.Name, &MP.DeregisterJITDylib.Addr},
688 {*MP.RegisterObjectSections.Name, &MP.RegisterObjectSections.Addr},
689 {*MP.DeregisterObjectSections.Name, &MP.DeregisterObjectSections.Addr},
690 {*MP.RegisterInitSections.Name, &MP.RegisterInitSections.Addr},
691 {*MP.DeregisterInitSections.Name, &MP.DeregisterInitSections.Addr},
692 {*MP.RegisterFiniSections.Name, &MP.RegisterFiniSections.Addr},
693 {*MP.DeregisterFiniSections.Name, &MP.DeregisterFiniSections.Addr},
694 {*MP.CreatePThreadKey.Name, &MP.CreatePThreadKey.Addr}};
695
696 bool RegisterELFNixHeader = false;
697
698 for (auto *Sym : G.defined_symbols()) {
699 for (auto &RTSym : RuntimeSymbols) {
700 if (Sym->hasName() && *Sym->getName() == RTSym.first) {
701 if (*RTSym.second)
702 return make_error<StringError>(
703 Args: "Duplicate " + RTSym.first +
704 " detected during ELFNixPlatform bootstrap",
705 Args: inconvertibleErrorCode());
706
707 if (*Sym->getName() == *MP.DSOHandleSymbol)
708 RegisterELFNixHeader = true;
709
710 *RTSym.second = Sym->getAddress();
711 }
712 }
713 }
714
715 if (RegisterELFNixHeader) {
716 // If this graph defines the elfnix header symbol then create the internal
717 // mapping between it and PlatformJD.
718 std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
719 MP.JITDylibToHandleAddr[&MP.PlatformJD] =
720 MP.Bootstrap.load()->ELFNixHeaderAddr;
721 MP.HandleAddrToJITDylib[MP.Bootstrap.load()->ELFNixHeaderAddr] =
722 &MP.PlatformJD;
723 }
724
725 return Error::success();
726}
727
728Error ELFNixPlatform::ELFNixPlatformPlugin::bootstrapPipelineEnd(
729 jitlink::LinkGraph &G) {
730 std::lock_guard<std::mutex> Lock(MP.Bootstrap.load()->Mutex);
731 assert(MP.Bootstrap && "DeferredAAs reset before bootstrap completed");
732 --MP.Bootstrap.load()->ActiveGraphs;
733 // Notify Bootstrap->CV while holding the mutex because the mutex is
734 // also keeping Bootstrap->CV alive.
735 if (MP.Bootstrap.load()->ActiveGraphs == 0)
736 MP.Bootstrap.load()->CV.notify_all();
737 return Error::success();
738}
739
740Error ELFNixPlatform::registerPerObjectSections(
741 jitlink::LinkGraph &G, const ELFPerObjectSectionsToRegister &POSR,
742 bool IsBootstrapping) {
743 using SPSRegisterPerObjSectionsArgs =
744 SPSArgList<SPSELFPerObjectSectionsToRegister>;
745
746 if (LLVM_UNLIKELY(IsBootstrapping)) {
747 Bootstrap.load()->addArgumentsToRTFnMap(
748 func1: &RegisterObjectSections, func2: &DeregisterObjectSections,
749 arg1: getArgDataBufferType<SPSRegisterPerObjSectionsArgs>(Args: POSR),
750 arg2: getArgDataBufferType<SPSRegisterPerObjSectionsArgs>(Args: POSR));
751 return Error::success();
752 }
753
754 G.allocActions().push_back(
755 x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSRegisterPerObjSectionsArgs>(
756 FnAddr: RegisterObjectSections.Addr, Args: POSR)),
757 .Dealloc: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSRegisterPerObjSectionsArgs>(
758 FnAddr: DeregisterObjectSections.Addr, Args: POSR))});
759
760 return Error::success();
761}
762
763Expected<uint64_t> ELFNixPlatform::createPThreadKey() {
764 if (!CreatePThreadKey.Addr)
765 return make_error<StringError>(
766 Args: "Attempting to create pthread key in target, but runtime support has "
767 "not been loaded yet",
768 Args: inconvertibleErrorCode());
769
770 using CreatePThreadKeyProxy = Proxy<Expected<uint64_t>()>;
771 CreatePThreadKeyProxy CreateKey(
772 sps::ProxySpec<elfnix_sps_ci::CreatePThreadKey>::dispatch,
773 CreatePThreadKey.Addr);
774 return CreateKey(ES);
775}
776
777void ELFNixPlatform::ELFNixPlatformPlugin::modifyPassConfig(
778 MaterializationResponsibility &MR, jitlink::LinkGraph &LG,
779 jitlink::PassConfiguration &Config) {
780 using namespace jitlink;
781
782 bool InBootstrapPhase =
783 &MR.getTargetJITDylib() == &MP.PlatformJD && MP.Bootstrap;
784
785 // If we're in the bootstrap phase then increment the active graphs.
786 if (InBootstrapPhase) {
787 Config.PrePrunePasses.push_back(
788 x: [this](LinkGraph &G) { return bootstrapPipelineStart(G); });
789 Config.PostAllocationPasses.push_back(x: [this](LinkGraph &G) {
790 return bootstrapPipelineRecordRuntimeFunctions(G);
791 });
792 }
793
794 // If the initializer symbol is the __dso_handle symbol then just add
795 // the DSO handle support passes.
796 if (auto InitSymbol = MR.getInitializerSymbol()) {
797 if (InitSymbol == MP.DSOHandleSymbol && !InBootstrapPhase) {
798 addDSOHandleSupportPasses(MR, Config);
799 // The DSOHandle materialization unit doesn't require any other
800 // support, so we can bail out early.
801 return;
802 }
803
804 /// Preserve init sections.
805 Config.PrePrunePasses.push_back(
806 x: [this, &MR](jitlink::LinkGraph &G) -> Error {
807 if (auto Err = preserveInitSections(G, MR))
808 return Err;
809 return Error::success();
810 });
811 }
812
813 // Add passes for eh-frame and TLV support.
814 addEHAndTLVSupportPasses(MR, Config, IsBootstrapping: InBootstrapPhase);
815
816 // If the object contains initializers then add passes to record them.
817 Config.PostFixupPasses.push_back(x: [this, &JD = MR.getTargetJITDylib(),
818 InBootstrapPhase](jitlink::LinkGraph &G) {
819 return registerInitSections(G, JD, IsBootstrapping: InBootstrapPhase);
820 });
821
822 // If the object contains finalizers then add passes to record them.
823 Config.PostFixupPasses.push_back(x: [this, &JD = MR.getTargetJITDylib(),
824 InBootstrapPhase](jitlink::LinkGraph &G) {
825 return registerFiniSections(G, JD, IsBootstrapping: InBootstrapPhase);
826 });
827
828 // If we're in the bootstrap phase then steal allocation actions and then
829 // decrement the active graphs.
830 if (InBootstrapPhase)
831 Config.PostFixupPasses.push_back(
832 x: [this](LinkGraph &G) { return bootstrapPipelineEnd(G); });
833}
834
835void ELFNixPlatform::ELFNixPlatformPlugin::addDSOHandleSupportPasses(
836 MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) {
837
838 Config.PostAllocationPasses.push_back(x: [this, &JD = MR.getTargetJITDylib()](
839 jitlink::LinkGraph &G) -> Error {
840 auto I = llvm::find_if(Range: G.defined_symbols(), P: [this](jitlink::Symbol *Sym) {
841 return Sym->getName() == MP.DSOHandleSymbol;
842 });
843 assert(I != G.defined_symbols().end() && "Missing DSO handle symbol");
844 {
845 std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
846 auto HandleAddr = (*I)->getAddress();
847 MP.HandleAddrToJITDylib[HandleAddr] = &JD;
848 MP.JITDylibToHandleAddr[&JD] = HandleAddr;
849
850 G.allocActions().push_back(
851 x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<
852 SPSArgList<SPSString, SPSExecutorAddr>>(
853 FnAddr: MP.RegisterJITDylib.Addr, Args: JD.getName(), Args: HandleAddr)),
854 .Dealloc: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
855 FnAddr: MP.DeregisterJITDylib.Addr, Args: HandleAddr))});
856 }
857 return Error::success();
858 });
859}
860
861void ELFNixPlatform::ELFNixPlatformPlugin::addEHAndTLVSupportPasses(
862 MaterializationResponsibility &MR, jitlink::PassConfiguration &Config,
863 bool IsBootstrapping) {
864
865 // Insert TLV lowering at the start of the PostPrunePasses, since we want
866 // it to run before GOT/PLT lowering.
867
868 // TODO: Check that before the fixTLVSectionsAndEdges pass, the GOT/PLT build
869 // pass has done. Because the TLS descriptor need to be allocate in GOT.
870 Config.PostPrunePasses.push_back(
871 x: [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
872 return fixTLVSectionsAndEdges(G, JD);
873 });
874
875 // Add a pass to register the final addresses of the eh-frame and TLV sections
876 // with the runtime.
877 Config.PostFixupPasses.push_back(x: [this, IsBootstrapping](
878 jitlink::LinkGraph &G) -> Error {
879 ELFPerObjectSectionsToRegister POSR;
880
881 if (auto *EHFrameSection = G.findSectionByName(Name: ELFEHFrameSectionName)) {
882 jitlink::SectionRange R(*EHFrameSection);
883 if (!R.empty())
884 POSR.EHFrameSection = R.getRange();
885 }
886
887 // Get a pointer to the thread data section if there is one. It will be used
888 // below.
889 jitlink::Section *ThreadDataSection =
890 G.findSectionByName(Name: ELFThreadDataSectionName);
891
892 // Handle thread BSS section if there is one.
893 if (auto *ThreadBSSSection = G.findSectionByName(Name: ELFThreadBSSSectionName)) {
894 // If there's already a thread data section in this graph then merge the
895 // thread BSS section content into it, otherwise just treat the thread
896 // BSS section as the thread data section.
897 if (ThreadDataSection)
898 G.mergeSections(DstSection&: *ThreadDataSection, SrcSection&: *ThreadBSSSection);
899 else
900 ThreadDataSection = ThreadBSSSection;
901 }
902
903 // Having merged thread BSS (if present) and thread data (if present),
904 // record the resulting section range.
905 if (ThreadDataSection) {
906 jitlink::SectionRange R(*ThreadDataSection);
907 if (!R.empty())
908 POSR.ThreadDataSection = R.getRange();
909 }
910
911 if (POSR.EHFrameSection.Start || POSR.ThreadDataSection.Start) {
912 if (auto Err = MP.registerPerObjectSections(G, POSR, IsBootstrapping))
913 return Err;
914 }
915
916 return Error::success();
917 });
918}
919
920Error ELFNixPlatform::ELFNixPlatformPlugin::preserveInitSections(
921 jitlink::LinkGraph &G, MaterializationResponsibility &MR) {
922
923 if (const auto &InitSymName = MR.getInitializerSymbol()) {
924
925 jitlink::Symbol *InitSym = nullptr;
926
927 for (auto &InitSection : G.sections()) {
928 // Skip non-init sections.
929 if (!isELFInitializerSection(SecName: InitSection.getName()) ||
930 InitSection.empty())
931 continue;
932
933 // Create the init symbol if it has not been created already and attach it
934 // to the first block.
935 if (!InitSym) {
936 auto &B = **InitSection.blocks().begin();
937 InitSym = &G.addDefinedSymbol(
938 Content&: B, Offset: 0, Name: *InitSymName, Size: B.getSize(), L: jitlink::Linkage::Strong,
939 S: jitlink::Scope::SideEffectsOnly, IsCallable: false, IsLive: true);
940 }
941
942 // Add keep-alive edges to anonymous symbols in all other init blocks.
943 for (auto *B : InitSection.blocks()) {
944 if (B == &InitSym->getBlock())
945 continue;
946
947 auto &S = G.addAnonymousSymbol(Content&: *B, Offset: 0, Size: B->getSize(), IsCallable: false, IsLive: true);
948 InitSym->getBlock().addEdge(K: jitlink::Edge::KeepAlive, Offset: 0, Target&: S, Addend: 0);
949 }
950 }
951
952 // Also preserve fini sections (.fini_array, .fini, .dtors)
953 for (auto &FiniSection : G.sections()) {
954 // Skip non-fini sections.
955 if (!isELFFinalizerSection(SecName: FiniSection.getName()) || FiniSection.empty())
956 continue;
957
958 // Create the init symbol if it has not been created already and attach it
959 // to the first fini block.
960 if (!InitSym) {
961 auto &B = **FiniSection.blocks().begin();
962 InitSym = &G.addDefinedSymbol(
963 Content&: B, Offset: 0, Name: *InitSymName, Size: B.getSize(), L: jitlink::Linkage::Strong,
964 S: jitlink::Scope::SideEffectsOnly, IsCallable: false, IsLive: true);
965 }
966
967 // Add keep-alive edges to anonymous symbols in all fini blocks.
968 for (auto *B : FiniSection.blocks()) {
969 if (B == &InitSym->getBlock())
970 continue;
971
972 auto &S = G.addAnonymousSymbol(Content&: *B, Offset: 0, Size: B->getSize(), IsCallable: false, IsLive: true);
973 InitSym->getBlock().addEdge(K: jitlink::Edge::KeepAlive, Offset: 0, Target&: S, Addend: 0);
974 }
975 }
976 }
977
978 return Error::success();
979}
980
981Error ELFNixPlatform::ELFNixPlatformPlugin::registerInitSections(
982 jitlink::LinkGraph &G, JITDylib &JD, bool IsBootstrapping) {
983 SmallVector<ExecutorAddrRange> ELFNixPlatformSecs;
984 LLVM_DEBUG(dbgs() << "ELFNixPlatform::registerInitSections\n");
985
986 SmallVector<jitlink::Section *> OrderedInitSections;
987 for (auto &Sec : G.sections())
988 if (isELFInitializerSection(SecName: Sec.getName()))
989 OrderedInitSections.push_back(Elt: &Sec);
990
991 // Helper to get section type and priority for sorting.
992 // Returns: {type_order, priority, has_priority}
993 // type_order: 0 = .init_array, 1 = .init, 2 = .ctors
994 auto getInitSectionInfo =
995 [](const jitlink::Section *Sec) -> std::tuple<int, uint64_t, bool> {
996 StringRef Name = Sec->getName();
997 if (Name.starts_with(Prefix: ".init_array")) {
998 StringRef PrioStr = Name;
999 uint64_t Prio = 0;
1000 bool HasPrio = PrioStr.consume_front(Prefix: ".init_array.") &&
1001 !PrioStr.getAsInteger(Radix: 10, Result&: Prio);
1002 return {0, Prio, HasPrio};
1003 }
1004 if (Name.starts_with(Prefix: ".init"))
1005 return {1, 0, false};
1006 if (Name.starts_with(Prefix: ".ctors")) {
1007 StringRef PrioStr = Name;
1008 uint64_t Prio = 0;
1009 bool HasPrio =
1010 PrioStr.consume_front(Prefix: ".ctors.") && !PrioStr.getAsInteger(Radix: 10, Result&: Prio);
1011 return {2, Prio, HasPrio};
1012 }
1013 return {3, 0, false};
1014 };
1015
1016 // Sort init sections:
1017 // 1. .init_array sections first (ascending priority - lower runs first)
1018 // 2. .init sections next
1019 // 3. .ctors sections last (descending priority - higher runs first, legacy
1020 // behavior)
1021 llvm::sort(C&: OrderedInitSections,
1022 Comp: [&](const jitlink::Section *LHS, const jitlink::Section *RHS) {
1023 auto [LType, LPrio, LHasPrio] = getInitSectionInfo(LHS);
1024 auto [RType, RPrio, RHasPrio] = getInitSectionInfo(RHS);
1025
1026 if (LType != RType)
1027 return LType < RType;
1028
1029 // Same type - sort by priority
1030 if (LType == 0) {
1031 // .init_array: ascending priority (lower priority number runs
1032 // first)
1033 if (LHasPrio && RHasPrio)
1034 return LPrio < RPrio;
1035 if (LHasPrio)
1036 return true;
1037 if (RHasPrio)
1038 return false;
1039 } else if (LType == 2) {
1040 // .ctors: descending priority (higher priority number runs
1041 // first)
1042 if (LHasPrio && RHasPrio)
1043 return LPrio > RPrio;
1044 if (LHasPrio)
1045 return true;
1046 if (RHasPrio)
1047 return false;
1048 }
1049 return LHS->getName() < RHS->getName();
1050 });
1051
1052 for (auto &Sec : OrderedInitSections)
1053 ELFNixPlatformSecs.push_back(Elt: jitlink::SectionRange(*Sec).getRange());
1054
1055 // Dump the scraped inits.
1056 LLVM_DEBUG({
1057 dbgs() << "ELFNixPlatform: Scraped " << G.getName() << " init sections:\n";
1058 for (auto &Sec : G.sections()) {
1059 jitlink::SectionRange R(Sec);
1060 dbgs() << " " << Sec.getName() << ": " << R.getRange() << "\n";
1061 }
1062 });
1063
1064 ExecutorAddr HeaderAddr;
1065 {
1066 std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
1067 auto I = MP.JITDylibToHandleAddr.find(Val: &JD);
1068 assert(I != MP.JITDylibToHandleAddr.end() && "No header registered for JD");
1069 assert(I->second && "Null header registered for JD");
1070 HeaderAddr = I->second;
1071 }
1072
1073 using SPSRegisterInitSectionsArgs =
1074 SPSArgList<SPSExecutorAddr, SPSSequence<SPSExecutorAddrRange>>;
1075
1076 if (LLVM_UNLIKELY(IsBootstrapping)) {
1077 MP.Bootstrap.load()->addArgumentsToRTFnMap(
1078 func1: &MP.RegisterInitSections, func2: &MP.DeregisterInitSections,
1079 arg1: getArgDataBufferType<SPSRegisterInitSectionsArgs>(Args: HeaderAddr,
1080 Args: ELFNixPlatformSecs),
1081 arg2: getArgDataBufferType<SPSRegisterInitSectionsArgs>(Args: HeaderAddr,
1082 Args: ELFNixPlatformSecs));
1083 return Error::success();
1084 }
1085
1086 G.allocActions().push_back(
1087 x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSRegisterInitSectionsArgs>(
1088 FnAddr: MP.RegisterInitSections.Addr, Args: HeaderAddr, Args: ELFNixPlatformSecs)),
1089 .Dealloc: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSRegisterInitSectionsArgs>(
1090 FnAddr: MP.DeregisterInitSections.Addr, Args: HeaderAddr, Args: ELFNixPlatformSecs))});
1091
1092 return Error::success();
1093}
1094
1095Error ELFNixPlatform::ELFNixPlatformPlugin::registerFiniSections(
1096 jitlink::LinkGraph &G, JITDylib &JD, bool IsBootstrapping) {
1097 SmallVector<ExecutorAddrRange> ELFNixFiniSecs;
1098 LLVM_DEBUG(dbgs() << "ELFNixPlatform::registerFiniSections\n");
1099
1100 SmallVector<jitlink::Section *> OrderedFiniSections;
1101 for (auto &Sec : G.sections())
1102 if (isELFFinalizerSection(SecName: Sec.getName()))
1103 OrderedFiniSections.push_back(Elt: &Sec);
1104
1105 // Helper to get section type and priority for sorting.
1106 // Returns: {type_order, priority, has_priority}
1107 // type_order: 0 = .dtors, 1 = .fini, 2 = .fini_array
1108 auto getFiniSectionInfo =
1109 [](const jitlink::Section *Sec) -> std::tuple<int, uint64_t, bool> {
1110 StringRef Name = Sec->getName();
1111 if (Name.starts_with(Prefix: ".dtors")) {
1112 StringRef PrioStr = Name;
1113 uint64_t Prio = 0;
1114 bool HasPrio =
1115 PrioStr.consume_front(Prefix: ".dtors.") && !PrioStr.getAsInteger(Radix: 10, Result&: Prio);
1116 return {0, Prio, HasPrio};
1117 }
1118 if (Name.starts_with(Prefix: ".fini"))
1119 if (!Name.starts_with(Prefix: ".fini_array"))
1120 return {1, 0, false};
1121 if (Name.starts_with(Prefix: ".fini_array")) {
1122 StringRef PrioStr = Name;
1123 uint64_t Prio = 0;
1124 bool HasPrio = PrioStr.consume_front(Prefix: ".fini_array.") &&
1125 !PrioStr.getAsInteger(Radix: 10, Result&: Prio);
1126 return {2, Prio, HasPrio};
1127 }
1128 return {3, 0, false};
1129 };
1130
1131 // Sort fini sections:
1132 // 1. .dtors sections first (ascending priority, as they appear)
1133 // 2. .fini sections next
1134 // 3. .fini_array sections last (descending priority - higher runs first)
1135 llvm::sort(C&: OrderedFiniSections,
1136 Comp: [&](const jitlink::Section *LHS, const jitlink::Section *RHS) {
1137 auto [LType, LPrio, LHasPrio] = getFiniSectionInfo(LHS);
1138 auto [RType, RPrio, RHasPrio] = getFiniSectionInfo(RHS);
1139
1140 if (LType != RType)
1141 return LType < RType;
1142
1143 // Same type - sort by priority
1144 if (LType == 0) {
1145 // .dtors: no-priority first, then ascending priority (lower
1146 // runs first)
1147 if (LHasPrio && RHasPrio)
1148 return LPrio < RPrio;
1149 if (LHasPrio)
1150 return false;
1151 if (RHasPrio)
1152 return true;
1153 } else if (LType == 2) {
1154 // .fini_array: no-priority first, then descending priority
1155 // (higher runs first)
1156 if (LHasPrio && RHasPrio)
1157 return LPrio > RPrio;
1158 if (LHasPrio)
1159 return false;
1160 if (RHasPrio)
1161 return true;
1162 }
1163 return LHS->getName() < RHS->getName();
1164 });
1165
1166 for (auto *Sec : OrderedFiniSections)
1167 ELFNixFiniSecs.push_back(Elt: jitlink::SectionRange(*Sec).getRange());
1168
1169 if (ELFNixFiniSecs.empty())
1170 return Error::success();
1171
1172 // Dump the scraped finis.
1173 LLVM_DEBUG({
1174 dbgs() << "ELFNixPlatform: Scraped " << G.getName() << " fini sections:\n";
1175 for (auto *Sec : OrderedFiniSections) {
1176 jitlink::SectionRange R(*Sec);
1177 dbgs() << " " << Sec->getName() << ": " << R.getRange() << "\n";
1178 }
1179 });
1180
1181 ExecutorAddr HeaderAddr;
1182 {
1183 std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
1184 auto I = MP.JITDylibToHandleAddr.find(Val: &JD);
1185 assert(I != MP.JITDylibToHandleAddr.end() && "No header registered for JD");
1186 assert(I->second && "Null header registered for JD");
1187 HeaderAddr = I->second;
1188 }
1189
1190 using SPSRegisterFiniSectionsArgs =
1191 SPSArgList<SPSExecutorAddr, SPSSequence<SPSExecutorAddrRange>>;
1192
1193 if (LLVM_UNLIKELY(IsBootstrapping)) {
1194 MP.Bootstrap.load()->addArgumentsToRTFnMap(
1195 func1: &MP.RegisterFiniSections, func2: &MP.DeregisterFiniSections,
1196 arg1: getArgDataBufferType<SPSRegisterFiniSectionsArgs>(Args: HeaderAddr,
1197 Args: ELFNixFiniSecs),
1198 arg2: getArgDataBufferType<SPSRegisterFiniSectionsArgs>(Args: HeaderAddr,
1199 Args: ELFNixFiniSecs));
1200 return Error::success();
1201 }
1202
1203 G.allocActions().push_back(
1204 x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSRegisterFiniSectionsArgs>(
1205 FnAddr: MP.RegisterFiniSections.Addr, Args: HeaderAddr, Args: ELFNixFiniSecs)),
1206 .Dealloc: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSRegisterFiniSectionsArgs>(
1207 FnAddr: MP.DeregisterFiniSections.Addr, Args: HeaderAddr, Args: ELFNixFiniSecs))});
1208
1209 return Error::success();
1210}
1211
1212Error ELFNixPlatform::ELFNixPlatformPlugin::fixTLVSectionsAndEdges(
1213 jitlink::LinkGraph &G, JITDylib &JD) {
1214 auto TLSGetAddrSymbolName = G.intern(SymbolName: "__tls_get_addr");
1215 auto TLSDescResolveSymbolName = G.intern(SymbolName: "__tlsdesc_resolver");
1216 auto TLSGetOffsetSymbolName = G.intern(SymbolName: "__tls_get_offset");
1217 for (auto *Sym : G.external_symbols()) {
1218 if (Sym->getName() == TLSGetAddrSymbolName) {
1219 auto TLSGetAddr =
1220 MP.getExecutionSession().intern(SymName: "___orc_rt_elfnix_tls_get_addr");
1221 Sym->setName(std::move(TLSGetAddr));
1222 } else if (Sym->getName() == TLSDescResolveSymbolName) {
1223 auto TLSGetAddr =
1224 MP.getExecutionSession().intern(SymName: "___orc_rt_elfnix_tlsdesc_resolver");
1225 Sym->setName(std::move(TLSGetAddr));
1226 } else if (Sym->getName() == TLSGetOffsetSymbolName) {
1227 auto TLSGetAddr =
1228 MP.getExecutionSession().intern(SymName: "___orc_rt_elfnix_tls_get_offset");
1229 Sym->setName(std::move(TLSGetAddr));
1230 }
1231 }
1232
1233 auto *TLSInfoEntrySection = G.findSectionByName(Name: "$__TLSINFO");
1234
1235 if (TLSInfoEntrySection) {
1236 std::optional<uint64_t> Key;
1237 {
1238 std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
1239 auto I = MP.JITDylibToPThreadKey.find(Val: &JD);
1240 if (I != MP.JITDylibToPThreadKey.end())
1241 Key = I->second;
1242 }
1243 if (!Key) {
1244 if (auto KeyOrErr = MP.createPThreadKey())
1245 Key = *KeyOrErr;
1246 else
1247 return KeyOrErr.takeError();
1248 }
1249
1250 for (auto *B : TLSInfoEntrySection->blocks()) {
1251 // FIXME: The TLS descriptor byte length may different with different
1252 // ISA
1253 assert(B->getSize() == (G.getPointerSize() * 2) &&
1254 "TLS descriptor must be 2 words length");
1255 auto TLSInfoEntryContent = B->getMutableContent(G);
1256 if (G.getPointerSize() == 4)
1257 support::endian::write<uint32_t>(memory: TLSInfoEntryContent.data(), value: *Key,
1258 endian: G.getEndianness());
1259 else
1260 support::endian::write<uint64_t>(memory: TLSInfoEntryContent.data(), value: *Key,
1261 endian: G.getEndianness());
1262 }
1263 }
1264
1265 return Error::success();
1266}
1267
1268} // End namespace orc.
1269} // End namespace llvm.
1270