1//===------ ELFNixPlatform.cpp - Utilities for executing MachO in Orc -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/Orc/ELFNixPlatform.h"
10
11#include "llvm/BinaryFormat/ELF.h"
12#include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h"
13#include "llvm/ExecutionEngine/JITLink/aarch64.h"
14#include "llvm/ExecutionEngine/JITLink/ppc64.h"
15#include "llvm/ExecutionEngine/JITLink/x86_64.h"
16#include "llvm/ExecutionEngine/Orc/DebugUtils.h"
17#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
18#include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h"
19#include "llvm/Support/BinaryByteStream.h"
20#include "llvm/Support/Debug.h"
21#include <optional>
22
23#define DEBUG_TYPE "orc"
24
25using namespace llvm;
26using namespace llvm::orc;
27using namespace llvm::orc::shared;
28
29namespace {
30
31class DSOHandleMaterializationUnit : public MaterializationUnit {
32public:
33 DSOHandleMaterializationUnit(ELFNixPlatform &ENP,
34 const SymbolStringPtr &DSOHandleSymbol)
35 : MaterializationUnit(
36 createDSOHandleSectionInterface(ENP, DSOHandleSymbol)),
37 ENP(ENP) {}
38
39 StringRef getName() const override { return "DSOHandleMU"; }
40
41 void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
42 unsigned PointerSize;
43 llvm::endianness Endianness;
44 jitlink::Edge::Kind EdgeKind;
45 const auto &TT = ENP.getExecutionSession().getTargetTriple();
46
47 switch (TT.getArch()) {
48 case Triple::x86_64:
49 PointerSize = 8;
50 Endianness = llvm::endianness::little;
51 EdgeKind = jitlink::x86_64::Pointer64;
52 break;
53 case Triple::aarch64:
54 PointerSize = 8;
55 Endianness = llvm::endianness::little;
56 EdgeKind = jitlink::aarch64::Pointer64;
57 break;
58 case Triple::ppc64:
59 PointerSize = 8;
60 Endianness = llvm::endianness::big;
61 EdgeKind = jitlink::ppc64::Pointer64;
62 break;
63 case Triple::ppc64le:
64 PointerSize = 8;
65 Endianness = llvm::endianness::little;
66 EdgeKind = jitlink::ppc64::Pointer64;
67 break;
68 default:
69 llvm_unreachable("Unrecognized architecture");
70 }
71
72 // void *__dso_handle = &__dso_handle;
73 auto G = std::make_unique<jitlink::LinkGraph>(
74 args: "<DSOHandleMU>", args: TT, args&: PointerSize, args&: Endianness,
75 args&: jitlink::getGenericEdgeKindName);
76 auto &DSOHandleSection =
77 G->createSection(Name: ".data.__dso_handle", Prot: MemProt::Read);
78 auto &DSOHandleBlock = G->createContentBlock(
79 Parent&: DSOHandleSection, Content: getDSOHandleContent(PointerSize), Address: orc::ExecutorAddr(),
80 Alignment: 8, AlignmentOffset: 0);
81 auto &DSOHandleSymbol = G->addDefinedSymbol(
82 Content&: DSOHandleBlock, Offset: 0, Name: *R->getInitializerSymbol(), Size: DSOHandleBlock.getSize(),
83 L: jitlink::Linkage::Strong, S: jitlink::Scope::Default, IsCallable: false, IsLive: true);
84 DSOHandleBlock.addEdge(K: EdgeKind, Offset: 0, Target&: DSOHandleSymbol, Addend: 0);
85
86 ENP.getObjectLinkingLayer().emit(R: std::move(R), G: std::move(G));
87 }
88
89 void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
90
91private:
92 static MaterializationUnit::Interface
93 createDSOHandleSectionInterface(ELFNixPlatform &ENP,
94 const SymbolStringPtr &DSOHandleSymbol) {
95 SymbolFlagsMap SymbolFlags;
96 SymbolFlags[DSOHandleSymbol] = JITSymbolFlags::Exported;
97 return MaterializationUnit::Interface(std::move(SymbolFlags),
98 DSOHandleSymbol);
99 }
100
101 ArrayRef<char> getDSOHandleContent(size_t PointerSize) {
102 static const char Content[8] = {0};
103 assert(PointerSize <= sizeof Content);
104 return {Content, PointerSize};
105 }
106
107 ELFNixPlatform &ENP;
108};
109
110} // end anonymous namespace
111
112namespace llvm {
113namespace orc {
114
115Expected<std::unique_ptr<ELFNixPlatform>> ELFNixPlatform::Create(
116 ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer,
117 JITDylib &PlatformJD, std::unique_ptr<DefinitionGenerator> OrcRuntime,
118 std::optional<SymbolAliasMap> RuntimeAliases) {
119
120 // If the target is not supported then bail out immediately.
121 if (!supportedTarget(TT: ES.getTargetTriple()))
122 return make_error<StringError>(Args: "Unsupported ELFNixPlatform triple: " +
123 ES.getTargetTriple().str(),
124 Args: inconvertibleErrorCode());
125
126 auto &EPC = ES.getExecutorProcessControl();
127
128 // Create default aliases if the caller didn't supply any.
129 if (!RuntimeAliases) {
130 auto StandardRuntimeAliases = standardPlatformAliases(ES, PlatformJD);
131 if (!StandardRuntimeAliases)
132 return StandardRuntimeAliases.takeError();
133 RuntimeAliases = std::move(*StandardRuntimeAliases);
134 }
135
136 // Define the aliases.
137 if (auto Err = PlatformJD.define(MU: symbolAliases(Aliases: std::move(*RuntimeAliases))))
138 return std::move(Err);
139
140 // Add JIT-dispatch function support symbols.
141 if (auto Err = PlatformJD.define(
142 MU: absoluteSymbols(Symbols: {{ES.intern(SymName: "__orc_rt_jit_dispatch"),
143 {EPC.getJITDispatchInfo().JITDispatchFunction,
144 JITSymbolFlags::Exported}},
145 {ES.intern(SymName: "__orc_rt_jit_dispatch_ctx"),
146 {EPC.getJITDispatchInfo().JITDispatchContext,
147 JITSymbolFlags::Exported}}})))
148 return std::move(Err);
149
150 // Create the instance.
151 Error Err = Error::success();
152 auto P = std::unique_ptr<ELFNixPlatform>(new ELFNixPlatform(
153 ES, ObjLinkingLayer, PlatformJD, std::move(OrcRuntime), Err));
154 if (Err)
155 return std::move(Err);
156 return std::move(P);
157}
158
159Expected<std::unique_ptr<ELFNixPlatform>>
160ELFNixPlatform::Create(ExecutionSession &ES,
161 ObjectLinkingLayer &ObjLinkingLayer,
162 JITDylib &PlatformJD, const char *OrcRuntimePath,
163 std::optional<SymbolAliasMap> RuntimeAliases) {
164
165 // Create a generator for the ORC runtime archive.
166 auto OrcRuntimeArchiveGenerator =
167 StaticLibraryDefinitionGenerator::Load(L&: ObjLinkingLayer, FileName: OrcRuntimePath);
168 if (!OrcRuntimeArchiveGenerator)
169 return OrcRuntimeArchiveGenerator.takeError();
170
171 return Create(ES, ObjLinkingLayer, PlatformJD,
172 OrcRuntime: std::move(*OrcRuntimeArchiveGenerator),
173 RuntimeAliases: std::move(RuntimeAliases));
174}
175
176Error ELFNixPlatform::setupJITDylib(JITDylib &JD) {
177 return JD.define(
178 MU: std::make_unique<DSOHandleMaterializationUnit>(args&: *this, args&: DSOHandleSymbol));
179}
180
181Error ELFNixPlatform::teardownJITDylib(JITDylib &JD) {
182 return Error::success();
183}
184
185Error ELFNixPlatform::notifyAdding(ResourceTracker &RT,
186 const MaterializationUnit &MU) {
187 auto &JD = RT.getJITDylib();
188 const auto &InitSym = MU.getInitializerSymbol();
189 if (!InitSym)
190 return Error::success();
191
192 RegisteredInitSymbols[&JD].add(Name: InitSym,
193 Flags: SymbolLookupFlags::WeaklyReferencedSymbol);
194 LLVM_DEBUG({
195 dbgs() << "ELFNixPlatform: Registered init symbol " << *InitSym
196 << " for MU " << MU.getName() << "\n";
197 });
198 return Error::success();
199}
200
201Error ELFNixPlatform::notifyRemoving(ResourceTracker &RT) {
202 llvm_unreachable("Not supported yet");
203}
204
205static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases,
206 ArrayRef<std::pair<const char *, const char *>> AL) {
207 for (auto &KV : AL) {
208 auto AliasName = ES.intern(SymName: KV.first);
209 assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map");
210 Aliases[std::move(AliasName)] = {ES.intern(SymName: KV.second),
211 JITSymbolFlags::Exported};
212 }
213}
214
215Expected<SymbolAliasMap>
216ELFNixPlatform::standardPlatformAliases(ExecutionSession &ES,
217 JITDylib &PlatformJD) {
218 SymbolAliasMap Aliases;
219 addAliases(ES, Aliases, AL: requiredCXXAliases());
220 addAliases(ES, Aliases, AL: standardRuntimeUtilityAliases());
221 return Aliases;
222}
223
224ArrayRef<std::pair<const char *, const char *>>
225ELFNixPlatform::requiredCXXAliases() {
226 static const std::pair<const char *, const char *> RequiredCXXAliases[] = {
227 {"__cxa_atexit", "__orc_rt_elfnix_cxa_atexit"},
228 {"atexit", "__orc_rt_elfnix_atexit"}};
229
230 return ArrayRef<std::pair<const char *, const char *>>(RequiredCXXAliases);
231}
232
233ArrayRef<std::pair<const char *, const char *>>
234ELFNixPlatform::standardRuntimeUtilityAliases() {
235 static const std::pair<const char *, const char *>
236 StandardRuntimeUtilityAliases[] = {
237 {"__orc_rt_run_program", "__orc_rt_elfnix_run_program"},
238 {"__orc_rt_jit_dlerror", "__orc_rt_elfnix_jit_dlerror"},
239 {"__orc_rt_jit_dlopen", "__orc_rt_elfnix_jit_dlopen"},
240 {"__orc_rt_jit_dlclose", "__orc_rt_elfnix_jit_dlclose"},
241 {"__orc_rt_jit_dlsym", "__orc_rt_elfnix_jit_dlsym"},
242 {"__orc_rt_log_error", "__orc_rt_log_error_to_stderr"}};
243
244 return ArrayRef<std::pair<const char *, const char *>>(
245 StandardRuntimeUtilityAliases);
246}
247
248bool ELFNixPlatform::supportedTarget(const Triple &TT) {
249 switch (TT.getArch()) {
250 case Triple::x86_64:
251 case Triple::aarch64:
252 // FIXME: jitlink for ppc64 hasn't been well tested, leave it unsupported
253 // right now.
254 case Triple::ppc64le:
255 return true;
256 default:
257 return false;
258 }
259}
260
261ELFNixPlatform::ELFNixPlatform(
262 ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer,
263 JITDylib &PlatformJD,
264 std::unique_ptr<DefinitionGenerator> OrcRuntimeGenerator, Error &Err)
265 : ES(ES), ObjLinkingLayer(ObjLinkingLayer),
266 DSOHandleSymbol(ES.intern(SymName: "__dso_handle")) {
267 ErrorAsOutParameter _(&Err);
268
269 ObjLinkingLayer.addPlugin(P: std::make_unique<ELFNixPlatformPlugin>(args&: *this));
270
271 PlatformJD.addGenerator(DefGenerator: std::move(OrcRuntimeGenerator));
272
273 // PlatformJD hasn't been 'set-up' by the platform yet (since we're creating
274 // the platform now), so set it up.
275 if (auto E2 = setupJITDylib(PlatformJD)) {
276 Err = std::move(E2);
277 return;
278 }
279
280 RegisteredInitSymbols[&PlatformJD].add(
281 Name: DSOHandleSymbol, Flags: SymbolLookupFlags::WeaklyReferencedSymbol);
282
283 // Associate wrapper function tags with JIT-side function implementations.
284 if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) {
285 Err = std::move(E2);
286 return;
287 }
288
289 // Lookup addresses of runtime functions callable by the platform,
290 // call the platform bootstrap function to initialize the platform-state
291 // object in the executor.
292 if (auto E2 = bootstrapELFNixRuntime(PlatformJD)) {
293 Err = std::move(E2);
294 return;
295 }
296}
297
298Error ELFNixPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) {
299 ExecutionSession::JITDispatchHandlerAssociationMap WFs;
300
301 using GetInitializersSPSSig =
302 SPSExpected<SPSELFNixJITDylibInitializerSequence>(SPSString);
303 WFs[ES.intern(SymName: "__orc_rt_elfnix_get_initializers_tag")] =
304 ES.wrapAsyncWithSPS<GetInitializersSPSSig>(
305 Instance: this, Method: &ELFNixPlatform::rt_getInitializers);
306
307 using GetDeinitializersSPSSig =
308 SPSExpected<SPSELFJITDylibDeinitializerSequence>(SPSExecutorAddr);
309 WFs[ES.intern(SymName: "__orc_rt_elfnix_get_deinitializers_tag")] =
310 ES.wrapAsyncWithSPS<GetDeinitializersSPSSig>(
311 Instance: this, Method: &ELFNixPlatform::rt_getDeinitializers);
312
313 using LookupSymbolSPSSig =
314 SPSExpected<SPSExecutorAddr>(SPSExecutorAddr, SPSString);
315 WFs[ES.intern(SymName: "__orc_rt_elfnix_symbol_lookup_tag")] =
316 ES.wrapAsyncWithSPS<LookupSymbolSPSSig>(Instance: this,
317 Method: &ELFNixPlatform::rt_lookupSymbol);
318
319 return ES.registerJITDispatchHandlers(JD&: PlatformJD, WFs: std::move(WFs));
320}
321
322void ELFNixPlatform::getInitializersBuildSequencePhase(
323 SendInitializerSequenceFn SendResult, JITDylib &JD,
324 std::vector<JITDylibSP> DFSLinkOrder) {
325 ELFNixJITDylibInitializerSequence FullInitSeq;
326 {
327 std::lock_guard<std::mutex> Lock(PlatformMutex);
328 for (auto &InitJD : reverse(C&: DFSLinkOrder)) {
329 LLVM_DEBUG({
330 dbgs() << "ELFNixPlatform: Appending inits for \"" << InitJD->getName()
331 << "\" to sequence\n";
332 });
333 auto ISItr = InitSeqs.find(Val: InitJD.get());
334 if (ISItr != InitSeqs.end()) {
335 FullInitSeq.emplace_back(args: std::move(ISItr->second));
336 InitSeqs.erase(I: ISItr);
337 }
338 }
339 }
340
341 SendResult(std::move(FullInitSeq));
342}
343
344void ELFNixPlatform::getInitializersLookupPhase(
345 SendInitializerSequenceFn SendResult, JITDylib &JD) {
346
347 auto DFSLinkOrder = JD.getDFSLinkOrder();
348 if (!DFSLinkOrder) {
349 SendResult(DFSLinkOrder.takeError());
350 return;
351 }
352
353 DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols;
354 ES.runSessionLocked(F: [&]() {
355 for (auto &InitJD : *DFSLinkOrder) {
356 auto RISItr = RegisteredInitSymbols.find(Val: InitJD.get());
357 if (RISItr != RegisteredInitSymbols.end()) {
358 NewInitSymbols[InitJD.get()] = std::move(RISItr->second);
359 RegisteredInitSymbols.erase(I: RISItr);
360 }
361 }
362 });
363
364 // If there are no further init symbols to look up then move on to the next
365 // phase.
366 if (NewInitSymbols.empty()) {
367 getInitializersBuildSequencePhase(SendResult: std::move(SendResult), JD,
368 DFSLinkOrder: std::move(*DFSLinkOrder));
369 return;
370 }
371
372 // Otherwise issue a lookup and re-run this phase when it completes.
373 lookupInitSymbolsAsync(
374 OnComplete: [this, SendResult = std::move(SendResult), &JD](Error Err) mutable {
375 if (Err)
376 SendResult(std::move(Err));
377 else
378 getInitializersLookupPhase(SendResult: std::move(SendResult), JD);
379 },
380 ES, InitSyms: std::move(NewInitSymbols));
381}
382
383void ELFNixPlatform::rt_getInitializers(SendInitializerSequenceFn SendResult,
384 StringRef JDName) {
385 LLVM_DEBUG({
386 dbgs() << "ELFNixPlatform::rt_getInitializers(\"" << JDName << "\")\n";
387 });
388
389 JITDylib *JD = ES.getJITDylibByName(Name: JDName);
390 if (!JD) {
391 LLVM_DEBUG({
392 dbgs() << " No such JITDylib \"" << JDName << "\". Sending error.\n";
393 });
394 SendResult(make_error<StringError>(Args: "No JITDylib named " + JDName,
395 Args: inconvertibleErrorCode()));
396 return;
397 }
398
399 getInitializersLookupPhase(SendResult: std::move(SendResult), JD&: *JD);
400}
401
402void ELFNixPlatform::rt_getDeinitializers(
403 SendDeinitializerSequenceFn SendResult, ExecutorAddr Handle) {
404 LLVM_DEBUG({
405 dbgs() << "ELFNixPlatform::rt_getDeinitializers(\"" << Handle << "\")\n";
406 });
407
408 JITDylib *JD = nullptr;
409
410 {
411 std::lock_guard<std::mutex> Lock(PlatformMutex);
412 auto I = HandleAddrToJITDylib.find(Val: Handle);
413 if (I != HandleAddrToJITDylib.end())
414 JD = I->second;
415 }
416
417 if (!JD) {
418 LLVM_DEBUG(dbgs() << " No JITDylib for handle " << Handle << "\n");
419 SendResult(make_error<StringError>(Args: "No JITDylib associated with handle " +
420 formatv(Fmt: "{0:x}", Vals&: Handle),
421 Args: inconvertibleErrorCode()));
422 return;
423 }
424
425 SendResult(ELFNixJITDylibDeinitializerSequence());
426}
427
428void ELFNixPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult,
429 ExecutorAddr Handle,
430 StringRef SymbolName) {
431 LLVM_DEBUG({
432 dbgs() << "ELFNixPlatform::rt_lookupSymbol(\"" << Handle << "\")\n";
433 });
434
435 JITDylib *JD = nullptr;
436
437 {
438 std::lock_guard<std::mutex> Lock(PlatformMutex);
439 auto I = HandleAddrToJITDylib.find(Val: Handle);
440 if (I != HandleAddrToJITDylib.end())
441 JD = I->second;
442 }
443
444 if (!JD) {
445 LLVM_DEBUG(dbgs() << " No JITDylib for handle " << Handle << "\n");
446 SendResult(make_error<StringError>(Args: "No JITDylib associated with handle " +
447 formatv(Fmt: "{0:x}", Vals&: Handle),
448 Args: inconvertibleErrorCode()));
449 return;
450 }
451
452 // Use functor class to work around XL build compiler issue on AIX.
453 class RtLookupNotifyComplete {
454 public:
455 RtLookupNotifyComplete(SendSymbolAddressFn &&SendResult)
456 : SendResult(std::move(SendResult)) {}
457 void operator()(Expected<SymbolMap> Result) {
458 if (Result) {
459 assert(Result->size() == 1 && "Unexpected result map count");
460 SendResult(Result->begin()->second.getAddress());
461 } else {
462 SendResult(Result.takeError());
463 }
464 }
465
466 private:
467 SendSymbolAddressFn SendResult;
468 };
469
470 ES.lookup(
471 K: LookupKind::DLSym, SearchOrder: {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}},
472 Symbols: SymbolLookupSet(ES.intern(SymName: SymbolName)), RequiredState: SymbolState::Ready,
473 NotifyComplete: RtLookupNotifyComplete(std::move(SendResult)), RegisterDependencies: NoDependenciesToRegister);
474}
475
476Error ELFNixPlatform::bootstrapELFNixRuntime(JITDylib &PlatformJD) {
477
478 std::pair<const char *, ExecutorAddr *> Symbols[] = {
479 {"__orc_rt_elfnix_platform_bootstrap", &orc_rt_elfnix_platform_bootstrap},
480 {"__orc_rt_elfnix_platform_shutdown", &orc_rt_elfnix_platform_shutdown},
481 {"__orc_rt_elfnix_register_object_sections",
482 &orc_rt_elfnix_register_object_sections},
483 {"__orc_rt_elfnix_create_pthread_key",
484 &orc_rt_elfnix_create_pthread_key}};
485
486 SymbolLookupSet RuntimeSymbols;
487 std::vector<std::pair<SymbolStringPtr, ExecutorAddr *>> AddrsToRecord;
488 for (const auto &KV : Symbols) {
489 auto Name = ES.intern(SymName: KV.first);
490 RuntimeSymbols.add(Name);
491 AddrsToRecord.push_back(x: {std::move(Name), KV.second});
492 }
493
494 auto RuntimeSymbolAddrs = ES.lookup(
495 SearchOrder: {{&PlatformJD, JITDylibLookupFlags::MatchAllSymbols}}, Symbols: RuntimeSymbols);
496 if (!RuntimeSymbolAddrs)
497 return RuntimeSymbolAddrs.takeError();
498
499 for (const auto &KV : AddrsToRecord) {
500 auto &Name = KV.first;
501 assert(RuntimeSymbolAddrs->count(Name) && "Missing runtime symbol?");
502 *KV.second = (*RuntimeSymbolAddrs)[Name].getAddress();
503 }
504
505 auto PJDDSOHandle = ES.lookup(
506 SearchOrder: {{&PlatformJD, JITDylibLookupFlags::MatchAllSymbols}}, Symbol: DSOHandleSymbol);
507 if (!PJDDSOHandle)
508 return PJDDSOHandle.takeError();
509
510 if (auto Err = ES.callSPSWrapper<void(uint64_t)>(
511 WrapperFnAddr: orc_rt_elfnix_platform_bootstrap,
512 WrapperCallArgs: PJDDSOHandle->getAddress().getValue()))
513 return Err;
514
515 // FIXME: Ordering is fuzzy here. We're probably best off saying
516 // "behavior is undefined if code that uses the runtime is added before
517 // the platform constructor returns", then move all this to the constructor.
518 RuntimeBootstrapped = true;
519 std::vector<ELFPerObjectSectionsToRegister> DeferredPOSRs;
520 {
521 std::lock_guard<std::mutex> Lock(PlatformMutex);
522 DeferredPOSRs = std::move(BootstrapPOSRs);
523 }
524
525 for (auto &D : DeferredPOSRs)
526 if (auto Err = registerPerObjectSections(POSR: D))
527 return Err;
528
529 return Error::success();
530}
531
532Error ELFNixPlatform::registerInitInfo(
533 JITDylib &JD, ArrayRef<jitlink::Section *> InitSections) {
534
535 std::unique_lock<std::mutex> Lock(PlatformMutex);
536
537 ELFNixJITDylibInitializers *InitSeq = nullptr;
538 {
539 auto I = InitSeqs.find(Val: &JD);
540 if (I == InitSeqs.end()) {
541 // If there's no init sequence entry yet then we need to look up the
542 // header symbol to force creation of one.
543 Lock.unlock();
544
545 auto SearchOrder =
546 JD.withLinkOrderDo(F: [](const JITDylibSearchOrder &SO) { return SO; });
547 if (auto Err = ES.lookup(SearchOrder, Symbol: DSOHandleSymbol).takeError())
548 return Err;
549
550 Lock.lock();
551 I = InitSeqs.find(Val: &JD);
552 assert(I != InitSeqs.end() &&
553 "Entry missing after header symbol lookup?");
554 }
555 InitSeq = &I->second;
556 }
557
558 for (auto *Sec : InitSections) {
559 // FIXME: Avoid copy here.
560 jitlink::SectionRange R(*Sec);
561 InitSeq->InitSections[Sec->getName()].push_back(x: R.getRange());
562 }
563
564 return Error::success();
565}
566
567Error ELFNixPlatform::registerPerObjectSections(
568 const ELFPerObjectSectionsToRegister &POSR) {
569
570 if (!orc_rt_elfnix_register_object_sections)
571 return make_error<StringError>(Args: "Attempting to register per-object "
572 "sections, but runtime support has not "
573 "been loaded yet",
574 Args: inconvertibleErrorCode());
575
576 Error ErrResult = Error::success();
577 if (auto Err = ES.callSPSWrapper<shared::SPSError(
578 SPSELFPerObjectSectionsToRegister)>(
579 WrapperFnAddr: orc_rt_elfnix_register_object_sections, WrapperCallArgs&: ErrResult, WrapperCallArgs: POSR))
580 return Err;
581 return ErrResult;
582}
583
584Expected<uint64_t> ELFNixPlatform::createPThreadKey() {
585 if (!orc_rt_elfnix_create_pthread_key)
586 return make_error<StringError>(
587 Args: "Attempting to create pthread key in target, but runtime support has "
588 "not been loaded yet",
589 Args: inconvertibleErrorCode());
590
591 Expected<uint64_t> Result(0);
592 if (auto Err = ES.callSPSWrapper<SPSExpected<uint64_t>(void)>(
593 WrapperFnAddr: orc_rt_elfnix_create_pthread_key, WrapperCallArgs&: Result))
594 return std::move(Err);
595 return Result;
596}
597
598void ELFNixPlatform::ELFNixPlatformPlugin::modifyPassConfig(
599 MaterializationResponsibility &MR, jitlink::LinkGraph &LG,
600 jitlink::PassConfiguration &Config) {
601
602 // If the initializer symbol is the __dso_handle symbol then just add
603 // the DSO handle support passes.
604 if (MR.getInitializerSymbol() == MP.DSOHandleSymbol) {
605 addDSOHandleSupportPasses(MR, Config);
606 // The DSOHandle materialization unit doesn't require any other
607 // support, so we can bail out early.
608 return;
609 }
610
611 // If the object contains initializers then add passes to record them.
612 if (MR.getInitializerSymbol())
613 addInitializerSupportPasses(MR, Config);
614
615 // Add passes for eh-frame and TLV support.
616 addEHAndTLVSupportPasses(MR, Config);
617}
618
619ObjectLinkingLayer::Plugin::SyntheticSymbolDependenciesMap
620ELFNixPlatform::ELFNixPlatformPlugin::getSyntheticSymbolDependencies(
621 MaterializationResponsibility &MR) {
622 std::lock_guard<std::mutex> Lock(PluginMutex);
623 auto I = InitSymbolDeps.find(Val: &MR);
624 if (I != InitSymbolDeps.end()) {
625 SyntheticSymbolDependenciesMap Result;
626 Result[MR.getInitializerSymbol()] = std::move(I->second);
627 InitSymbolDeps.erase(Val: &MR);
628 return Result;
629 }
630 return SyntheticSymbolDependenciesMap();
631}
632
633void ELFNixPlatform::ELFNixPlatformPlugin::addInitializerSupportPasses(
634 MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) {
635
636 /// Preserve init sections.
637 Config.PrePrunePasses.push_back(x: [this, &MR](jitlink::LinkGraph &G) -> Error {
638 if (auto Err = preserveInitSections(G, MR))
639 return Err;
640 return Error::success();
641 });
642
643 Config.PostFixupPasses.push_back(
644 x: [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
645 return registerInitSections(G, JD);
646 });
647}
648
649void ELFNixPlatform::ELFNixPlatformPlugin::addDSOHandleSupportPasses(
650 MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) {
651
652 Config.PostAllocationPasses.push_back(x: [this, &JD = MR.getTargetJITDylib()](
653 jitlink::LinkGraph &G) -> Error {
654 auto I = llvm::find_if(Range: G.defined_symbols(), P: [this](jitlink::Symbol *Sym) {
655 return Sym->getName() == *MP.DSOHandleSymbol;
656 });
657 assert(I != G.defined_symbols().end() && "Missing DSO handle symbol");
658 {
659 std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
660 auto HandleAddr = (*I)->getAddress();
661 MP.HandleAddrToJITDylib[HandleAddr] = &JD;
662 assert(!MP.InitSeqs.count(&JD) && "InitSeq entry for JD already exists");
663 MP.InitSeqs.insert(KV: std::make_pair(
664 x: &JD, y: ELFNixJITDylibInitializers(JD.getName(), HandleAddr)));
665 }
666 return Error::success();
667 });
668}
669
670void ELFNixPlatform::ELFNixPlatformPlugin::addEHAndTLVSupportPasses(
671 MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) {
672
673 // Insert TLV lowering at the start of the PostPrunePasses, since we want
674 // it to run before GOT/PLT lowering.
675
676 // TODO: Check that before the fixTLVSectionsAndEdges pass, the GOT/PLT build
677 // pass has done. Because the TLS descriptor need to be allocate in GOT.
678 Config.PostPrunePasses.push_back(
679 x: [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
680 return fixTLVSectionsAndEdges(G, JD);
681 });
682
683 // Add a pass to register the final addresses of the eh-frame and TLV sections
684 // with the runtime.
685 Config.PostFixupPasses.push_back(x: [this](jitlink::LinkGraph &G) -> Error {
686 ELFPerObjectSectionsToRegister POSR;
687
688 if (auto *EHFrameSection = G.findSectionByName(Name: ELFEHFrameSectionName)) {
689 jitlink::SectionRange R(*EHFrameSection);
690 if (!R.empty())
691 POSR.EHFrameSection = R.getRange();
692 }
693
694 // Get a pointer to the thread data section if there is one. It will be used
695 // below.
696 jitlink::Section *ThreadDataSection =
697 G.findSectionByName(Name: ELFThreadDataSectionName);
698
699 // Handle thread BSS section if there is one.
700 if (auto *ThreadBSSSection = G.findSectionByName(Name: ELFThreadBSSSectionName)) {
701 // If there's already a thread data section in this graph then merge the
702 // thread BSS section content into it, otherwise just treat the thread
703 // BSS section as the thread data section.
704 if (ThreadDataSection)
705 G.mergeSections(DstSection&: *ThreadDataSection, SrcSection&: *ThreadBSSSection);
706 else
707 ThreadDataSection = ThreadBSSSection;
708 }
709
710 // Having merged thread BSS (if present) and thread data (if present),
711 // record the resulting section range.
712 if (ThreadDataSection) {
713 jitlink::SectionRange R(*ThreadDataSection);
714 if (!R.empty())
715 POSR.ThreadDataSection = R.getRange();
716 }
717
718 if (POSR.EHFrameSection.Start || POSR.ThreadDataSection.Start) {
719
720 // If we're still bootstrapping the runtime then just record this
721 // frame for now.
722 if (!MP.RuntimeBootstrapped) {
723 std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
724 MP.BootstrapPOSRs.push_back(x: POSR);
725 return Error::success();
726 }
727
728 // Otherwise register it immediately.
729 if (auto Err = MP.registerPerObjectSections(POSR))
730 return Err;
731 }
732
733 return Error::success();
734 });
735}
736
737Error ELFNixPlatform::ELFNixPlatformPlugin::preserveInitSections(
738 jitlink::LinkGraph &G, MaterializationResponsibility &MR) {
739
740 JITLinkSymbolSet InitSectionSymbols;
741 for (auto &InitSection : G.sections()) {
742 // Skip non-init sections.
743 if (!isELFInitializerSection(SecName: InitSection.getName()))
744 continue;
745
746 // Make a pass over live symbols in the section: those blocks are already
747 // preserved.
748 DenseSet<jitlink::Block *> AlreadyLiveBlocks;
749 for (auto &Sym : InitSection.symbols()) {
750 auto &B = Sym->getBlock();
751 if (Sym->isLive() && Sym->getOffset() == 0 &&
752 Sym->getSize() == B.getSize() && !AlreadyLiveBlocks.count(V: &B)) {
753 InitSectionSymbols.insert(V: Sym);
754 AlreadyLiveBlocks.insert(V: &B);
755 }
756 }
757
758 // Add anonymous symbols to preserve any not-already-preserved blocks.
759 for (auto *B : InitSection.blocks())
760 if (!AlreadyLiveBlocks.count(V: B))
761 InitSectionSymbols.insert(
762 V: &G.addAnonymousSymbol(Content&: *B, Offset: 0, Size: B->getSize(), IsCallable: false, IsLive: true));
763 }
764
765 if (!InitSectionSymbols.empty()) {
766 std::lock_guard<std::mutex> Lock(PluginMutex);
767 InitSymbolDeps[&MR] = std::move(InitSectionSymbols);
768 }
769
770 return Error::success();
771}
772
773Error ELFNixPlatform::ELFNixPlatformPlugin::registerInitSections(
774 jitlink::LinkGraph &G, JITDylib &JD) {
775
776 SmallVector<jitlink::Section *> InitSections;
777
778 LLVM_DEBUG(dbgs() << "ELFNixPlatform::registerInitSections\n");
779
780 for (auto &Sec : G.sections()) {
781 if (isELFInitializerSection(SecName: Sec.getName())) {
782 InitSections.push_back(Elt: &Sec);
783 }
784 }
785
786 // Dump the scraped inits.
787 LLVM_DEBUG({
788 dbgs() << "ELFNixPlatform: Scraped " << G.getName() << " init sections:\n";
789 for (auto *Sec : InitSections) {
790 jitlink::SectionRange R(*Sec);
791 dbgs() << " " << Sec->getName() << ": " << R.getRange() << "\n";
792 }
793 });
794
795 return MP.registerInitInfo(JD, InitSections);
796}
797
798Error ELFNixPlatform::ELFNixPlatformPlugin::fixTLVSectionsAndEdges(
799 jitlink::LinkGraph &G, JITDylib &JD) {
800
801 for (auto *Sym : G.external_symbols()) {
802 if (Sym->getName() == "__tls_get_addr") {
803 Sym->setName("___orc_rt_elfnix_tls_get_addr");
804 } else if (Sym->getName() == "__tlsdesc_resolver") {
805 Sym->setName("___orc_rt_elfnix_tlsdesc_resolver");
806 }
807 }
808
809 auto *TLSInfoEntrySection = G.findSectionByName(Name: "$__TLSINFO");
810
811 if (TLSInfoEntrySection) {
812 std::optional<uint64_t> Key;
813 {
814 std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
815 auto I = MP.JITDylibToPThreadKey.find(Val: &JD);
816 if (I != MP.JITDylibToPThreadKey.end())
817 Key = I->second;
818 }
819 if (!Key) {
820 if (auto KeyOrErr = MP.createPThreadKey())
821 Key = *KeyOrErr;
822 else
823 return KeyOrErr.takeError();
824 }
825
826 uint64_t PlatformKeyBits =
827 support::endian::byte_swap(value: *Key, endian: G.getEndianness());
828
829 for (auto *B : TLSInfoEntrySection->blocks()) {
830 // FIXME: The TLS descriptor byte length may different with different
831 // ISA
832 assert(B->getSize() == (G.getPointerSize() * 2) &&
833 "TLS descriptor must be 2 words length");
834 auto TLSInfoEntryContent = B->getMutableContent(G);
835 memcpy(dest: TLSInfoEntryContent.data(), src: &PlatformKeyBits, n: G.getPointerSize());
836 }
837 }
838
839 return Error::success();
840}
841
842} // End namespace orc.
843} // End namespace llvm.
844