1//===- elfnix_platform.cpp ------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains code required to load the rest of the ELF-on-*IX runtime.
10//
11//===----------------------------------------------------------------------===//
12
13#include "elfnix_platform.h"
14#include "common.h"
15#include "compiler.h"
16#include "error.h"
17#include "jit_dispatch.h"
18#include "record_section_tracker.h"
19#include "wrapper_function_utils.h"
20
21#include <algorithm>
22#include <map>
23#include <mutex>
24#include <sstream>
25#include <string_view>
26#include <unordered_map>
27#include <vector>
28
29using namespace orc_rt;
30using namespace orc_rt::elfnix;
31
32// Declare function tags for functions in the JIT process.
33ORC_RT_JIT_DISPATCH_TAG(__orc_rt_reoptimize_tag)
34ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_push_initializers_tag)
35ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_symbol_lookup_tag)
36
37// eh-frame registration functions, made available via aliases
38// installed by the Platform
39extern "C" void __register_frame(const void *);
40extern "C" void __deregister_frame(const void *);
41
42extern "C" void
43__unw_add_dynamic_eh_frame_section(const void *) ORC_RT_WEAK_IMPORT;
44extern "C" void
45__unw_remove_dynamic_eh_frame_section(const void *) ORC_RT_WEAK_IMPORT;
46
47namespace {
48
49struct TLSInfoEntry {
50 unsigned long Key = 0;
51 unsigned long DataAddress = 0;
52};
53
54struct TLSDescriptor {
55 void (*Resolver)(void *);
56 TLSInfoEntry *InfoEntry;
57};
58
59class ELFNixPlatformRuntimeState {
60private:
61 struct AtExitEntry {
62 void (*Func)(void *);
63 void *Arg;
64 };
65
66 using AtExitsVector = std::vector<AtExitEntry>;
67
68 struct PerJITDylibState {
69 std::string Name;
70 void *Header = nullptr;
71 size_t RefCount = 0;
72 size_t LinkedAgainstRefCount = 0;
73 bool AllowReinitialization = false;
74 AtExitsVector AtExits;
75 std::vector<PerJITDylibState *> Deps;
76 RecordSectionsTracker<void (*)()> RecordedInits;
77
78 bool referenced() const {
79 return LinkedAgainstRefCount != 0 || RefCount != 0;
80 }
81 };
82
83public:
84 static void initialize(void *DSOHandle);
85 static ELFNixPlatformRuntimeState &get();
86 static void destroy();
87
88 ELFNixPlatformRuntimeState(void *DSOHandle);
89
90 // Delete copy and move constructors.
91 ELFNixPlatformRuntimeState(const ELFNixPlatformRuntimeState &) = delete;
92 ELFNixPlatformRuntimeState &
93 operator=(const ELFNixPlatformRuntimeState &) = delete;
94 ELFNixPlatformRuntimeState(ELFNixPlatformRuntimeState &&) = delete;
95 ELFNixPlatformRuntimeState &operator=(ELFNixPlatformRuntimeState &&) = delete;
96
97 Error registerObjectSections(ELFNixPerObjectSectionsToRegister POSR);
98 Error registerJITDylib(std::string &Name, void *Handle);
99 Error deregisterJITDylib(void *Handle);
100 Error registerInits(ExecutorAddr HeaderAddr,
101 std::vector<ExecutorAddrRange> Inits);
102 Error deregisterInits(ExecutorAddr HeaderAddr,
103 std::vector<ExecutorAddrRange> Inits);
104 Error deregisterObjectSections(ELFNixPerObjectSectionsToRegister POSR);
105
106 const char *dlerror();
107 void *dlopen(std::string_view Name, int Mode);
108 int dlupdate(void *DSOHandle);
109 int dlclose(void *DSOHandle);
110 void *dlsym(void *DSOHandle, std::string_view Symbol);
111
112 int registerAtExit(void (*F)(void *), void *Arg, void *DSOHandle);
113 void runAtExits(void *DSOHandle);
114 void runAtExits(std::unique_lock<std::recursive_mutex> &JDStateLock,
115 PerJITDylibState &JDS);
116
117 /// Returns the base address of the section containing ThreadData.
118 Expected<std::pair<const char *, size_t>>
119 getThreadDataSectionFor(const char *ThreadData);
120
121 void *getPlatformJDDSOHandle() { return PlatformJDDSOHandle; }
122
123private:
124 PerJITDylibState *getJITDylibStateByHeaderAddr(void *DSOHandle);
125 PerJITDylibState *getJITDylibStateByName(std::string_view Path);
126
127 Error registerThreadDataSection(span<const char> ThreadDataSection);
128
129 Expected<ExecutorAddr> lookupSymbolInJITDylib(void *DSOHandle,
130 std::string_view Symbol);
131
132 Error runInits(std::unique_lock<std::recursive_mutex> &JDStatesLock,
133 PerJITDylibState &JDS);
134 Expected<void *> dlopenImpl(std::string_view Path, int Mode);
135 Error dlopenFull(std::unique_lock<std::recursive_mutex> &JDStatesLock,
136 PerJITDylibState &JDS);
137 Error dlopenInitialize(std::unique_lock<std::recursive_mutex> &JDStatesLock,
138 PerJITDylibState &JDS,
139 ELFNixJITDylibDepInfoMap &DepInfo);
140 Error dlupdateImpl(void *DSOHandle);
141 Error dlupdateFull(std::unique_lock<std::recursive_mutex> &JDStatesLock,
142 PerJITDylibState &JDS);
143
144 Error dlcloseImpl(void *DSOHandle);
145 Error dlcloseInitialize(std::unique_lock<std::recursive_mutex> &JDStatesLock,
146 PerJITDylibState &JDS);
147
148 static ELFNixPlatformRuntimeState *MOPS;
149
150 void *PlatformJDDSOHandle;
151
152 // Frame registration functions:
153 void (*registerEHFrameSection)(const void *) = nullptr;
154 void (*deregisterEHFrameSection)(const void *) = nullptr;
155
156 // FIXME: Move to thread-state.
157 std::string DLFcnError;
158
159 std::recursive_mutex JDStatesMutex;
160 std::unordered_map<void *, PerJITDylibState> JDStates;
161 std::unordered_map<std::string, void *> JDNameToHeader;
162
163 std::mutex ThreadDataSectionsMutex;
164 std::map<const char *, size_t> ThreadDataSections;
165};
166
167ELFNixPlatformRuntimeState *ELFNixPlatformRuntimeState::MOPS = nullptr;
168
169void ELFNixPlatformRuntimeState::initialize(void *DSOHandle) {
170 assert(!MOPS && "ELFNixPlatformRuntimeState should be null");
171 MOPS = new ELFNixPlatformRuntimeState(DSOHandle);
172}
173
174ELFNixPlatformRuntimeState &ELFNixPlatformRuntimeState::get() {
175 assert(MOPS && "ELFNixPlatformRuntimeState not initialized");
176 return *MOPS;
177}
178
179void ELFNixPlatformRuntimeState::destroy() {
180 assert(MOPS && "ELFNixPlatformRuntimeState not initialized");
181 delete MOPS;
182}
183
184ELFNixPlatformRuntimeState::ELFNixPlatformRuntimeState(void *DSOHandle)
185 : PlatformJDDSOHandle(DSOHandle) {
186 if (__unw_add_dynamic_eh_frame_section &&
187 __unw_remove_dynamic_eh_frame_section) {
188 registerEHFrameSection = __unw_add_dynamic_eh_frame_section;
189 deregisterEHFrameSection = __unw_remove_dynamic_eh_frame_section;
190 } else {
191 registerEHFrameSection = __register_frame;
192 deregisterEHFrameSection = __deregister_frame;
193 }
194}
195
196Error ELFNixPlatformRuntimeState::registerObjectSections(
197 ELFNixPerObjectSectionsToRegister POSR) {
198 if (POSR.EHFrameSection.Start)
199 registerEHFrameSection(POSR.EHFrameSection.Start.toPtr<const char *>());
200
201 if (POSR.ThreadDataSection.Start) {
202 if (auto Err = registerThreadDataSection(
203 ThreadDataSection: POSR.ThreadDataSection.toSpan<const char>()))
204 return Err;
205 }
206
207 return Error::success();
208}
209
210Error ELFNixPlatformRuntimeState::deregisterObjectSections(
211 ELFNixPerObjectSectionsToRegister POSR) {
212 if (POSR.EHFrameSection.Start)
213 deregisterEHFrameSection(POSR.EHFrameSection.Start.toPtr<const char *>());
214
215 return Error::success();
216}
217
218Error ELFNixPlatformRuntimeState::registerJITDylib(std::string &Name,
219 void *Handle) {
220 std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
221
222 if (JDStates.count(x: Handle)) {
223 std::ostringstream ErrStream;
224 ErrStream << "Duplicate JITDylib registration for header " << Handle
225 << " (name = " << Name << ")";
226 return make_error<StringError>(Args: ErrStream.str());
227 }
228
229 if (JDNameToHeader.count(x: Name)) {
230 std::ostringstream ErrStream;
231 ErrStream << "Duplicate JITDylib registration for header " << Handle
232 << " (header = " << Handle << ")";
233 return make_error<StringError>(Args: ErrStream.str());
234 }
235
236 auto &JD = JDStates[Handle];
237 JD.Header = Handle;
238 JD.Name = std::move(t&: Name);
239 JDNameToHeader[JD.Name] = Handle;
240 return Error::success();
241}
242
243Error ELFNixPlatformRuntimeState::deregisterJITDylib(void *Handle) {
244 std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
245
246 auto I = JDStates.find(x: Handle);
247 if (I == JDStates.end()) {
248 std::ostringstream ErrStream;
249 ErrStream << "Attempted to deregister unrecognized header " << Handle;
250 return make_error<StringError>(Args: ErrStream.str());
251 }
252
253 auto J = JDNameToHeader.find(
254 x: std::string(I->second.Name.data(), I->second.Name.size()));
255 assert(J != JDNameToHeader.end() &&
256 "Missing JDNameToHeader entry for JITDylib");
257 JDNameToHeader.erase(position: J);
258 JDStates.erase(position: I);
259 return Error::success();
260}
261
262Error ELFNixPlatformRuntimeState::registerInits(
263 ExecutorAddr HeaderAddr, std::vector<ExecutorAddrRange> Inits) {
264 std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
265 PerJITDylibState *JDS =
266 getJITDylibStateByHeaderAddr(DSOHandle: HeaderAddr.toPtr<void *>());
267
268 if (!JDS) {
269 std::ostringstream ErrStream;
270 ErrStream << "Could not register object platform sections for "
271 "unrecognized header "
272 << HeaderAddr.toPtr<void *>();
273 return make_error<StringError>(Args: ErrStream.str());
274 }
275
276 for (auto &I : Inits) {
277 JDS->RecordedInits.add(Sec: I.toSpan<void (*)()>());
278 }
279
280 return Error::success();
281}
282
283Error ELFNixPlatformRuntimeState::deregisterInits(
284 ExecutorAddr HeaderAddr, std::vector<ExecutorAddrRange> Inits) {
285
286 std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
287 PerJITDylibState *JDS =
288 getJITDylibStateByHeaderAddr(DSOHandle: HeaderAddr.toPtr<void *>());
289
290 if (!JDS) {
291 std::ostringstream ErrStream;
292 ErrStream << "Could not register object platform sections for unrecognized "
293 "header "
294 << HeaderAddr.toPtr<void *>();
295 return make_error<StringError>(Args: ErrStream.str());
296 }
297
298 for (auto &I : Inits) {
299 JDS->RecordedInits.removeIfPresent(R: I);
300 }
301
302 return Error::success();
303}
304
305const char *ELFNixPlatformRuntimeState::dlerror() { return DLFcnError.c_str(); }
306
307void *ELFNixPlatformRuntimeState::dlopen(std::string_view Path, int Mode) {
308 if (auto H = dlopenImpl(Path, Mode))
309 return *H;
310 else {
311 // FIXME: Make dlerror thread safe.
312 DLFcnError = toString(Err: H.takeError());
313 return nullptr;
314 }
315}
316
317int ELFNixPlatformRuntimeState::dlupdate(void *DSOHandle) {
318 if (auto Err = dlupdateImpl(DSOHandle)) {
319 // FIXME: Make dlerror thread safe.
320 DLFcnError = toString(Err: std::move(t&: Err));
321 return -1;
322 }
323 return 0;
324}
325
326int ELFNixPlatformRuntimeState::dlclose(void *DSOHandle) {
327 if (auto Err = dlcloseImpl(DSOHandle)) {
328 DLFcnError = toString(Err: std::move(t&: Err));
329 return -1;
330 }
331 return 0;
332}
333
334void *ELFNixPlatformRuntimeState::dlsym(void *DSOHandle,
335 std::string_view Symbol) {
336 auto Addr = lookupSymbolInJITDylib(DSOHandle, Symbol);
337 if (!Addr) {
338 DLFcnError = toString(Err: Addr.takeError());
339 return 0;
340 }
341
342 return Addr->toPtr<void *>();
343}
344
345int ELFNixPlatformRuntimeState::registerAtExit(void (*F)(void *), void *Arg,
346 void *DSOHandle) {
347 // FIXME: Handle out-of-memory errors, returning -1 if OOM.
348 std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
349 auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle);
350 assert(JDS && "JITDylib state not initialized");
351 JDS->AtExits.push_back(x: {.Func: F, .Arg: Arg});
352 return 0;
353}
354
355void ELFNixPlatformRuntimeState::runAtExits(void *DSOHandle) {
356 std::unique_lock<std::recursive_mutex> Lock(JDStatesMutex);
357 PerJITDylibState *JDS = getJITDylibStateByHeaderAddr(DSOHandle);
358
359 if (JDS)
360 runAtExits(JDStateLock&: Lock, JDS&: *JDS);
361}
362
363void ELFNixPlatformRuntimeState::runAtExits(
364 std::unique_lock<std::recursive_mutex> &JDStateLock,
365 PerJITDylibState &JDS) {
366 AtExitsVector V = std::move(t&: JDS.AtExits);
367
368 while (!V.empty()) {
369 auto &AE = V.back();
370 AE.Func(AE.Arg);
371 V.pop_back();
372 }
373}
374
375Expected<std::pair<const char *, size_t>>
376ELFNixPlatformRuntimeState::getThreadDataSectionFor(const char *ThreadData) {
377 std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
378 auto I = ThreadDataSections.upper_bound(x: ThreadData);
379 // Check that we have a valid entry conovering this address.
380 if (I == ThreadDataSections.begin())
381 return make_error<StringError>(Args: "No thread local data section for key");
382 I = std::prev(x: I);
383 if (ThreadData >= I->first + I->second)
384 return make_error<StringError>(Args: "No thread local data section for key");
385 return *I;
386}
387
388ELFNixPlatformRuntimeState::PerJITDylibState *
389ELFNixPlatformRuntimeState::getJITDylibStateByHeaderAddr(void *DSOHandle) {
390 auto I = JDStates.find(x: DSOHandle);
391 if (I == JDStates.end())
392 return nullptr;
393
394 return &I->second;
395}
396
397ELFNixPlatformRuntimeState::PerJITDylibState *
398ELFNixPlatformRuntimeState::getJITDylibStateByName(std::string_view Name) {
399 // FIXME: Avoid creating string copy here.
400 auto I = JDNameToHeader.find(x: std::string(Name.data(), Name.size()));
401 if (I == JDNameToHeader.end())
402 return nullptr;
403 void *H = I->second;
404 auto J = JDStates.find(x: H);
405 assert(J != JDStates.end() &&
406 "JITDylib has name map entry but no header map entry");
407 return &J->second;
408}
409
410Error ELFNixPlatformRuntimeState::registerThreadDataSection(
411 span<const char> ThreadDataSection) {
412 std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
413 auto I = ThreadDataSections.upper_bound(x: ThreadDataSection.data());
414 if (I != ThreadDataSections.begin()) {
415 auto J = std::prev(x: I);
416 if (J->first + J->second > ThreadDataSection.data())
417 return make_error<StringError>(Args: "Overlapping .tdata sections");
418 }
419 ThreadDataSections.insert(
420 position: I, x: std::make_pair(x: ThreadDataSection.data(), y: ThreadDataSection.size()));
421 return Error::success();
422}
423
424Expected<ExecutorAddr>
425ELFNixPlatformRuntimeState::lookupSymbolInJITDylib(void *DSOHandle,
426 std::string_view Sym) {
427 Expected<ExecutorAddr> Result((ExecutorAddr()));
428 if (auto Err = WrapperFunction<SPSExpected<SPSExecutorAddr>(
429 SPSExecutorAddr,
430 SPSString)>::call(Dispatch: JITDispatch(&__orc_rt_elfnix_symbol_lookup_tag),
431 Result, Args: ExecutorAddr::fromPtr(Ptr: DSOHandle), Args: Sym))
432 return std::move(t&: Err);
433 return Result;
434}
435
436Error ELFNixPlatformRuntimeState::runInits(
437 std::unique_lock<std::recursive_mutex> &JDStatesLock,
438 PerJITDylibState &JDS) {
439 std::vector<span<void (*)()>> InitSections;
440 InitSections.reserve(n: JDS.RecordedInits.numNewSections());
441
442 JDS.RecordedInits.processNewSections(
443 ProcessSection: [&](span<void (*)()> Inits) { InitSections.push_back(x: Inits); });
444
445 JDStatesLock.unlock();
446 for (auto Sec : InitSections)
447 for (auto *Init : Sec)
448 Init();
449
450 JDStatesLock.lock();
451
452 return Error::success();
453}
454
455Expected<void *> ELFNixPlatformRuntimeState::dlopenImpl(std::string_view Path,
456 int Mode) {
457 std::unique_lock<std::recursive_mutex> Lock(JDStatesMutex);
458 PerJITDylibState *JDS = getJITDylibStateByName(Name: Path);
459
460 if (!JDS)
461 return make_error<StringError>(Args: "No registered JTIDylib for path " +
462 std::string(Path.data(), Path.size()));
463
464 if (auto Err = dlopenFull(JDStatesLock&: Lock, JDS&: *JDS))
465 return std::move(t&: Err);
466
467 ++JDS->RefCount;
468
469 return JDS->Header;
470}
471
472Error ELFNixPlatformRuntimeState::dlopenFull(
473 std::unique_lock<std::recursive_mutex> &JDStateLock,
474 PerJITDylibState &JDS) {
475 Expected<ELFNixJITDylibDepInfoMap> DepInfo((ELFNixJITDylibDepInfoMap()));
476 JDStateLock.unlock();
477 if (auto Err = WrapperFunction<SPSExpected<SPSELFNixJITDylibDepInfoMap>(
478 SPSExecutorAddr)>::
479 call(Dispatch: JITDispatch(&__orc_rt_elfnix_push_initializers_tag), Result&: DepInfo,
480 Args: ExecutorAddr::fromPtr(Ptr: JDS.Header)))
481 return Err;
482 JDStateLock.lock();
483
484 if (!DepInfo)
485 return DepInfo.takeError();
486
487 if (auto Err = dlopenInitialize(JDStatesLock&: JDStateLock, JDS, DepInfo&: *DepInfo))
488 return Err;
489
490 if (!DepInfo->empty()) {
491 std::ostringstream ErrStream;
492 ErrStream << "Encountered unrecognized dep-info key headers "
493 "while processing dlopen of "
494 << JDS.Name;
495 return make_error<StringError>(Args: ErrStream.str());
496 }
497
498 return Error::success();
499}
500
501Error ELFNixPlatformRuntimeState::dlopenInitialize(
502 std::unique_lock<std::recursive_mutex> &JDStatesLock, PerJITDylibState &JDS,
503 ELFNixJITDylibDepInfoMap &DepInfo) {
504
505 auto I = DepInfo.find(x: ExecutorAddr::fromPtr(Ptr: JDS.Header));
506 if (I == DepInfo.end())
507 return Error::success();
508
509 auto Deps = std::move(t&: I->second);
510 DepInfo.erase(position: I);
511
512 std::vector<PerJITDylibState *> OldDeps;
513 std::swap(x&: JDS.Deps, y&: OldDeps);
514 JDS.Deps.reserve(n: Deps.size());
515 for (auto H : Deps) {
516 PerJITDylibState *DepJDS = getJITDylibStateByHeaderAddr(DSOHandle: H.toPtr<void *>());
517 if (!DepJDS) {
518 std::ostringstream ErrStream;
519 ErrStream << "Encountered unrecognized dep header " << H.toPtr<void *>()
520 << " while initializing " << JDS.Name;
521 return make_error<StringError>(Args: ErrStream.str());
522 }
523 ++DepJDS->LinkedAgainstRefCount;
524 if (auto Err = dlopenInitialize(JDStatesLock, JDS&: *DepJDS, DepInfo))
525 return Err;
526 }
527
528 if (auto Err = runInits(JDStatesLock, JDS))
529 return Err;
530
531 for (auto *DepJDS : OldDeps) {
532 --DepJDS->LinkedAgainstRefCount;
533 if (!DepJDS->referenced())
534 if (auto Err = dlcloseInitialize(JDStatesLock, JDS&: *DepJDS))
535 return Err;
536 }
537 return Error::success();
538}
539
540Error ELFNixPlatformRuntimeState::dlupdateImpl(void *DSOHandle) {
541 std::unique_lock<std::recursive_mutex> Lock(JDStatesMutex);
542
543 // Try to find JITDylib state by name.
544 auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle);
545
546 if (!JDS) {
547 std::ostringstream ErrStream;
548 ErrStream << "No registered JITDylib for " << DSOHandle;
549 return make_error<StringError>(Args: ErrStream.str());
550 }
551
552 if (!JDS->referenced())
553 return make_error<StringError>(Args: "dlupdate failed, JITDylib must be open.");
554
555 if (auto Err = dlupdateFull(JDStatesLock&: Lock, JDS&: *JDS))
556 return Err;
557
558 return Error::success();
559}
560
561Error ELFNixPlatformRuntimeState::dlupdateFull(
562 std::unique_lock<std::recursive_mutex> &JDStatesLock,
563 PerJITDylibState &JDS) {
564 // Call back to the JIT to push the initializers.
565 Expected<ELFNixJITDylibDepInfoMap> DepInfo((ELFNixJITDylibDepInfoMap()));
566 // Unlock so that we can accept the initializer update.
567 JDStatesLock.unlock();
568 if (auto Err = WrapperFunction<SPSExpected<SPSELFNixJITDylibDepInfoMap>(
569 SPSExecutorAddr)>::
570 call(Dispatch: JITDispatch(&__orc_rt_elfnix_push_initializers_tag), Result&: DepInfo,
571 Args: ExecutorAddr::fromPtr(Ptr: JDS.Header)))
572 return Err;
573 JDStatesLock.lock();
574
575 if (!DepInfo)
576 return DepInfo.takeError();
577
578 if (auto Err = runInits(JDStatesLock, JDS))
579 return Err;
580
581 return Error::success();
582}
583
584Error ELFNixPlatformRuntimeState::dlcloseImpl(void *DSOHandle) {
585
586 std::unique_lock<std::recursive_mutex> Lock(JDStatesMutex);
587 PerJITDylibState *JDS = getJITDylibStateByHeaderAddr(DSOHandle);
588
589 if (!JDS) {
590 std::ostringstream ErrStream;
591 ErrStream << "No registered JITDylib for " << DSOHandle;
592 return make_error<StringError>(Args: ErrStream.str());
593 }
594
595 --JDS->RefCount;
596
597 if (!JDS->referenced())
598 return dlcloseInitialize(JDStatesLock&: Lock, JDS&: *JDS);
599
600 return Error::success();
601}
602
603Error ELFNixPlatformRuntimeState::dlcloseInitialize(
604 std::unique_lock<std::recursive_mutex> &JDStatesLock,
605 PerJITDylibState &JDS) {
606 runAtExits(JDStateLock&: JDStatesLock, JDS);
607 JDS.RecordedInits.reset();
608 for (auto *DepJDS : JDS.Deps)
609 if (!JDS.referenced())
610 if (auto Err = dlcloseInitialize(JDStatesLock, JDS&: *DepJDS))
611 return Err;
612
613 return Error::success();
614}
615
616class ELFNixPlatformRuntimeTLVManager {
617public:
618 void *getInstance(const char *ThreadData);
619
620private:
621 std::unordered_map<const char *, char *> Instances;
622 std::unordered_map<const char *, std::unique_ptr<char[]>> AllocatedSections;
623};
624
625void *ELFNixPlatformRuntimeTLVManager::getInstance(const char *ThreadData) {
626 auto I = Instances.find(x: ThreadData);
627 if (I != Instances.end())
628 return I->second;
629 auto TDS =
630 ELFNixPlatformRuntimeState::get().getThreadDataSectionFor(ThreadData);
631 if (!TDS) {
632 __orc_rt_log_error(ErrMsg: toString(Err: TDS.takeError()).c_str());
633 return nullptr;
634 }
635
636 auto &Allocated = AllocatedSections[TDS->first];
637 if (!Allocated) {
638 Allocated = std::make_unique<char[]>(num: TDS->second);
639 memcpy(dest: Allocated.get(), src: TDS->first, n: TDS->second);
640 }
641 size_t ThreadDataDelta = ThreadData - TDS->first;
642 assert(ThreadDataDelta <= TDS->second && "ThreadData outside section bounds");
643
644 char *Instance = Allocated.get() + ThreadDataDelta;
645 Instances[ThreadData] = Instance;
646 return Instance;
647}
648
649void destroyELFNixTLVMgr(void *ELFNixTLVMgr) {
650 delete static_cast<ELFNixPlatformRuntimeTLVManager *>(ELFNixTLVMgr);
651}
652
653} // end anonymous namespace
654
655//------------------------------------------------------------------------------
656// JIT entry points
657//------------------------------------------------------------------------------
658
659ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
660__orc_rt_elfnix_platform_bootstrap(char *ArgData, size_t ArgSize) {
661 return WrapperFunction<SPSError(SPSExecutorAddr)>::handle(
662 ArgData, ArgSize,
663 Handler: [](ExecutorAddr DSOHandle) {
664 ELFNixPlatformRuntimeState::initialize(
665 DSOHandle: DSOHandle.toPtr<void *>());
666 return Error::success();
667 })
668 .release();
669}
670
671ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
672__orc_rt_elfnix_platform_shutdown(char *ArgData, size_t ArgSize) {
673 return WrapperFunction<SPSError()>::handle(
674 ArgData, ArgSize,
675 Handler: []() {
676 ELFNixPlatformRuntimeState::destroy();
677 return Error::success();
678 })
679 .release();
680}
681
682ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
683__orc_rt_elfnix_register_jitdylib(char *ArgData, size_t ArgSize) {
684 return WrapperFunction<SPSError(SPSString, SPSExecutorAddr)>::handle(
685 ArgData, ArgSize,
686 Handler: [](std::string &JDName, ExecutorAddr HeaderAddr) {
687 return ELFNixPlatformRuntimeState::get().registerJITDylib(
688 Name&: JDName, Handle: HeaderAddr.toPtr<void *>());
689 })
690 .release();
691}
692
693ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
694__orc_rt_elfnix_deregister_jitdylib(char *ArgData, size_t ArgSize) {
695 return WrapperFunction<SPSError(SPSExecutorAddr)>::handle(
696 ArgData, ArgSize,
697 Handler: [](ExecutorAddr HeaderAddr) {
698 return ELFNixPlatformRuntimeState::get().deregisterJITDylib(
699 Handle: HeaderAddr.toPtr<void *>());
700 })
701 .release();
702}
703
704ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
705__orc_rt_elfnix_register_init_sections(char *ArgData, size_t ArgSize) {
706 return WrapperFunction<SPSError(SPSExecutorAddr,
707 SPSSequence<SPSExecutorAddrRange>)>::
708 handle(ArgData, ArgSize,
709 Handler: [](ExecutorAddr HeaderAddr,
710 std::vector<ExecutorAddrRange> &Inits) {
711 return ELFNixPlatformRuntimeState::get().registerInits(
712 HeaderAddr, Inits: std::move(t&: Inits));
713 })
714 .release();
715}
716
717ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
718__orc_rt_elfnix_deregister_init_sections(char *ArgData, size_t ArgSize) {
719 return WrapperFunction<SPSError(SPSExecutorAddr,
720 SPSSequence<SPSExecutorAddrRange>)>::
721 handle(ArgData, ArgSize,
722 Handler: [](ExecutorAddr HeaderAddr,
723 std::vector<ExecutorAddrRange> &Inits) {
724 return ELFNixPlatformRuntimeState::get().deregisterInits(
725 HeaderAddr, Inits: std::move(t&: Inits));
726 })
727 .release();
728}
729
730/// Wrapper function for registering metadata on a per-object basis.
731ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
732__orc_rt_elfnix_register_object_sections(char *ArgData, size_t ArgSize) {
733 return WrapperFunction<SPSError(SPSELFNixPerObjectSectionsToRegister)>::
734 handle(ArgData, ArgSize,
735 Handler: [](ELFNixPerObjectSectionsToRegister &POSR) {
736 return ELFNixPlatformRuntimeState::get().registerObjectSections(
737 POSR: std::move(t&: POSR));
738 })
739 .release();
740}
741
742/// Wrapper for releasing per-object metadat.
743ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
744__orc_rt_elfnix_deregister_object_sections(char *ArgData, size_t ArgSize) {
745 return WrapperFunction<SPSError(SPSELFNixPerObjectSectionsToRegister)>::
746 handle(ArgData, ArgSize,
747 Handler: [](ELFNixPerObjectSectionsToRegister &POSR) {
748 return ELFNixPlatformRuntimeState::get()
749 .deregisterObjectSections(POSR: std::move(t&: POSR));
750 })
751 .release();
752}
753
754//------------------------------------------------------------------------------
755// TLV support
756//------------------------------------------------------------------------------
757
758ORC_RT_INTERFACE void *__orc_rt_elfnix_tls_get_addr_impl(TLSInfoEntry *D) {
759 auto *TLVMgr = static_cast<ELFNixPlatformRuntimeTLVManager *>(
760 pthread_getspecific(key: D->Key));
761 if (!TLVMgr)
762 TLVMgr = new ELFNixPlatformRuntimeTLVManager();
763 if (pthread_setspecific(key: D->Key, pointer: TLVMgr)) {
764 __orc_rt_log_error(ErrMsg: "Call to pthread_setspecific failed");
765 return nullptr;
766 }
767
768 return TLVMgr->getInstance(
769 ThreadData: reinterpret_cast<char *>(static_cast<uintptr_t>(D->DataAddress)));
770}
771
772ORC_RT_INTERFACE ptrdiff_t ___orc_rt_elfnix_tlsdesc_resolver_impl(
773 TLSDescriptor *D, const char *ThreadPointer) {
774 const char *TLVPtr = reinterpret_cast<const char *>(
775 __orc_rt_elfnix_tls_get_addr_impl(D: D->InfoEntry));
776 return TLVPtr - ThreadPointer;
777}
778
779ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
780__orc_rt_elfnix_create_pthread_key(char *ArgData, size_t ArgSize) {
781 return WrapperFunction<SPSExpected<uint64_t>(void)>::handle(
782 ArgData, ArgSize,
783 Handler: []() -> Expected<uint64_t> {
784 pthread_key_t Key;
785 if (int Err = pthread_key_create(key: &Key, destr_function: destroyELFNixTLVMgr)) {
786 __orc_rt_log_error(ErrMsg: "Call to pthread_key_create failed");
787 return make_error<StringError>(Args: strerror(errnum: Err));
788 }
789 return static_cast<uint64_t>(Key);
790 })
791 .release();
792}
793
794//------------------------------------------------------------------------------
795// cxa_atexit support
796//------------------------------------------------------------------------------
797
798int __orc_rt_elfnix_cxa_atexit(void (*func)(void *), void *arg,
799 void *dso_handle) {
800 return ELFNixPlatformRuntimeState::get().registerAtExit(F: func, Arg: arg,
801 DSOHandle: dso_handle);
802}
803
804int __orc_rt_elfnix_atexit(void (*func)(void *)) {
805 auto &PlatformRTState = ELFNixPlatformRuntimeState::get();
806 return ELFNixPlatformRuntimeState::get().registerAtExit(
807 F: func, NULL, DSOHandle: PlatformRTState.getPlatformJDDSOHandle());
808}
809
810void __orc_rt_elfnix_cxa_finalize(void *dso_handle) {
811 ELFNixPlatformRuntimeState::get().runAtExits(DSOHandle: dso_handle);
812}
813
814//------------------------------------------------------------------------------
815// JIT'd dlfcn alternatives.
816//------------------------------------------------------------------------------
817
818const char *__orc_rt_elfnix_jit_dlerror() {
819 return ELFNixPlatformRuntimeState::get().dlerror();
820}
821
822void *__orc_rt_elfnix_jit_dlopen(const char *path, int mode) {
823 return ELFNixPlatformRuntimeState::get().dlopen(Path: path, Mode: mode);
824}
825
826int __orc_rt_elfnix_jit_dlupdate(void *dso_handle) {
827 return ELFNixPlatformRuntimeState::get().dlupdate(DSOHandle: dso_handle);
828}
829
830int __orc_rt_elfnix_jit_dlclose(void *dso_handle) {
831 return ELFNixPlatformRuntimeState::get().dlclose(DSOHandle: dso_handle);
832}
833
834void *__orc_rt_elfnix_jit_dlsym(void *dso_handle, const char *symbol) {
835 return ELFNixPlatformRuntimeState::get().dlsym(DSOHandle: dso_handle, Symbol: symbol);
836}
837
838//------------------------------------------------------------------------------
839// ELFNix Run Program
840//------------------------------------------------------------------------------
841
842ORC_RT_INTERFACE int64_t __orc_rt_elfnix_run_program(
843 const char *JITDylibName, const char *EntrySymbolName, int argc,
844 char *argv[]) {
845 using MainTy = int (*)(int, char *[]);
846
847 void *H = __orc_rt_elfnix_jit_dlopen(path: JITDylibName,
848 mode: orc_rt::elfnix::ORC_RT_RTLD_LAZY);
849 if (!H) {
850 __orc_rt_log_error(ErrMsg: __orc_rt_elfnix_jit_dlerror());
851 return -1;
852 }
853
854 auto *Main =
855 reinterpret_cast<MainTy>(__orc_rt_elfnix_jit_dlsym(dso_handle: H, symbol: EntrySymbolName));
856
857 if (!Main) {
858 __orc_rt_log_error(ErrMsg: __orc_rt_elfnix_jit_dlerror());
859 return -1;
860 }
861
862 int Result = Main(argc, argv);
863
864 if (__orc_rt_elfnix_jit_dlclose(dso_handle: H) == -1)
865 __orc_rt_log_error(ErrMsg: __orc_rt_elfnix_jit_dlerror());
866
867 return Result;
868}
869