1//===- elfnix_platform.cpp ------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains code required to load the rest of the ELF-on-*IX runtime.
10//
11//===----------------------------------------------------------------------===//
12
13#include "elfnix_platform.h"
14#include "common.h"
15#include "compiler.h"
16#include "error.h"
17#include "jit_dispatch.h"
18#include "record_section_tracker.h"
19#include "wrapper_function_utils.h"
20
21#include <algorithm>
22#include <map>
23#include <mutex>
24#include <sstream>
25#include <string_view>
26#include <unordered_map>
27#include <vector>
28
29using namespace orc_rt;
30using namespace orc_rt::elfnix;
31
32// Declare function tags for functions in the JIT process.
33ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_push_initializers_tag)
34ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_symbol_lookup_tag)
35
36// eh-frame registration functions, made available via aliases
37// installed by the Platform
38extern "C" void __register_frame(const void *);
39extern "C" void __deregister_frame(const void *);
40
41extern "C" void
42__unw_add_dynamic_eh_frame_section(const void *) ORC_RT_WEAK_IMPORT;
43extern "C" void
44__unw_remove_dynamic_eh_frame_section(const void *) ORC_RT_WEAK_IMPORT;
45
46namespace {
47
48struct TLSInfoEntry {
49 unsigned long Key = 0;
50 unsigned long DataAddress = 0;
51};
52
53struct TLSDescriptor {
54 void (*Resolver)(void *);
55 TLSInfoEntry *InfoEntry;
56};
57
58class ELFNixPlatformRuntimeState {
59private:
60 struct AtExitEntry {
61 void (*Func)(void *);
62 void *Arg;
63 };
64
65 using AtExitsVector = std::vector<AtExitEntry>;
66
67 struct PerJITDylibState {
68 std::string Name;
69 void *Header = nullptr;
70 size_t RefCount = 0;
71 size_t LinkedAgainstRefCount = 0;
72 bool AllowReinitialization = false;
73 AtExitsVector AtExits;
74 std::vector<PerJITDylibState *> Deps;
75 RecordSectionsTracker<void (*)()> RecordedInits;
76 RecordSectionsTracker<void (*)()> RecordedFinis;
77
78 bool referenced() const {
79 return LinkedAgainstRefCount != 0 || RefCount != 0;
80 }
81 };
82
83public:
84 static void initialize(void *DSOHandle);
85 static ELFNixPlatformRuntimeState &get();
86 static void destroy();
87
88 ELFNixPlatformRuntimeState(void *DSOHandle);
89
90 // Delete copy and move constructors.
91 ELFNixPlatformRuntimeState(const ELFNixPlatformRuntimeState &) = delete;
92 ELFNixPlatformRuntimeState &
93 operator=(const ELFNixPlatformRuntimeState &) = delete;
94 ELFNixPlatformRuntimeState(ELFNixPlatformRuntimeState &&) = delete;
95 ELFNixPlatformRuntimeState &operator=(ELFNixPlatformRuntimeState &&) = delete;
96
97 Error registerObjectSections(ELFNixPerObjectSectionsToRegister POSR);
98 Error registerJITDylib(std::string &Name, void *Handle);
99 Error deregisterJITDylib(void *Handle);
100 Error registerInits(ExecutorAddr HeaderAddr,
101 std::vector<ExecutorAddrRange> Inits);
102 Error deregisterInits(ExecutorAddr HeaderAddr,
103 std::vector<ExecutorAddrRange> Inits);
104 Error registerFinis(ExecutorAddr HeaderAddr,
105 std::vector<ExecutorAddrRange> Finis);
106 Error deregisterFinis(ExecutorAddr HeaderAddr,
107 std::vector<ExecutorAddrRange> Finis);
108 Error deregisterObjectSections(ELFNixPerObjectSectionsToRegister POSR);
109
110 const char *dlerror();
111 void *dlopen(std::string_view Name, int Mode);
112 int dlupdate(void *DSOHandle);
113 int dlclose(void *DSOHandle);
114 void *dlsym(void *DSOHandle, std::string_view Symbol);
115
116 int registerAtExit(void (*F)(void *), void *Arg, void *DSOHandle);
117 void runAtExits(void *DSOHandle);
118 void runAtExits(std::unique_lock<std::recursive_mutex> &JDStateLock,
119 PerJITDylibState &JDS);
120
121 /// Returns the base address of the section containing ThreadData.
122 Expected<std::pair<const char *, size_t>>
123 getThreadDataSectionFor(const char *ThreadData);
124
125 void *getPlatformJDDSOHandle() { return PlatformJDDSOHandle; }
126
127private:
128 PerJITDylibState *getJITDylibStateByHeaderAddr(void *DSOHandle);
129 PerJITDylibState *getJITDylibStateByName(std::string_view Path);
130
131 Error registerThreadDataSection(span<const char> ThreadDataSection);
132
133 Expected<ExecutorAddr> lookupSymbolInJITDylib(void *DSOHandle,
134 std::string_view Symbol);
135
136 Error runInits(std::unique_lock<std::recursive_mutex> &JDStatesLock,
137 PerJITDylibState &JDS);
138 Error runFinis(std::unique_lock<std::recursive_mutex> &JDStatesLock,
139 PerJITDylibState &JDS);
140 Expected<void *> dlopenImpl(std::string_view Path, int Mode);
141 Error dlopenFull(std::unique_lock<std::recursive_mutex> &JDStatesLock,
142 PerJITDylibState &JDS);
143 Error dlopenInitialize(std::unique_lock<std::recursive_mutex> &JDStatesLock,
144 PerJITDylibState &JDS,
145 ELFNixJITDylibDepInfoMap &DepInfo);
146 Error dlupdateImpl(void *DSOHandle);
147 Error dlupdateFull(std::unique_lock<std::recursive_mutex> &JDStatesLock,
148 PerJITDylibState &JDS);
149
150 Error dlcloseImpl(void *DSOHandle);
151 Error dlcloseInitialize(std::unique_lock<std::recursive_mutex> &JDStatesLock,
152 PerJITDylibState &JDS);
153
154 static ELFNixPlatformRuntimeState *MOPS;
155
156 void *PlatformJDDSOHandle;
157
158 // Frame registration functions:
159 void (*registerEHFrameSection)(const void *) = nullptr;
160 void (*deregisterEHFrameSection)(const void *) = nullptr;
161
162 // FIXME: Move to thread-state.
163 std::string DLFcnError;
164
165 std::recursive_mutex JDStatesMutex;
166 std::unordered_map<void *, PerJITDylibState> JDStates;
167 std::unordered_map<std::string, void *> JDNameToHeader;
168
169 std::mutex ThreadDataSectionsMutex;
170 std::map<const char *, size_t> ThreadDataSections;
171};
172
173ELFNixPlatformRuntimeState *ELFNixPlatformRuntimeState::MOPS = nullptr;
174
175void ELFNixPlatformRuntimeState::initialize(void *DSOHandle) {
176 assert(!MOPS && "ELFNixPlatformRuntimeState should be null");
177 MOPS = new ELFNixPlatformRuntimeState(DSOHandle);
178}
179
180ELFNixPlatformRuntimeState &ELFNixPlatformRuntimeState::get() {
181 assert(MOPS && "ELFNixPlatformRuntimeState not initialized");
182 return *MOPS;
183}
184
185void ELFNixPlatformRuntimeState::destroy() {
186 assert(MOPS && "ELFNixPlatformRuntimeState not initialized");
187 delete MOPS;
188}
189
190ELFNixPlatformRuntimeState::ELFNixPlatformRuntimeState(void *DSOHandle)
191 : PlatformJDDSOHandle(DSOHandle) {
192 if (__unw_add_dynamic_eh_frame_section &&
193 __unw_remove_dynamic_eh_frame_section) {
194 registerEHFrameSection = __unw_add_dynamic_eh_frame_section;
195 deregisterEHFrameSection = __unw_remove_dynamic_eh_frame_section;
196 } else {
197 registerEHFrameSection = __register_frame;
198 deregisterEHFrameSection = __deregister_frame;
199 }
200}
201
202Error ELFNixPlatformRuntimeState::registerObjectSections(
203 ELFNixPerObjectSectionsToRegister POSR) {
204 if (POSR.EHFrameSection.Start)
205 registerEHFrameSection(POSR.EHFrameSection.Start.toPtr<const char *>());
206
207 if (POSR.ThreadDataSection.Start) {
208 if (auto Err = registerThreadDataSection(
209 ThreadDataSection: POSR.ThreadDataSection.toSpan<const char>()))
210 return Err;
211 }
212
213 return Error::success();
214}
215
216Error ELFNixPlatformRuntimeState::deregisterObjectSections(
217 ELFNixPerObjectSectionsToRegister POSR) {
218 if (POSR.EHFrameSection.Start)
219 deregisterEHFrameSection(POSR.EHFrameSection.Start.toPtr<const char *>());
220
221 return Error::success();
222}
223
224Error ELFNixPlatformRuntimeState::registerJITDylib(std::string &Name,
225 void *Handle) {
226 std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
227
228 if (JDStates.count(x: Handle)) {
229 std::ostringstream ErrStream;
230 ErrStream << "Duplicate JITDylib registration for header " << Handle
231 << " (name = " << Name << ")";
232 return make_error<StringError>(Args: ErrStream.str());
233 }
234
235 if (JDNameToHeader.count(x: Name)) {
236 std::ostringstream ErrStream;
237 ErrStream << "Duplicate JITDylib registration for header " << Handle
238 << " (header = " << Handle << ")";
239 return make_error<StringError>(Args: ErrStream.str());
240 }
241
242 auto &JD = JDStates[Handle];
243 JD.Header = Handle;
244 JD.Name = std::move(t&: Name);
245 JDNameToHeader[JD.Name] = Handle;
246 return Error::success();
247}
248
249Error ELFNixPlatformRuntimeState::deregisterJITDylib(void *Handle) {
250 std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
251
252 auto I = JDStates.find(x: Handle);
253 if (I == JDStates.end()) {
254 std::ostringstream ErrStream;
255 ErrStream << "Attempted to deregister unrecognized header " << Handle;
256 return make_error<StringError>(Args: ErrStream.str());
257 }
258
259 auto J = JDNameToHeader.find(
260 x: std::string(I->second.Name.data(), I->second.Name.size()));
261 assert(J != JDNameToHeader.end() &&
262 "Missing JDNameToHeader entry for JITDylib");
263 JDNameToHeader.erase(position: J);
264 JDStates.erase(position: I);
265 return Error::success();
266}
267
268Error ELFNixPlatformRuntimeState::registerInits(
269 ExecutorAddr HeaderAddr, std::vector<ExecutorAddrRange> Inits) {
270 std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
271 PerJITDylibState *JDS =
272 getJITDylibStateByHeaderAddr(DSOHandle: HeaderAddr.toPtr<void *>());
273
274 if (!JDS) {
275 std::ostringstream ErrStream;
276 ErrStream << "Could not register object platform sections for "
277 "unrecognized header "
278 << HeaderAddr.toPtr<void *>();
279 return make_error<StringError>(Args: ErrStream.str());
280 }
281
282 for (auto &I : Inits) {
283 JDS->RecordedInits.add(Sec: I.toSpan<void (*)()>());
284 }
285
286 return Error::success();
287}
288
289Error ELFNixPlatformRuntimeState::deregisterInits(
290 ExecutorAddr HeaderAddr, std::vector<ExecutorAddrRange> Inits) {
291
292 std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
293 PerJITDylibState *JDS =
294 getJITDylibStateByHeaderAddr(DSOHandle: HeaderAddr.toPtr<void *>());
295
296 if (!JDS) {
297 std::ostringstream ErrStream;
298 ErrStream << "Could not register object platform sections for unrecognized "
299 "header "
300 << HeaderAddr.toPtr<void *>();
301 return make_error<StringError>(Args: ErrStream.str());
302 }
303
304 for (auto &I : Inits) {
305 JDS->RecordedInits.removeIfPresent(R: I);
306 }
307
308 return Error::success();
309}
310
311Error ELFNixPlatformRuntimeState::registerFinis(
312 ExecutorAddr HeaderAddr, std::vector<ExecutorAddrRange> Finis) {
313 std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
314 PerJITDylibState *JDS =
315 getJITDylibStateByHeaderAddr(DSOHandle: HeaderAddr.toPtr<void *>());
316
317 if (!JDS) {
318 std::ostringstream ErrStream;
319 ErrStream << "Could not register fini sections for unrecognized header "
320 << HeaderAddr.toPtr<void *>();
321 return make_error<StringError>(Args: ErrStream.str());
322 }
323
324 for (auto &F : Finis) {
325 JDS->RecordedFinis.add(Sec: F.toSpan<void (*)()>());
326 }
327
328 return Error::success();
329}
330
331Error ELFNixPlatformRuntimeState::deregisterFinis(
332 ExecutorAddr HeaderAddr, std::vector<ExecutorAddrRange> Finis) {
333 std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
334 PerJITDylibState *JDS =
335 getJITDylibStateByHeaderAddr(DSOHandle: HeaderAddr.toPtr<void *>());
336
337 if (!JDS) {
338 std::ostringstream ErrStream;
339 ErrStream << "Could not deregister fini sections for unrecognized header "
340 << HeaderAddr.toPtr<void *>();
341 return make_error<StringError>(Args: ErrStream.str());
342 }
343
344 for (auto &F : Finis) {
345 JDS->RecordedFinis.removeIfPresent(R: F);
346 }
347
348 return Error::success();
349}
350
351const char *ELFNixPlatformRuntimeState::dlerror() { return DLFcnError.c_str(); }
352
353void *ELFNixPlatformRuntimeState::dlopen(std::string_view Path, int Mode) {
354 if (auto H = dlopenImpl(Path, Mode))
355 return *H;
356 else {
357 // FIXME: Make dlerror thread safe.
358 DLFcnError = toString(Err: H.takeError());
359 return nullptr;
360 }
361}
362
363int ELFNixPlatformRuntimeState::dlupdate(void *DSOHandle) {
364 if (auto Err = dlupdateImpl(DSOHandle)) {
365 // FIXME: Make dlerror thread safe.
366 DLFcnError = toString(Err: std::move(t&: Err));
367 return -1;
368 }
369 return 0;
370}
371
372int ELFNixPlatformRuntimeState::dlclose(void *DSOHandle) {
373 if (auto Err = dlcloseImpl(DSOHandle)) {
374 DLFcnError = toString(Err: std::move(t&: Err));
375 return -1;
376 }
377 return 0;
378}
379
380void *ELFNixPlatformRuntimeState::dlsym(void *DSOHandle,
381 std::string_view Symbol) {
382 auto Addr = lookupSymbolInJITDylib(DSOHandle, Symbol);
383 if (!Addr) {
384 DLFcnError = toString(Err: Addr.takeError());
385 return 0;
386 }
387
388 return Addr->toPtr<void *>();
389}
390
391int ELFNixPlatformRuntimeState::registerAtExit(void (*F)(void *), void *Arg,
392 void *DSOHandle) {
393 // FIXME: Handle out-of-memory errors, returning -1 if OOM.
394 std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
395 auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle);
396 assert(JDS && "JITDylib state not initialized");
397 JDS->AtExits.push_back(x: {.Func: F, .Arg: Arg});
398 return 0;
399}
400
401void ELFNixPlatformRuntimeState::runAtExits(void *DSOHandle) {
402 std::unique_lock<std::recursive_mutex> Lock(JDStatesMutex);
403 PerJITDylibState *JDS = getJITDylibStateByHeaderAddr(DSOHandle);
404
405 if (JDS)
406 runAtExits(JDStateLock&: Lock, JDS&: *JDS);
407}
408
409void ELFNixPlatformRuntimeState::runAtExits(
410 std::unique_lock<std::recursive_mutex> &JDStateLock,
411 PerJITDylibState &JDS) {
412 AtExitsVector V = std::move(t&: JDS.AtExits);
413
414 while (!V.empty()) {
415 auto &AE = V.back();
416 AE.Func(AE.Arg);
417 V.pop_back();
418 }
419}
420
421Expected<std::pair<const char *, size_t>>
422ELFNixPlatformRuntimeState::getThreadDataSectionFor(const char *ThreadData) {
423 std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
424 auto I = ThreadDataSections.upper_bound(x: ThreadData);
425 // Check that we have a valid entry conovering this address.
426 if (I == ThreadDataSections.begin())
427 return make_error<StringError>(Args: "No thread local data section for key");
428 I = std::prev(x: I);
429 if (ThreadData >= I->first + I->second)
430 return make_error<StringError>(Args: "No thread local data section for key");
431 return *I;
432}
433
434ELFNixPlatformRuntimeState::PerJITDylibState *
435ELFNixPlatformRuntimeState::getJITDylibStateByHeaderAddr(void *DSOHandle) {
436 auto I = JDStates.find(x: DSOHandle);
437 if (I == JDStates.end())
438 return nullptr;
439
440 return &I->second;
441}
442
443ELFNixPlatformRuntimeState::PerJITDylibState *
444ELFNixPlatformRuntimeState::getJITDylibStateByName(std::string_view Name) {
445 // FIXME: Avoid creating string copy here.
446 auto I = JDNameToHeader.find(x: std::string(Name.data(), Name.size()));
447 if (I == JDNameToHeader.end())
448 return nullptr;
449 void *H = I->second;
450 auto J = JDStates.find(x: H);
451 assert(J != JDStates.end() &&
452 "JITDylib has name map entry but no header map entry");
453 return &J->second;
454}
455
456Error ELFNixPlatformRuntimeState::registerThreadDataSection(
457 span<const char> ThreadDataSection) {
458 std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
459 auto I = ThreadDataSections.upper_bound(x: ThreadDataSection.data());
460 if (I != ThreadDataSections.begin()) {
461 auto J = std::prev(x: I);
462 if (J->first + J->second > ThreadDataSection.data())
463 return make_error<StringError>(Args: "Overlapping .tdata sections");
464 }
465 ThreadDataSections.insert(
466 position: I, x: std::make_pair(x: ThreadDataSection.data(), y: ThreadDataSection.size()));
467 return Error::success();
468}
469
470Expected<ExecutorAddr>
471ELFNixPlatformRuntimeState::lookupSymbolInJITDylib(void *DSOHandle,
472 std::string_view Sym) {
473 Expected<ExecutorAddr> Result((ExecutorAddr()));
474 if (auto Err = WrapperFunction<SPSExpected<SPSExecutorAddr>(
475 SPSExecutorAddr,
476 SPSString)>::call(Dispatch: JITDispatch(&__orc_rt_elfnix_symbol_lookup_tag),
477 Result, Args: ExecutorAddr::fromPtr(Ptr: DSOHandle), Args: Sym))
478 return std::move(t&: Err);
479 return Result;
480}
481
482Error ELFNixPlatformRuntimeState::runInits(
483 std::unique_lock<std::recursive_mutex> &JDStatesLock,
484 PerJITDylibState &JDS) {
485 std::vector<span<void (*)()>> InitSections;
486 InitSections.reserve(n: JDS.RecordedInits.numNewSections());
487
488 JDS.RecordedInits.processNewSections(
489 ProcessSection: [&](span<void (*)()> Inits) { InitSections.push_back(x: Inits); });
490
491 JDStatesLock.unlock();
492 for (auto Sec : InitSections)
493 for (auto *Init : Sec)
494 Init();
495
496 JDStatesLock.lock();
497
498 return Error::success();
499}
500
501Error ELFNixPlatformRuntimeState::runFinis(
502 std::unique_lock<std::recursive_mutex> &JDStatesLock,
503 PerJITDylibState &JDS) {
504 std::vector<span<void (*)()>> FiniSections;
505
506 // Collect all fini sections (reset to move all to "new" for processing)
507 JDS.RecordedFinis.reset();
508 FiniSections.reserve(n: JDS.RecordedFinis.numNewSections());
509
510 JDS.RecordedFinis.processNewSections(
511 ProcessSection: [&](span<void (*)()> Finis) { FiniSections.push_back(x: Finis); });
512
513 JDStatesLock.unlock();
514
515 // Run in forward order - sections are already sorted correctly by the JIT:
516 // .dtors first (in order), then .fini_array (in descending priority order)
517 for (auto Sec : FiniSections)
518 for (auto *Fini : Sec)
519 Fini();
520
521 JDStatesLock.lock();
522
523 return Error::success();
524}
525
526Expected<void *> ELFNixPlatformRuntimeState::dlopenImpl(std::string_view Path,
527 int Mode) {
528 std::unique_lock<std::recursive_mutex> Lock(JDStatesMutex);
529 PerJITDylibState *JDS = getJITDylibStateByName(Name: Path);
530
531 if (!JDS)
532 return make_error<StringError>(Args: "No registered JTIDylib for path " +
533 std::string(Path.data(), Path.size()));
534
535 if (auto Err = dlopenFull(JDStatesLock&: Lock, JDS&: *JDS))
536 return std::move(t&: Err);
537
538 ++JDS->RefCount;
539
540 return JDS->Header;
541}
542
543Error ELFNixPlatformRuntimeState::dlopenFull(
544 std::unique_lock<std::recursive_mutex> &JDStateLock,
545 PerJITDylibState &JDS) {
546 Expected<ELFNixJITDylibDepInfoMap> DepInfo((ELFNixJITDylibDepInfoMap()));
547 JDStateLock.unlock();
548 if (auto Err = WrapperFunction<SPSExpected<SPSELFNixJITDylibDepInfoMap>(
549 SPSExecutorAddr)>::
550 call(Dispatch: JITDispatch(&__orc_rt_elfnix_push_initializers_tag), Result&: DepInfo,
551 Args: ExecutorAddr::fromPtr(Ptr: JDS.Header)))
552 return Err;
553 JDStateLock.lock();
554
555 if (!DepInfo)
556 return DepInfo.takeError();
557
558 if (auto Err = dlopenInitialize(JDStatesLock&: JDStateLock, JDS, DepInfo&: *DepInfo))
559 return Err;
560
561 if (!DepInfo->empty()) {
562 std::ostringstream ErrStream;
563 ErrStream << "Encountered unrecognized dep-info key headers "
564 "while processing dlopen of "
565 << JDS.Name;
566 return make_error<StringError>(Args: ErrStream.str());
567 }
568
569 return Error::success();
570}
571
572Error ELFNixPlatformRuntimeState::dlopenInitialize(
573 std::unique_lock<std::recursive_mutex> &JDStatesLock, PerJITDylibState &JDS,
574 ELFNixJITDylibDepInfoMap &DepInfo) {
575
576 auto I = DepInfo.find(x: ExecutorAddr::fromPtr(Ptr: JDS.Header));
577 if (I == DepInfo.end())
578 return Error::success();
579
580 auto Deps = std::move(t&: I->second);
581 DepInfo.erase(position: I);
582
583 std::vector<PerJITDylibState *> OldDeps;
584 std::swap(x&: JDS.Deps, y&: OldDeps);
585 JDS.Deps.reserve(n: Deps.size());
586 for (auto H : Deps) {
587 PerJITDylibState *DepJDS = getJITDylibStateByHeaderAddr(DSOHandle: H.toPtr<void *>());
588 if (!DepJDS) {
589 std::ostringstream ErrStream;
590 ErrStream << "Encountered unrecognized dep header " << H.toPtr<void *>()
591 << " while initializing " << JDS.Name;
592 return make_error<StringError>(Args: ErrStream.str());
593 }
594 ++DepJDS->LinkedAgainstRefCount;
595 if (auto Err = dlopenInitialize(JDStatesLock, JDS&: *DepJDS, DepInfo))
596 return Err;
597 }
598
599 if (auto Err = runInits(JDStatesLock, JDS))
600 return Err;
601
602 for (auto *DepJDS : OldDeps) {
603 --DepJDS->LinkedAgainstRefCount;
604 if (!DepJDS->referenced())
605 if (auto Err = dlcloseInitialize(JDStatesLock, JDS&: *DepJDS))
606 return Err;
607 }
608 return Error::success();
609}
610
611Error ELFNixPlatformRuntimeState::dlupdateImpl(void *DSOHandle) {
612 std::unique_lock<std::recursive_mutex> Lock(JDStatesMutex);
613
614 // Try to find JITDylib state by name.
615 auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle);
616
617 if (!JDS) {
618 std::ostringstream ErrStream;
619 ErrStream << "No registered JITDylib for " << DSOHandle;
620 return make_error<StringError>(Args: ErrStream.str());
621 }
622
623 if (!JDS->referenced())
624 return make_error<StringError>(Args: "dlupdate failed, JITDylib must be open.");
625
626 if (auto Err = dlupdateFull(JDStatesLock&: Lock, JDS&: *JDS))
627 return Err;
628
629 return Error::success();
630}
631
632Error ELFNixPlatformRuntimeState::dlupdateFull(
633 std::unique_lock<std::recursive_mutex> &JDStatesLock,
634 PerJITDylibState &JDS) {
635 // Call back to the JIT to push the initializers.
636 Expected<ELFNixJITDylibDepInfoMap> DepInfo((ELFNixJITDylibDepInfoMap()));
637 // Unlock so that we can accept the initializer update.
638 JDStatesLock.unlock();
639 if (auto Err = WrapperFunction<SPSExpected<SPSELFNixJITDylibDepInfoMap>(
640 SPSExecutorAddr)>::
641 call(Dispatch: JITDispatch(&__orc_rt_elfnix_push_initializers_tag), Result&: DepInfo,
642 Args: ExecutorAddr::fromPtr(Ptr: JDS.Header)))
643 return Err;
644 JDStatesLock.lock();
645
646 if (!DepInfo)
647 return DepInfo.takeError();
648
649 if (auto Err = runInits(JDStatesLock, JDS))
650 return Err;
651
652 return Error::success();
653}
654
655Error ELFNixPlatformRuntimeState::dlcloseImpl(void *DSOHandle) {
656
657 std::unique_lock<std::recursive_mutex> Lock(JDStatesMutex);
658 PerJITDylibState *JDS = getJITDylibStateByHeaderAddr(DSOHandle);
659
660 if (!JDS) {
661 std::ostringstream ErrStream;
662 ErrStream << "No registered JITDylib for " << DSOHandle;
663 return make_error<StringError>(Args: ErrStream.str());
664 }
665
666 --JDS->RefCount;
667
668 if (!JDS->referenced())
669 return dlcloseInitialize(JDStatesLock&: Lock, JDS&: *JDS);
670
671 return Error::success();
672}
673
674Error ELFNixPlatformRuntimeState::dlcloseInitialize(
675 std::unique_lock<std::recursive_mutex> &JDStatesLock,
676 PerJITDylibState &JDS) {
677 // Run fini sections BEFORE atexits (mirrors static dtor order)
678 if (auto Err = runFinis(JDStatesLock, JDS))
679 return Err;
680
681 runAtExits(JDStateLock&: JDStatesLock, JDS);
682 JDS.RecordedInits.reset();
683 JDS.RecordedFinis.reset();
684 for (auto *DepJDS : JDS.Deps)
685 if (!JDS.referenced())
686 if (auto Err = dlcloseInitialize(JDStatesLock, JDS&: *DepJDS))
687 return Err;
688
689 return Error::success();
690}
691
692class ELFNixPlatformRuntimeTLVManager {
693public:
694 void *getInstance(const char *ThreadData);
695
696private:
697 std::unordered_map<const char *, char *> Instances;
698 std::unordered_map<const char *, std::unique_ptr<char[]>> AllocatedSections;
699};
700
701void *ELFNixPlatformRuntimeTLVManager::getInstance(const char *ThreadData) {
702 auto I = Instances.find(x: ThreadData);
703 if (I != Instances.end())
704 return I->second;
705 auto TDS =
706 ELFNixPlatformRuntimeState::get().getThreadDataSectionFor(ThreadData);
707 if (!TDS) {
708 __orc_rt_log_error(ErrMsg: toString(Err: TDS.takeError()).c_str());
709 return nullptr;
710 }
711
712 auto &Allocated = AllocatedSections[TDS->first];
713 if (!Allocated) {
714 Allocated = std::make_unique<char[]>(num: TDS->second);
715 memcpy(dest: Allocated.get(), src: TDS->first, n: TDS->second);
716 }
717 size_t ThreadDataDelta = ThreadData - TDS->first;
718 assert(ThreadDataDelta <= TDS->second && "ThreadData outside section bounds");
719
720 char *Instance = Allocated.get() + ThreadDataDelta;
721 Instances[ThreadData] = Instance;
722 return Instance;
723}
724
725void destroyELFNixTLVMgr(void *ELFNixTLVMgr) {
726 delete static_cast<ELFNixPlatformRuntimeTLVManager *>(ELFNixTLVMgr);
727}
728
729} // end anonymous namespace
730
731//------------------------------------------------------------------------------
732// JIT entry points
733//------------------------------------------------------------------------------
734
735ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
736__orc_rt_elfnix_platform_bootstrap(char *ArgData, size_t ArgSize) {
737 return WrapperFunction<SPSError(SPSExecutorAddr)>::handle(
738 ArgData, ArgSize,
739 Handler: [](ExecutorAddr DSOHandle) {
740 ELFNixPlatformRuntimeState::initialize(
741 DSOHandle: DSOHandle.toPtr<void *>());
742 return Error::success();
743 })
744 .release();
745}
746
747ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
748__orc_rt_elfnix_platform_shutdown(char *ArgData, size_t ArgSize) {
749 return WrapperFunction<SPSError()>::handle(
750 ArgData, ArgSize,
751 Handler: []() {
752 ELFNixPlatformRuntimeState::destroy();
753 return Error::success();
754 })
755 .release();
756}
757
758ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
759__orc_rt_elfnix_register_jitdylib(char *ArgData, size_t ArgSize) {
760 return WrapperFunction<SPSError(SPSString, SPSExecutorAddr)>::handle(
761 ArgData, ArgSize,
762 Handler: [](std::string &JDName, ExecutorAddr HeaderAddr) {
763 return ELFNixPlatformRuntimeState::get().registerJITDylib(
764 Name&: JDName, Handle: HeaderAddr.toPtr<void *>());
765 })
766 .release();
767}
768
769ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
770__orc_rt_elfnix_deregister_jitdylib(char *ArgData, size_t ArgSize) {
771 return WrapperFunction<SPSError(SPSExecutorAddr)>::handle(
772 ArgData, ArgSize,
773 Handler: [](ExecutorAddr HeaderAddr) {
774 return ELFNixPlatformRuntimeState::get().deregisterJITDylib(
775 Handle: HeaderAddr.toPtr<void *>());
776 })
777 .release();
778}
779
780ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
781__orc_rt_elfnix_register_init_sections(char *ArgData, size_t ArgSize) {
782 return WrapperFunction<SPSError(SPSExecutorAddr,
783 SPSSequence<SPSExecutorAddrRange>)>::
784 handle(ArgData, ArgSize,
785 Handler: [](ExecutorAddr HeaderAddr,
786 std::vector<ExecutorAddrRange> &Inits) {
787 return ELFNixPlatformRuntimeState::get().registerInits(
788 HeaderAddr, Inits: std::move(t&: Inits));
789 })
790 .release();
791}
792
793ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
794__orc_rt_elfnix_deregister_init_sections(char *ArgData, size_t ArgSize) {
795 return WrapperFunction<SPSError(SPSExecutorAddr,
796 SPSSequence<SPSExecutorAddrRange>)>::
797 handle(ArgData, ArgSize,
798 Handler: [](ExecutorAddr HeaderAddr,
799 std::vector<ExecutorAddrRange> &Inits) {
800 return ELFNixPlatformRuntimeState::get().deregisterInits(
801 HeaderAddr, Inits: std::move(t&: Inits));
802 })
803 .release();
804}
805
806ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
807__orc_rt_elfnix_register_fini_sections(char *ArgData, size_t ArgSize) {
808 return WrapperFunction<SPSError(SPSExecutorAddr,
809 SPSSequence<SPSExecutorAddrRange>)>::
810 handle(ArgData, ArgSize,
811 Handler: [](ExecutorAddr HeaderAddr,
812 std::vector<ExecutorAddrRange> &Finis) {
813 return ELFNixPlatformRuntimeState::get().registerFinis(
814 HeaderAddr, Finis: std::move(t&: Finis));
815 })
816 .release();
817}
818
819ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
820__orc_rt_elfnix_deregister_fini_sections(char *ArgData, size_t ArgSize) {
821 return WrapperFunction<SPSError(SPSExecutorAddr,
822 SPSSequence<SPSExecutorAddrRange>)>::
823 handle(ArgData, ArgSize,
824 Handler: [](ExecutorAddr HeaderAddr,
825 std::vector<ExecutorAddrRange> &Finis) {
826 return ELFNixPlatformRuntimeState::get().deregisterFinis(
827 HeaderAddr, Finis: std::move(t&: Finis));
828 })
829 .release();
830}
831
832/// Wrapper function for registering metadata on a per-object basis.
833ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
834__orc_rt_elfnix_register_object_sections(char *ArgData, size_t ArgSize) {
835 return WrapperFunction<SPSError(SPSELFNixPerObjectSectionsToRegister)>::
836 handle(ArgData, ArgSize,
837 Handler: [](ELFNixPerObjectSectionsToRegister &POSR) {
838 return ELFNixPlatformRuntimeState::get().registerObjectSections(
839 POSR: std::move(t&: POSR));
840 })
841 .release();
842}
843
844/// Wrapper for releasing per-object metadat.
845ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
846__orc_rt_elfnix_deregister_object_sections(char *ArgData, size_t ArgSize) {
847 return WrapperFunction<SPSError(SPSELFNixPerObjectSectionsToRegister)>::
848 handle(ArgData, ArgSize,
849 Handler: [](ELFNixPerObjectSectionsToRegister &POSR) {
850 return ELFNixPlatformRuntimeState::get()
851 .deregisterObjectSections(POSR: std::move(t&: POSR));
852 })
853 .release();
854}
855
856//------------------------------------------------------------------------------
857// TLV support
858//------------------------------------------------------------------------------
859
860ORC_RT_INTERFACE void *__orc_rt_elfnix_tls_get_addr_impl(TLSInfoEntry *D) {
861 auto *TLVMgr = static_cast<ELFNixPlatformRuntimeTLVManager *>(
862 pthread_getspecific(key: D->Key));
863 if (!TLVMgr)
864 TLVMgr = new ELFNixPlatformRuntimeTLVManager();
865 if (pthread_setspecific(key: D->Key, pointer: TLVMgr)) {
866 __orc_rt_log_error(ErrMsg: "Call to pthread_setspecific failed");
867 return nullptr;
868 }
869
870 return TLVMgr->getInstance(
871 ThreadData: reinterpret_cast<char *>(static_cast<uintptr_t>(D->DataAddress)));
872}
873
874ORC_RT_INTERFACE ptrdiff_t ___orc_rt_elfnix_tlsdesc_resolver_impl(
875 TLSDescriptor *D, const char *ThreadPointer) {
876 const char *TLVPtr = reinterpret_cast<const char *>(
877 __orc_rt_elfnix_tls_get_addr_impl(D: D->InfoEntry));
878 return TLVPtr - ThreadPointer;
879}
880
881ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
882__orc_rt_elfnix_create_pthread_key(char *ArgData, size_t ArgSize) {
883 return WrapperFunction<SPSExpected<uint64_t>(void)>::handle(
884 ArgData, ArgSize,
885 Handler: []() -> Expected<uint64_t> {
886 pthread_key_t Key;
887 if (int Err = pthread_key_create(key: &Key, destr_function: destroyELFNixTLVMgr)) {
888 __orc_rt_log_error(ErrMsg: "Call to pthread_key_create failed");
889 return make_error<StringError>(Args: strerror(errnum: Err));
890 }
891 return static_cast<uint64_t>(Key);
892 })
893 .release();
894}
895
896//------------------------------------------------------------------------------
897// cxa_atexit support
898//------------------------------------------------------------------------------
899
900int __orc_rt_elfnix_cxa_atexit(void (*func)(void *), void *arg,
901 void *dso_handle) {
902 return ELFNixPlatformRuntimeState::get().registerAtExit(F: func, Arg: arg,
903 DSOHandle: dso_handle);
904}
905
906int __orc_rt_elfnix_atexit(void (*func)(void *)) {
907 auto &PlatformRTState = ELFNixPlatformRuntimeState::get();
908 return ELFNixPlatformRuntimeState::get().registerAtExit(
909 F: func, NULL, DSOHandle: PlatformRTState.getPlatformJDDSOHandle());
910}
911
912void __orc_rt_elfnix_cxa_finalize(void *dso_handle) {
913 ELFNixPlatformRuntimeState::get().runAtExits(DSOHandle: dso_handle);
914}
915
916//------------------------------------------------------------------------------
917// JIT'd dlfcn alternatives.
918//------------------------------------------------------------------------------
919
920const char *__orc_rt_elfnix_jit_dlerror() {
921 return ELFNixPlatformRuntimeState::get().dlerror();
922}
923
924void *__orc_rt_elfnix_jit_dlopen(const char *path, int mode) {
925 return ELFNixPlatformRuntimeState::get().dlopen(Path: path, Mode: mode);
926}
927
928int __orc_rt_elfnix_jit_dlupdate(void *dso_handle) {
929 return ELFNixPlatformRuntimeState::get().dlupdate(DSOHandle: dso_handle);
930}
931
932int __orc_rt_elfnix_jit_dlclose(void *dso_handle) {
933 return ELFNixPlatformRuntimeState::get().dlclose(DSOHandle: dso_handle);
934}
935
936void *__orc_rt_elfnix_jit_dlsym(void *dso_handle, const char *symbol) {
937 return ELFNixPlatformRuntimeState::get().dlsym(DSOHandle: dso_handle, Symbol: symbol);
938}
939
940//------------------------------------------------------------------------------
941// ELFNix Run Program
942//------------------------------------------------------------------------------
943
944ORC_RT_INTERFACE int64_t __orc_rt_elfnix_run_program(
945 const char *JITDylibName, const char *EntrySymbolName, int argc,
946 char *argv[]) {
947 using MainTy = int (*)(int, char *[]);
948
949 void *H = __orc_rt_elfnix_jit_dlopen(path: JITDylibName,
950 mode: orc_rt::elfnix::ORC_RT_RTLD_LAZY);
951 if (!H) {
952 __orc_rt_log_error(ErrMsg: __orc_rt_elfnix_jit_dlerror());
953 return -1;
954 }
955
956 auto *Main =
957 reinterpret_cast<MainTy>(__orc_rt_elfnix_jit_dlsym(dso_handle: H, symbol: EntrySymbolName));
958
959 if (!Main) {
960 __orc_rt_log_error(ErrMsg: __orc_rt_elfnix_jit_dlerror());
961 return -1;
962 }
963
964 int Result = Main(argc, argv);
965
966 if (__orc_rt_elfnix_jit_dlclose(dso_handle: H) == -1)
967 __orc_rt_log_error(ErrMsg: __orc_rt_elfnix_jit_dlerror());
968
969 return Result;
970}
971