1//===------- COFFPlatform.cpp - Utilities for executing COFF in Orc -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/Orc/COFFPlatform.h"
10#include "llvm/ExecutionEngine/Orc/DebugUtils.h"
11#include "llvm/ExecutionEngine/Orc/LookupAndRecordAddrs.h"
12#include "llvm/ExecutionEngine/Orc/ObjectFileInterface.h"
13#include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h"
14
15#include "llvm/Object/COFF.h"
16
17#include "llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h"
18
19#include "llvm/ExecutionEngine/JITLink/x86_64.h"
20
21#define DEBUG_TYPE "orc"
22
23using namespace llvm;
24using namespace llvm::orc;
25using namespace llvm::orc::shared;
26
27namespace llvm {
28namespace orc {
29namespace shared {
30
31using SPSCOFFJITDylibDepInfo = SPSSequence<SPSExecutorAddr>;
32using SPSCOFFJITDylibDepInfoMap =
33 SPSSequence<SPSTuple<SPSExecutorAddr, SPSCOFFJITDylibDepInfo>>;
34using SPSCOFFObjectSectionsMap =
35 SPSSequence<SPSTuple<SPSString, SPSExecutorAddrRange>>;
36using SPSCOFFRegisterObjectSectionsArgs =
37 SPSArgList<SPSExecutorAddr, SPSCOFFObjectSectionsMap, bool>;
38using SPSCOFFDeregisterObjectSectionsArgs =
39 SPSArgList<SPSExecutorAddr, SPSCOFFObjectSectionsMap>;
40
41} // namespace shared
42} // namespace orc
43} // namespace llvm
44namespace {
45
46class COFFHeaderMaterializationUnit : public MaterializationUnit {
47public:
48 COFFHeaderMaterializationUnit(COFFPlatform &CP,
49 const SymbolStringPtr &HeaderStartSymbol)
50 : MaterializationUnit(createHeaderInterface(MOP&: CP, HeaderStartSymbol)),
51 CP(CP) {}
52
53 StringRef getName() const override { return "COFFHeaderMU"; }
54
55 void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
56 unsigned PointerSize;
57 llvm::endianness Endianness;
58 const auto &TT = CP.getExecutionSession().getTargetTriple();
59
60 switch (TT.getArch()) {
61 case Triple::x86_64:
62 PointerSize = 8;
63 Endianness = llvm::endianness::little;
64 break;
65 default:
66 llvm_unreachable("Unrecognized architecture");
67 }
68
69 auto G = std::make_unique<jitlink::LinkGraph>(
70 args: "<COFFHeaderMU>", args: TT, args&: PointerSize, args&: Endianness,
71 args&: jitlink::getGenericEdgeKindName);
72 auto &HeaderSection = G->createSection(Name: "__header", Prot: MemProt::Read);
73 auto &HeaderBlock = createHeaderBlock(G&: *G, HeaderSection);
74
75 // Init symbol is __ImageBase symbol.
76 auto &ImageBaseSymbol = G->addDefinedSymbol(
77 Content&: HeaderBlock, Offset: 0, Name: *R->getInitializerSymbol(), Size: HeaderBlock.getSize(),
78 L: jitlink::Linkage::Strong, S: jitlink::Scope::Default, IsCallable: false, IsLive: true);
79
80 addImageBaseRelocationEdge(B&: HeaderBlock, ImageBase&: ImageBaseSymbol);
81
82 CP.getObjectLinkingLayer().emit(R: std::move(R), G: std::move(G));
83 }
84
85 void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
86
87private:
88 struct HeaderSymbol {
89 const char *Name;
90 uint64_t Offset;
91 };
92
93 struct NTHeader {
94 support::ulittle32_t PEMagic;
95 object::coff_file_header FileHeader;
96 struct PEHeader {
97 object::pe32plus_header Header;
98 object::data_directory DataDirectory[COFF::NUM_DATA_DIRECTORIES + 1];
99 } OptionalHeader;
100 };
101
102 struct HeaderBlockContent {
103 object::dos_header DOSHeader;
104 COFFHeaderMaterializationUnit::NTHeader NTHeader;
105 };
106
107 static jitlink::Block &createHeaderBlock(jitlink::LinkGraph &G,
108 jitlink::Section &HeaderSection) {
109 HeaderBlockContent Hdr = {};
110
111 // Set up magic
112 Hdr.DOSHeader.Magic[0] = 'M';
113 Hdr.DOSHeader.Magic[1] = 'Z';
114 Hdr.DOSHeader.AddressOfNewExeHeader =
115 offsetof(HeaderBlockContent, NTHeader);
116 uint32_t PEMagic = *reinterpret_cast<const uint32_t *>(COFF::PEMagic);
117 Hdr.NTHeader.PEMagic = PEMagic;
118 Hdr.NTHeader.OptionalHeader.Header.Magic = COFF::PE32Header::PE32_PLUS;
119
120 switch (G.getTargetTriple().getArch()) {
121 case Triple::x86_64:
122 Hdr.NTHeader.FileHeader.Machine = COFF::IMAGE_FILE_MACHINE_AMD64;
123 break;
124 default:
125 llvm_unreachable("Unrecognized architecture");
126 }
127
128 auto HeaderContent = G.allocateContent(
129 Source: ArrayRef<char>(reinterpret_cast<const char *>(&Hdr), sizeof(Hdr)));
130
131 return G.createContentBlock(Parent&: HeaderSection, Content: HeaderContent, Address: ExecutorAddr(), Alignment: 8,
132 AlignmentOffset: 0);
133 }
134
135 static void addImageBaseRelocationEdge(jitlink::Block &B,
136 jitlink::Symbol &ImageBase) {
137 auto ImageBaseOffset = offsetof(HeaderBlockContent, NTHeader) +
138 offsetof(NTHeader, OptionalHeader) +
139 offsetof(object::pe32plus_header, ImageBase);
140 B.addEdge(K: jitlink::x86_64::Pointer64, Offset: ImageBaseOffset, Target&: ImageBase, Addend: 0);
141 }
142
143 static MaterializationUnit::Interface
144 createHeaderInterface(COFFPlatform &MOP,
145 const SymbolStringPtr &HeaderStartSymbol) {
146 SymbolFlagsMap HeaderSymbolFlags;
147
148 HeaderSymbolFlags[HeaderStartSymbol] = JITSymbolFlags::Exported;
149
150 return MaterializationUnit::Interface(std::move(HeaderSymbolFlags),
151 HeaderStartSymbol);
152 }
153
154 COFFPlatform &CP;
155};
156
157} // end anonymous namespace
158
159namespace llvm {
160namespace orc {
161
162Expected<std::unique_ptr<COFFPlatform>> COFFPlatform::Create(
163 ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer,
164 JITDylib &PlatformJD, std::unique_ptr<MemoryBuffer> OrcRuntimeArchiveBuffer,
165 LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime,
166 const char *VCRuntimePath, std::optional<SymbolAliasMap> RuntimeAliases) {
167
168 // If the target is not supported then bail out immediately.
169 if (!supportedTarget(TT: ES.getTargetTriple()))
170 return make_error<StringError>(Args: "Unsupported COFFPlatform triple: " +
171 ES.getTargetTriple().str(),
172 Args: inconvertibleErrorCode());
173
174 auto &EPC = ES.getExecutorProcessControl();
175
176 auto GeneratorArchive =
177 object::Archive::create(Source: OrcRuntimeArchiveBuffer->getMemBufferRef());
178 if (!GeneratorArchive)
179 return GeneratorArchive.takeError();
180
181 auto OrcRuntimeArchiveGenerator = StaticLibraryDefinitionGenerator::Create(
182 L&: ObjLinkingLayer, ArchiveBuffer: nullptr, Archive: std::move(*GeneratorArchive));
183 if (!OrcRuntimeArchiveGenerator)
184 return OrcRuntimeArchiveGenerator.takeError();
185
186 // We need a second instance of the archive (for now) for the Platform. We
187 // can `cantFail` this call, since if it were going to fail it would have
188 // failed above.
189 auto RuntimeArchive = cantFail(
190 ValOrErr: object::Archive::create(Source: OrcRuntimeArchiveBuffer->getMemBufferRef()));
191
192 // Create default aliases if the caller didn't supply any.
193 if (!RuntimeAliases)
194 RuntimeAliases = standardPlatformAliases(ES);
195
196 // Define the aliases.
197 if (auto Err = PlatformJD.define(MU: symbolAliases(Aliases: std::move(*RuntimeAliases))))
198 return std::move(Err);
199
200 auto &HostFuncJD = ES.createBareJITDylib(Name: "$<PlatformRuntimeHostFuncJD>");
201
202 // Add JIT-dispatch function support symbols.
203 if (auto Err = HostFuncJD.define(
204 MU: absoluteSymbols(Symbols: {{ES.intern(SymName: "__orc_rt_jit_dispatch"),
205 {EPC.getJITDispatchInfo().JITDispatchFunction,
206 JITSymbolFlags::Exported}},
207 {ES.intern(SymName: "__orc_rt_jit_dispatch_ctx"),
208 {EPC.getJITDispatchInfo().JITDispatchContext,
209 JITSymbolFlags::Exported}}})))
210 return std::move(Err);
211
212 PlatformJD.addToLinkOrder(JD&: HostFuncJD);
213
214 // Create the instance.
215 Error Err = Error::success();
216 auto P = std::unique_ptr<COFFPlatform>(new COFFPlatform(
217 ES, ObjLinkingLayer, PlatformJD, std::move(*OrcRuntimeArchiveGenerator),
218 std::move(OrcRuntimeArchiveBuffer), std::move(RuntimeArchive),
219 std::move(LoadDynLibrary), StaticVCRuntime, VCRuntimePath, Err));
220 if (Err)
221 return std::move(Err);
222 return std::move(P);
223}
224
225Expected<std::unique_ptr<COFFPlatform>>
226COFFPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer,
227 JITDylib &PlatformJD, const char *OrcRuntimePath,
228 LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime,
229 const char *VCRuntimePath,
230 std::optional<SymbolAliasMap> RuntimeAliases) {
231
232 auto ArchiveBuffer = MemoryBuffer::getFile(Filename: OrcRuntimePath);
233 if (!ArchiveBuffer)
234 return createFileError(F: OrcRuntimePath, EC: ArchiveBuffer.getError());
235
236 return Create(ES, ObjLinkingLayer, PlatformJD, OrcRuntimeArchiveBuffer: std::move(*ArchiveBuffer),
237 LoadDynLibrary: std::move(LoadDynLibrary), StaticVCRuntime, VCRuntimePath,
238 RuntimeAliases: std::move(RuntimeAliases));
239}
240
241Expected<MemoryBufferRef> COFFPlatform::getPerJDObjectFile() {
242 auto PerJDObj = OrcRuntimeArchive->findSym(name: "__orc_rt_coff_per_jd_marker");
243 if (!PerJDObj)
244 return PerJDObj.takeError();
245
246 if (!*PerJDObj)
247 return make_error<StringError>(Args: "Could not find per jd object file",
248 Args: inconvertibleErrorCode());
249
250 auto Buffer = (*PerJDObj)->getAsBinary();
251 if (!Buffer)
252 return Buffer.takeError();
253
254 return (*Buffer)->getMemoryBufferRef();
255}
256
257static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases,
258 ArrayRef<std::pair<const char *, const char *>> AL) {
259 for (auto &KV : AL) {
260 auto AliasName = ES.intern(SymName: KV.first);
261 assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map");
262 Aliases[std::move(AliasName)] = {ES.intern(SymName: KV.second),
263 JITSymbolFlags::Exported};
264 }
265}
266
267Error COFFPlatform::setupJITDylib(JITDylib &JD) {
268 if (auto Err = JD.define(MU: std::make_unique<COFFHeaderMaterializationUnit>(
269 args&: *this, args&: COFFHeaderStartSymbol)))
270 return Err;
271
272 if (auto Err = ES.lookup(SearchOrder: {&JD}, Symbol: COFFHeaderStartSymbol).takeError())
273 return Err;
274
275 // Define the CXX aliases.
276 SymbolAliasMap CXXAliases;
277 addAliases(ES, Aliases&: CXXAliases, AL: requiredCXXAliases());
278 if (auto Err = JD.define(MU: symbolAliases(Aliases: std::move(CXXAliases))))
279 return Err;
280
281 auto PerJDObj = getPerJDObjectFile();
282 if (!PerJDObj)
283 return PerJDObj.takeError();
284
285 auto I = getObjectFileInterface(ES, ObjBuffer: *PerJDObj);
286 if (!I)
287 return I.takeError();
288
289 if (auto Err = ObjLinkingLayer.add(
290 JD, O: MemoryBuffer::getMemBuffer(Ref: *PerJDObj, RequiresNullTerminator: false), I: std::move(*I)))
291 return Err;
292
293 if (!Bootstrapping) {
294 auto ImportedLibs = StaticVCRuntime
295 ? VCRuntimeBootstrap->loadStaticVCRuntime(JD)
296 : VCRuntimeBootstrap->loadDynamicVCRuntime(JD);
297 if (!ImportedLibs)
298 return ImportedLibs.takeError();
299 for (auto &Lib : *ImportedLibs)
300 if (auto Err = LoadDynLibrary(JD, Lib))
301 return Err;
302 if (StaticVCRuntime)
303 if (auto Err = VCRuntimeBootstrap->initializeStaticVCRuntime(JD))
304 return Err;
305 }
306
307 JD.addGenerator(DefGenerator: DLLImportDefinitionGenerator::Create(ES, L&: ObjLinkingLayer));
308 return Error::success();
309}
310
311Error COFFPlatform::teardownJITDylib(JITDylib &JD) {
312 std::lock_guard<std::mutex> Lock(PlatformMutex);
313 auto I = JITDylibToHeaderAddr.find(Val: &JD);
314 if (I != JITDylibToHeaderAddr.end()) {
315 assert(HeaderAddrToJITDylib.count(I->second) &&
316 "HeaderAddrToJITDylib missing entry");
317 HeaderAddrToJITDylib.erase(Val: I->second);
318 JITDylibToHeaderAddr.erase(I);
319 }
320 return Error::success();
321}
322
323Error COFFPlatform::notifyAdding(ResourceTracker &RT,
324 const MaterializationUnit &MU) {
325 auto &JD = RT.getJITDylib();
326 const auto &InitSym = MU.getInitializerSymbol();
327 if (!InitSym)
328 return Error::success();
329
330 RegisteredInitSymbols[&JD].add(Name: InitSym,
331 Flags: SymbolLookupFlags::WeaklyReferencedSymbol);
332
333 LLVM_DEBUG({
334 dbgs() << "COFFPlatform: Registered init symbol " << *InitSym << " for MU "
335 << MU.getName() << "\n";
336 });
337 return Error::success();
338}
339
340Error COFFPlatform::notifyRemoving(ResourceTracker &RT) {
341 llvm_unreachable("Not supported yet");
342}
343
344SymbolAliasMap COFFPlatform::standardPlatformAliases(ExecutionSession &ES) {
345 SymbolAliasMap Aliases;
346 addAliases(ES, Aliases, AL: standardRuntimeUtilityAliases());
347 return Aliases;
348}
349
350ArrayRef<std::pair<const char *, const char *>>
351COFFPlatform::requiredCXXAliases() {
352 static const std::pair<const char *, const char *> RequiredCXXAliases[] = {
353 {"_CxxThrowException", "__orc_rt_coff_cxx_throw_exception"},
354 {"_onexit", "__orc_rt_coff_onexit_per_jd"},
355 {"atexit", "__orc_rt_coff_atexit_per_jd"}};
356
357 return ArrayRef<std::pair<const char *, const char *>>(RequiredCXXAliases);
358}
359
360ArrayRef<std::pair<const char *, const char *>>
361COFFPlatform::standardRuntimeUtilityAliases() {
362 static const std::pair<const char *, const char *>
363 StandardRuntimeUtilityAliases[] = {
364 {"__orc_rt_run_program", "__orc_rt_coff_run_program"},
365 {"__orc_rt_jit_dlerror", "__orc_rt_coff_jit_dlerror"},
366 {"__orc_rt_jit_dlopen", "__orc_rt_coff_jit_dlopen"},
367 {"__orc_rt_jit_dlclose", "__orc_rt_coff_jit_dlclose"},
368 {"__orc_rt_jit_dlsym", "__orc_rt_coff_jit_dlsym"},
369 {"__orc_rt_log_error", "__orc_rt_log_error_to_stderr"}};
370
371 return ArrayRef<std::pair<const char *, const char *>>(
372 StandardRuntimeUtilityAliases);
373}
374
375bool COFFPlatform::supportedTarget(const Triple &TT) {
376 switch (TT.getArch()) {
377 case Triple::x86_64:
378 return true;
379 default:
380 return false;
381 }
382}
383
384COFFPlatform::COFFPlatform(
385 ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer,
386 JITDylib &PlatformJD,
387 std::unique_ptr<StaticLibraryDefinitionGenerator> OrcRuntimeGenerator,
388 std::unique_ptr<MemoryBuffer> OrcRuntimeArchiveBuffer,
389 std::unique_ptr<object::Archive> OrcRuntimeArchive,
390 LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime,
391 const char *VCRuntimePath, Error &Err)
392 : ES(ES), ObjLinkingLayer(ObjLinkingLayer),
393 LoadDynLibrary(std::move(LoadDynLibrary)),
394 OrcRuntimeArchiveBuffer(std::move(OrcRuntimeArchiveBuffer)),
395 OrcRuntimeArchive(std::move(OrcRuntimeArchive)),
396 StaticVCRuntime(StaticVCRuntime),
397 COFFHeaderStartSymbol(ES.intern(SymName: "__ImageBase")) {
398 ErrorAsOutParameter _(&Err);
399
400 Bootstrapping.store(i: true);
401 ObjLinkingLayer.addPlugin(P: std::make_unique<COFFPlatformPlugin>(args&: *this));
402
403 // Load vc runtime
404 auto VCRT =
405 COFFVCRuntimeBootstrapper::Create(ES, ObjLinkingLayer, RuntimePath: VCRuntimePath);
406 if (!VCRT) {
407 Err = VCRT.takeError();
408 return;
409 }
410 VCRuntimeBootstrap = std::move(*VCRT);
411
412 for (auto &Lib : OrcRuntimeGenerator->getImportedDynamicLibraries())
413 DylibsToPreload.insert(x: Lib);
414
415 auto ImportedLibs =
416 StaticVCRuntime ? VCRuntimeBootstrap->loadStaticVCRuntime(JD&: PlatformJD)
417 : VCRuntimeBootstrap->loadDynamicVCRuntime(JD&: PlatformJD);
418 if (!ImportedLibs) {
419 Err = ImportedLibs.takeError();
420 return;
421 }
422
423 for (auto &Lib : *ImportedLibs)
424 DylibsToPreload.insert(x: Lib);
425
426 PlatformJD.addGenerator(DefGenerator: std::move(OrcRuntimeGenerator));
427
428 // PlatformJD hasn't been set up by the platform yet (since we're creating
429 // the platform now), so set it up.
430 if (auto E2 = setupJITDylib(PlatformJD)) {
431 Err = std::move(E2);
432 return;
433 }
434
435 for (auto& Lib : DylibsToPreload)
436 if (auto E2 = this->LoadDynLibrary(PlatformJD, Lib)) {
437 Err = std::move(E2);
438 return;
439 }
440
441 if (StaticVCRuntime)
442 if (auto E2 = VCRuntimeBootstrap->initializeStaticVCRuntime(JD&: PlatformJD)) {
443 Err = std::move(E2);
444 return;
445 }
446
447 // Associate wrapper function tags with JIT-side function implementations.
448 if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) {
449 Err = std::move(E2);
450 return;
451 }
452
453 // Lookup addresses of runtime functions callable by the platform,
454 // call the platform bootstrap function to initialize the platform-state
455 // object in the executor.
456 if (auto E2 = bootstrapCOFFRuntime(PlatformJD)) {
457 Err = std::move(E2);
458 return;
459 }
460
461 Bootstrapping.store(i: false);
462 JDBootstrapStates.clear();
463}
464
465Expected<COFFPlatform::JITDylibDepMap>
466COFFPlatform::buildJDDepMap(JITDylib &JD) {
467 return ES.runSessionLocked(F: [&]() -> Expected<JITDylibDepMap> {
468 JITDylibDepMap JDDepMap;
469
470 SmallVector<JITDylib *, 16> Worklist({&JD});
471 while (!Worklist.empty()) {
472 auto CurJD = Worklist.back();
473 Worklist.pop_back();
474
475 auto &DM = JDDepMap[CurJD];
476 CurJD->withLinkOrderDo(F: [&](const JITDylibSearchOrder &O) {
477 DM.reserve(N: O.size());
478 for (auto &KV : O) {
479 if (KV.first == CurJD)
480 continue;
481 {
482 // Bare jitdylibs not known to the platform
483 std::lock_guard<std::mutex> Lock(PlatformMutex);
484 if (!JITDylibToHeaderAddr.count(Val: KV.first)) {
485 LLVM_DEBUG({
486 dbgs() << "JITDylib unregistered to COFFPlatform detected in "
487 "LinkOrder: "
488 << CurJD->getName() << "\n";
489 });
490 continue;
491 }
492 }
493 DM.push_back(Elt: KV.first);
494 // Push unvisited entry.
495 if (!JDDepMap.count(Val: KV.first)) {
496 Worklist.push_back(Elt: KV.first);
497 JDDepMap[KV.first] = {};
498 }
499 }
500 });
501 }
502 return std::move(JDDepMap);
503 });
504}
505
506void COFFPlatform::pushInitializersLoop(PushInitializersSendResultFn SendResult,
507 JITDylibSP JD,
508 JITDylibDepMap &JDDepMap) {
509 SmallVector<JITDylib *, 16> Worklist({JD.get()});
510 DenseSet<JITDylib *> Visited({JD.get()});
511 DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols;
512 ES.runSessionLocked(F: [&]() {
513 while (!Worklist.empty()) {
514 auto CurJD = Worklist.back();
515 Worklist.pop_back();
516
517 auto RISItr = RegisteredInitSymbols.find(Val: CurJD);
518 if (RISItr != RegisteredInitSymbols.end()) {
519 NewInitSymbols[CurJD] = std::move(RISItr->second);
520 RegisteredInitSymbols.erase(I: RISItr);
521 }
522
523 for (auto *DepJD : JDDepMap[CurJD])
524 if (!Visited.count(V: DepJD)) {
525 Worklist.push_back(Elt: DepJD);
526 Visited.insert(V: DepJD);
527 }
528 }
529 });
530
531 // If there are no further init symbols to look up then send the link order
532 // (as a list of header addresses) to the caller.
533 if (NewInitSymbols.empty()) {
534 // Build the dep info map to return.
535 COFFJITDylibDepInfoMap DIM;
536 DIM.reserve(n: JDDepMap.size());
537 for (auto &KV : JDDepMap) {
538 std::lock_guard<std::mutex> Lock(PlatformMutex);
539 COFFJITDylibDepInfo DepInfo;
540 DepInfo.reserve(n: KV.second.size());
541 for (auto &Dep : KV.second) {
542 DepInfo.push_back(x: JITDylibToHeaderAddr[Dep]);
543 }
544 auto H = JITDylibToHeaderAddr[KV.first];
545 DIM.push_back(x: std::make_pair(x&: H, y: std::move(DepInfo)));
546 }
547 SendResult(DIM);
548 return;
549 }
550
551 // Otherwise issue a lookup and re-run this phase when it completes.
552 lookupInitSymbolsAsync(
553 OnComplete: [this, SendResult = std::move(SendResult), &JD,
554 JDDepMap = std::move(JDDepMap)](Error Err) mutable {
555 if (Err)
556 SendResult(std::move(Err));
557 else
558 pushInitializersLoop(SendResult: std::move(SendResult), JD, JDDepMap);
559 },
560 ES, InitSyms: std::move(NewInitSymbols));
561}
562
563void COFFPlatform::rt_pushInitializers(PushInitializersSendResultFn SendResult,
564 ExecutorAddr JDHeaderAddr) {
565 JITDylibSP JD;
566 {
567 std::lock_guard<std::mutex> Lock(PlatformMutex);
568 auto I = HeaderAddrToJITDylib.find(Val: JDHeaderAddr);
569 if (I != HeaderAddrToJITDylib.end())
570 JD = I->second;
571 }
572
573 LLVM_DEBUG({
574 dbgs() << "COFFPlatform::rt_pushInitializers(" << JDHeaderAddr << ") ";
575 if (JD)
576 dbgs() << "pushing initializers for " << JD->getName() << "\n";
577 else
578 dbgs() << "No JITDylib for header address.\n";
579 });
580
581 if (!JD) {
582 SendResult(make_error<StringError>(Args: "No JITDylib with header addr " +
583 formatv(Fmt: "{0:x}", Vals&: JDHeaderAddr),
584 Args: inconvertibleErrorCode()));
585 return;
586 }
587
588 auto JDDepMap = buildJDDepMap(JD&: *JD);
589 if (!JDDepMap) {
590 SendResult(JDDepMap.takeError());
591 return;
592 }
593
594 pushInitializersLoop(SendResult: std::move(SendResult), JD, JDDepMap&: *JDDepMap);
595}
596
597void COFFPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult,
598 ExecutorAddr Handle, StringRef SymbolName) {
599 LLVM_DEBUG(dbgs() << "COFFPlatform::rt_lookupSymbol(\"" << Handle << "\")\n");
600
601 JITDylib *JD = nullptr;
602
603 {
604 std::lock_guard<std::mutex> Lock(PlatformMutex);
605 auto I = HeaderAddrToJITDylib.find(Val: Handle);
606 if (I != HeaderAddrToJITDylib.end())
607 JD = I->second;
608 }
609
610 if (!JD) {
611 LLVM_DEBUG(dbgs() << " No JITDylib for handle " << Handle << "\n");
612 SendResult(make_error<StringError>(Args: "No JITDylib associated with handle " +
613 formatv(Fmt: "{0:x}", Vals&: Handle),
614 Args: inconvertibleErrorCode()));
615 return;
616 }
617
618 // Use functor class to work around XL build compiler issue on AIX.
619 class RtLookupNotifyComplete {
620 public:
621 RtLookupNotifyComplete(SendSymbolAddressFn &&SendResult)
622 : SendResult(std::move(SendResult)) {}
623 void operator()(Expected<SymbolMap> Result) {
624 if (Result) {
625 assert(Result->size() == 1 && "Unexpected result map count");
626 SendResult(Result->begin()->second.getAddress());
627 } else {
628 SendResult(Result.takeError());
629 }
630 }
631
632 private:
633 SendSymbolAddressFn SendResult;
634 };
635
636 ES.lookup(
637 K: LookupKind::DLSym, SearchOrder: {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}},
638 Symbols: SymbolLookupSet(ES.intern(SymName: SymbolName)), RequiredState: SymbolState::Ready,
639 NotifyComplete: RtLookupNotifyComplete(std::move(SendResult)), RegisterDependencies: NoDependenciesToRegister);
640}
641
642Error COFFPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) {
643 ExecutionSession::JITDispatchHandlerAssociationMap WFs;
644
645 using LookupSymbolSPSSig =
646 SPSExpected<SPSExecutorAddr>(SPSExecutorAddr, SPSString);
647 WFs[ES.intern(SymName: "__orc_rt_coff_symbol_lookup_tag")] =
648 ES.wrapAsyncWithSPS<LookupSymbolSPSSig>(Instance: this,
649 Method: &COFFPlatform::rt_lookupSymbol);
650 using PushInitializersSPSSig =
651 SPSExpected<SPSCOFFJITDylibDepInfoMap>(SPSExecutorAddr);
652 WFs[ES.intern(SymName: "__orc_rt_coff_push_initializers_tag")] =
653 ES.wrapAsyncWithSPS<PushInitializersSPSSig>(
654 Instance: this, Method: &COFFPlatform::rt_pushInitializers);
655
656 return ES.registerJITDispatchHandlers(JD&: PlatformJD, WFs: std::move(WFs));
657}
658
659Error COFFPlatform::runBootstrapInitializers(JDBootstrapState &BState) {
660 llvm::sort(C&: BState.Initializers);
661 if (auto Err =
662 runBootstrapSubsectionInitializers(BState, Start: ".CRT$XIA", End: ".CRT$XIZ"))
663 return Err;
664
665 if (auto Err = runSymbolIfExists(PlatformJD&: *BState.JD, SymbolName: "__run_after_c_init"))
666 return Err;
667
668 if (auto Err =
669 runBootstrapSubsectionInitializers(BState, Start: ".CRT$XCA", End: ".CRT$XCZ"))
670 return Err;
671 return Error::success();
672}
673
674Error COFFPlatform::runBootstrapSubsectionInitializers(JDBootstrapState &BState,
675 StringRef Start,
676 StringRef End) {
677 for (auto &Initializer : BState.Initializers)
678 if (Initializer.first >= Start && Initializer.first <= End &&
679 Initializer.second) {
680 auto Res =
681 ES.getExecutorProcessControl().runAsVoidFunction(VoidFnAddr: Initializer.second);
682 if (!Res)
683 return Res.takeError();
684 }
685 return Error::success();
686}
687
688Error COFFPlatform::bootstrapCOFFRuntime(JITDylib &PlatformJD) {
689 // Lookup of runtime symbols causes the collection of initializers if
690 // it's static linking setting.
691 if (auto Err = lookupAndRecordAddrs(
692 ES, K: LookupKind::Static, SearchOrder: makeJITDylibSearchOrder(JDs: &PlatformJD),
693 Pairs: {
694 {ES.intern(SymName: "__orc_rt_coff_platform_bootstrap"),
695 &orc_rt_coff_platform_bootstrap},
696 {ES.intern(SymName: "__orc_rt_coff_platform_shutdown"),
697 &orc_rt_coff_platform_shutdown},
698 {ES.intern(SymName: "__orc_rt_coff_register_jitdylib"),
699 &orc_rt_coff_register_jitdylib},
700 {ES.intern(SymName: "__orc_rt_coff_deregister_jitdylib"),
701 &orc_rt_coff_deregister_jitdylib},
702 {ES.intern(SymName: "__orc_rt_coff_register_object_sections"),
703 &orc_rt_coff_register_object_sections},
704 {ES.intern(SymName: "__orc_rt_coff_deregister_object_sections"),
705 &orc_rt_coff_deregister_object_sections},
706 }))
707 return Err;
708
709 // Call bootstrap functions
710 if (auto Err = ES.callSPSWrapper<void()>(WrapperFnAddr: orc_rt_coff_platform_bootstrap))
711 return Err;
712
713 // Do the pending jitdylib registration actions that we couldn't do
714 // because orc runtime was not linked fully.
715 for (auto KV : JDBootstrapStates) {
716 auto &JDBState = KV.second;
717 if (auto Err = ES.callSPSWrapper<void(SPSString, SPSExecutorAddr)>(
718 WrapperFnAddr: orc_rt_coff_register_jitdylib, WrapperCallArgs&: JDBState.JDName,
719 WrapperCallArgs&: JDBState.HeaderAddr))
720 return Err;
721
722 for (auto &ObjSectionMap : JDBState.ObjectSectionsMaps)
723 if (auto Err = ES.callSPSWrapper<void(SPSExecutorAddr,
724 SPSCOFFObjectSectionsMap, bool)>(
725 WrapperFnAddr: orc_rt_coff_register_object_sections, WrapperCallArgs&: JDBState.HeaderAddr,
726 WrapperCallArgs&: ObjSectionMap, WrapperCallArgs: false))
727 return Err;
728 }
729
730 // Run static initializers collected in bootstrap stage.
731 for (auto KV : JDBootstrapStates) {
732 auto &JDBState = KV.second;
733 if (auto Err = runBootstrapInitializers(BState&: JDBState))
734 return Err;
735 }
736
737 return Error::success();
738}
739
740Error COFFPlatform::runSymbolIfExists(JITDylib &PlatformJD,
741 StringRef SymbolName) {
742 ExecutorAddr jit_function;
743 auto AfterCLookupErr = lookupAndRecordAddrs(
744 ES, K: LookupKind::Static, SearchOrder: makeJITDylibSearchOrder(JDs: &PlatformJD),
745 Pairs: {{ES.intern(SymName: SymbolName), &jit_function}});
746 if (!AfterCLookupErr) {
747 auto Res = ES.getExecutorProcessControl().runAsVoidFunction(VoidFnAddr: jit_function);
748 if (!Res)
749 return Res.takeError();
750 return Error::success();
751 }
752 if (!AfterCLookupErr.isA<SymbolsNotFound>())
753 return AfterCLookupErr;
754 consumeError(Err: std::move(AfterCLookupErr));
755 return Error::success();
756}
757
758void COFFPlatform::COFFPlatformPlugin::modifyPassConfig(
759 MaterializationResponsibility &MR, jitlink::LinkGraph &LG,
760 jitlink::PassConfiguration &Config) {
761
762 bool IsBootstrapping = CP.Bootstrapping.load();
763
764 if (auto InitSymbol = MR.getInitializerSymbol()) {
765 if (InitSymbol == CP.COFFHeaderStartSymbol) {
766 Config.PostAllocationPasses.push_back(
767 x: [this, &MR, IsBootstrapping](jitlink::LinkGraph &G) {
768 return associateJITDylibHeaderSymbol(G, MR, Bootstrap: IsBootstrapping);
769 });
770 return;
771 }
772 Config.PrePrunePasses.push_back(x: [this, &MR](jitlink::LinkGraph &G) {
773 return preserveInitializerSections(G, MR);
774 });
775 }
776
777 if (!IsBootstrapping)
778 Config.PostFixupPasses.push_back(
779 x: [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
780 return registerObjectPlatformSections(G, JD);
781 });
782 else
783 Config.PostFixupPasses.push_back(
784 x: [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
785 return registerObjectPlatformSectionsInBootstrap(G, JD);
786 });
787}
788
789ObjectLinkingLayer::Plugin::SyntheticSymbolDependenciesMap
790COFFPlatform::COFFPlatformPlugin::getSyntheticSymbolDependencies(
791 MaterializationResponsibility &MR) {
792 std::lock_guard<std::mutex> Lock(PluginMutex);
793 auto I = InitSymbolDeps.find(Val: &MR);
794 if (I != InitSymbolDeps.end()) {
795 SyntheticSymbolDependenciesMap Result;
796 Result[MR.getInitializerSymbol()] = std::move(I->second);
797 InitSymbolDeps.erase(Val: &MR);
798 return Result;
799 }
800 return SyntheticSymbolDependenciesMap();
801}
802
803Error COFFPlatform::COFFPlatformPlugin::associateJITDylibHeaderSymbol(
804 jitlink::LinkGraph &G, MaterializationResponsibility &MR,
805 bool IsBootstraping) {
806 auto I = llvm::find_if(Range: G.defined_symbols(), P: [this](jitlink::Symbol *Sym) {
807 return Sym->getName() == *CP.COFFHeaderStartSymbol;
808 });
809 assert(I != G.defined_symbols().end() && "Missing COFF header start symbol");
810
811 auto &JD = MR.getTargetJITDylib();
812 std::lock_guard<std::mutex> Lock(CP.PlatformMutex);
813 auto HeaderAddr = (*I)->getAddress();
814 CP.JITDylibToHeaderAddr[&JD] = HeaderAddr;
815 CP.HeaderAddrToJITDylib[HeaderAddr] = &JD;
816 if (!IsBootstraping) {
817 G.allocActions().push_back(
818 x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<
819 SPSArgList<SPSString, SPSExecutorAddr>>(
820 FnAddr: CP.orc_rt_coff_register_jitdylib, Args: JD.getName(), Args: HeaderAddr)),
821 .Dealloc: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
822 FnAddr: CP.orc_rt_coff_deregister_jitdylib, Args: HeaderAddr))});
823 } else {
824 G.allocActions().push_back(
825 x: {.Finalize: {},
826 .Dealloc: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
827 FnAddr: CP.orc_rt_coff_deregister_jitdylib, Args: HeaderAddr))});
828 JDBootstrapState BState;
829 BState.JD = &JD;
830 BState.JDName = JD.getName();
831 BState.HeaderAddr = HeaderAddr;
832 CP.JDBootstrapStates.emplace(args: &JD, args&: BState);
833 }
834
835 return Error::success();
836}
837
838Error COFFPlatform::COFFPlatformPlugin::registerObjectPlatformSections(
839 jitlink::LinkGraph &G, JITDylib &JD) {
840 COFFObjectSectionsMap ObjSecs;
841 auto HeaderAddr = CP.JITDylibToHeaderAddr[&JD];
842 assert(HeaderAddr && "Must be registered jitdylib");
843 for (auto &S : G.sections()) {
844 jitlink::SectionRange Range(S);
845 if (Range.getSize())
846 ObjSecs.push_back(Elt: std::make_pair(x: S.getName().str(), y: Range.getRange()));
847 }
848
849 G.allocActions().push_back(
850 x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSCOFFRegisterObjectSectionsArgs>(
851 FnAddr: CP.orc_rt_coff_register_object_sections, Args: HeaderAddr, Args: ObjSecs, Args: true)),
852 .Dealloc: cantFail(
853 ValOrErr: WrapperFunctionCall::Create<SPSCOFFDeregisterObjectSectionsArgs>(
854 FnAddr: CP.orc_rt_coff_deregister_object_sections, Args: HeaderAddr,
855 Args: ObjSecs))});
856
857 return Error::success();
858}
859
860Error COFFPlatform::COFFPlatformPlugin::preserveInitializerSections(
861 jitlink::LinkGraph &G, MaterializationResponsibility &MR) {
862 JITLinkSymbolSet InitSectionSymbols;
863 for (auto &Sec : G.sections())
864 if (isCOFFInitializerSection(Name: Sec.getName()))
865 for (auto *B : Sec.blocks())
866 if (!B->edges_empty())
867 InitSectionSymbols.insert(
868 V: &G.addAnonymousSymbol(Content&: *B, Offset: 0, Size: 0, IsCallable: false, IsLive: true));
869
870 std::lock_guard<std::mutex> Lock(PluginMutex);
871 InitSymbolDeps[&MR] = InitSectionSymbols;
872 return Error::success();
873}
874
875Error COFFPlatform::COFFPlatformPlugin::
876 registerObjectPlatformSectionsInBootstrap(jitlink::LinkGraph &G,
877 JITDylib &JD) {
878 std::lock_guard<std::mutex> Lock(CP.PlatformMutex);
879 auto HeaderAddr = CP.JITDylibToHeaderAddr[&JD];
880 COFFObjectSectionsMap ObjSecs;
881 for (auto &S : G.sections()) {
882 jitlink::SectionRange Range(S);
883 if (Range.getSize())
884 ObjSecs.push_back(Elt: std::make_pair(x: S.getName().str(), y: Range.getRange()));
885 }
886
887 G.allocActions().push_back(
888 x: {.Finalize: {},
889 .Dealloc: cantFail(
890 ValOrErr: WrapperFunctionCall::Create<SPSCOFFDeregisterObjectSectionsArgs>(
891 FnAddr: CP.orc_rt_coff_deregister_object_sections, Args: HeaderAddr,
892 Args: ObjSecs))});
893
894 auto &BState = CP.JDBootstrapStates[&JD];
895 BState.ObjectSectionsMaps.push_back(x: std::move(ObjSecs));
896
897 // Collect static initializers
898 for (auto &S : G.sections())
899 if (isCOFFInitializerSection(Name: S.getName()))
900 for (auto *B : S.blocks()) {
901 if (B->edges_empty())
902 continue;
903 for (auto &E : B->edges())
904 BState.Initializers.push_back(Elt: std::make_pair(
905 x: S.getName().str(), y: E.getTarget().getAddress() + E.getAddend()));
906 }
907
908 return Error::success();
909}
910
911} // End namespace orc.
912} // End namespace llvm.
913