1//===------- COFFPlatform.cpp - Utilities for executing COFF in Orc -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/Orc/COFFPlatform.h"
10
11#include "llvm/ExecutionEngine/Orc/AbsoluteSymbols.h"
12#include "llvm/ExecutionEngine/Orc/COFF.h"
13#include "llvm/ExecutionEngine/Orc/DebugUtils.h"
14#include "llvm/ExecutionEngine/Orc/LookupAndRecordAddrs.h"
15#include "llvm/ExecutionEngine/Orc/ObjectFileInterface.h"
16#include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h"
17
18#include "llvm/Object/COFF.h"
19
20#include "llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h"
21
22#include "llvm/ExecutionEngine/JITLink/x86_64.h"
23
24#define DEBUG_TYPE "orc"
25
26using namespace llvm;
27using namespace llvm::orc;
28using namespace llvm::orc::shared;
29
30namespace llvm {
31namespace orc {
32namespace shared {
33
34using SPSCOFFJITDylibDepInfo = SPSSequence<SPSExecutorAddr>;
35using SPSCOFFJITDylibDepInfoMap =
36 SPSSequence<SPSTuple<SPSExecutorAddr, SPSCOFFJITDylibDepInfo>>;
37using SPSCOFFObjectSectionsMap =
38 SPSSequence<SPSTuple<SPSString, SPSExecutorAddrRange>>;
39using SPSCOFFRegisterObjectSectionsArgs =
40 SPSArgList<SPSExecutorAddr, SPSCOFFObjectSectionsMap, bool>;
41using SPSCOFFDeregisterObjectSectionsArgs =
42 SPSArgList<SPSExecutorAddr, SPSCOFFObjectSectionsMap>;
43
44} // namespace shared
45} // namespace orc
46} // namespace llvm
47namespace {
48
49class COFFHeaderMaterializationUnit : public MaterializationUnit {
50public:
51 COFFHeaderMaterializationUnit(COFFPlatform &CP,
52 const SymbolStringPtr &HeaderStartSymbol)
53 : MaterializationUnit(createHeaderInterface(MOP&: CP, HeaderStartSymbol)),
54 CP(CP) {}
55
56 StringRef getName() const override { return "COFFHeaderMU"; }
57
58 void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
59 auto G = std::make_unique<jitlink::LinkGraph>(
60 args: "<COFFHeaderMU>", args: CP.getExecutionSession().getSymbolStringPool(),
61 args: CP.getExecutionSession().getTargetTriple(), args: SubtargetFeatures(),
62 args&: jitlink::getGenericEdgeKindName);
63 auto &HeaderSection = G->createSection(Name: "__header", Prot: MemProt::Read);
64 auto &HeaderBlock = createHeaderBlock(G&: *G, HeaderSection);
65
66 // Init symbol is __ImageBase symbol.
67 auto &ImageBaseSymbol = G->addDefinedSymbol(
68 Content&: HeaderBlock, Offset: 0, Name: *R->getInitializerSymbol(), Size: HeaderBlock.getSize(),
69 L: jitlink::Linkage::Strong, S: jitlink::Scope::Default, IsCallable: false, IsLive: true);
70
71 addImageBaseRelocationEdge(B&: HeaderBlock, ImageBase&: ImageBaseSymbol);
72
73 CP.getObjectLinkingLayer().emit(R: std::move(R), G: std::move(G));
74 }
75
76 void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
77
78private:
79 struct HeaderSymbol {
80 const char *Name;
81 uint64_t Offset;
82 };
83
84 struct NTHeader {
85 support::ulittle32_t PEMagic;
86 object::coff_file_header FileHeader;
87 struct PEHeader {
88 object::pe32plus_header Header;
89 object::data_directory DataDirectory[COFF::NUM_DATA_DIRECTORIES + 1];
90 } OptionalHeader;
91 };
92
93 struct HeaderBlockContent {
94 object::dos_header DOSHeader;
95 COFFHeaderMaterializationUnit::NTHeader NTHeader;
96 };
97
98 static jitlink::Block &createHeaderBlock(jitlink::LinkGraph &G,
99 jitlink::Section &HeaderSection) {
100 HeaderBlockContent Hdr = {};
101
102 // Set up magic
103 Hdr.DOSHeader.Magic[0] = 'M';
104 Hdr.DOSHeader.Magic[1] = 'Z';
105 Hdr.DOSHeader.AddressOfNewExeHeader =
106 offsetof(HeaderBlockContent, NTHeader);
107 uint32_t PEMagic = *reinterpret_cast<const uint32_t *>(COFF::PEMagic);
108 Hdr.NTHeader.PEMagic = PEMagic;
109 Hdr.NTHeader.OptionalHeader.Header.Magic = COFF::PE32Header::PE32_PLUS;
110
111 switch (G.getTargetTriple().getArch()) {
112 case Triple::x86_64:
113 Hdr.NTHeader.FileHeader.Machine = COFF::IMAGE_FILE_MACHINE_AMD64;
114 break;
115 default:
116 llvm_unreachable("Unrecognized architecture");
117 }
118
119 auto HeaderContent = G.allocateContent(
120 Source: ArrayRef<char>(reinterpret_cast<const char *>(&Hdr), sizeof(Hdr)));
121
122 return G.createContentBlock(Parent&: HeaderSection, Content: HeaderContent, Address: ExecutorAddr(), Alignment: 8,
123 AlignmentOffset: 0);
124 }
125
126 static void addImageBaseRelocationEdge(jitlink::Block &B,
127 jitlink::Symbol &ImageBase) {
128 auto ImageBaseOffset = offsetof(HeaderBlockContent, NTHeader) +
129 offsetof(NTHeader, OptionalHeader) +
130 offsetof(object::pe32plus_header, ImageBase);
131 B.addEdge(K: jitlink::x86_64::Pointer64, Offset: ImageBaseOffset, Target&: ImageBase, Addend: 0);
132 }
133
134 static MaterializationUnit::Interface
135 createHeaderInterface(COFFPlatform &MOP,
136 const SymbolStringPtr &HeaderStartSymbol) {
137 SymbolFlagsMap HeaderSymbolFlags;
138
139 HeaderSymbolFlags[HeaderStartSymbol] = JITSymbolFlags::Exported;
140
141 return MaterializationUnit::Interface(std::move(HeaderSymbolFlags),
142 HeaderStartSymbol);
143 }
144
145 COFFPlatform &CP;
146};
147
148} // end anonymous namespace
149
150namespace llvm {
151namespace orc {
152
153Expected<std::unique_ptr<COFFPlatform>>
154COFFPlatform::Create(ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD,
155 std::unique_ptr<MemoryBuffer> OrcRuntimeArchiveBuffer,
156 LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime,
157 const char *VCRuntimePath,
158 std::optional<SymbolAliasMap> RuntimeAliases) {
159
160 auto &ES = ObjLinkingLayer.getExecutionSession();
161
162 // If the target is not supported then bail out immediately.
163 if (!supportedTarget(TT: ES.getTargetTriple()))
164 return make_error<StringError>(Args: "Unsupported COFFPlatform triple: " +
165 ES.getTargetTriple().str(),
166 Args: inconvertibleErrorCode());
167
168 auto &EPC = ES.getExecutorProcessControl();
169
170 auto GeneratorArchive =
171 object::Archive::create(Source: OrcRuntimeArchiveBuffer->getMemBufferRef());
172 if (!GeneratorArchive)
173 return GeneratorArchive.takeError();
174
175 std::set<std::string> DylibsToPreload;
176 auto OrcRuntimeArchiveGenerator = StaticLibraryDefinitionGenerator::Create(
177 L&: ObjLinkingLayer, ArchiveBuffer: nullptr, Archive: std::move(*GeneratorArchive),
178 VisitMembers: COFFImportFileScanner(DylibsToPreload));
179 if (!OrcRuntimeArchiveGenerator)
180 return OrcRuntimeArchiveGenerator.takeError();
181
182 // We need a second instance of the archive (for now) for the Platform. We
183 // can `cantFail` this call, since if it were going to fail it would have
184 // failed above.
185 auto RuntimeArchive = cantFail(
186 ValOrErr: object::Archive::create(Source: OrcRuntimeArchiveBuffer->getMemBufferRef()));
187
188 // Create default aliases if the caller didn't supply any.
189 if (!RuntimeAliases)
190 RuntimeAliases = standardPlatformAliases(ES);
191
192 // Define the aliases.
193 if (auto Err = PlatformJD.define(MU: symbolAliases(Aliases: std::move(*RuntimeAliases))))
194 return std::move(Err);
195
196 auto &HostFuncJD = ES.createBareJITDylib(Name: "$<PlatformRuntimeHostFuncJD>");
197
198 // Add JIT-dispatch function support symbols.
199 if (auto Err = HostFuncJD.define(
200 MU: absoluteSymbols(Symbols: {{ES.intern(SymName: "__orc_rt_jit_dispatch"),
201 {EPC.getJITDispatchInfo().JITDispatchFunction,
202 JITSymbolFlags::Exported}},
203 {ES.intern(SymName: "__orc_rt_jit_dispatch_ctx"),
204 {EPC.getJITDispatchInfo().JITDispatchContext,
205 JITSymbolFlags::Exported}}})))
206 return std::move(Err);
207
208 PlatformJD.addToLinkOrder(JD&: HostFuncJD);
209
210 // Create the instance.
211 Error Err = Error::success();
212 auto P = std::unique_ptr<COFFPlatform>(new COFFPlatform(
213 ObjLinkingLayer, PlatformJD, std::move(*OrcRuntimeArchiveGenerator),
214 std::move(DylibsToPreload), std::move(OrcRuntimeArchiveBuffer),
215 std::move(RuntimeArchive), std::move(LoadDynLibrary), StaticVCRuntime,
216 VCRuntimePath, Err));
217 if (Err)
218 return std::move(Err);
219 return std::move(P);
220}
221
222Expected<std::unique_ptr<COFFPlatform>>
223COFFPlatform::Create(ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD,
224 const char *OrcRuntimePath,
225 LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime,
226 const char *VCRuntimePath,
227 std::optional<SymbolAliasMap> RuntimeAliases) {
228
229 auto ArchiveBuffer = MemoryBuffer::getFile(Filename: OrcRuntimePath);
230 if (!ArchiveBuffer)
231 return createFileError(F: OrcRuntimePath, EC: ArchiveBuffer.getError());
232
233 return Create(ObjLinkingLayer, PlatformJD, OrcRuntimeArchiveBuffer: std::move(*ArchiveBuffer),
234 LoadDynLibrary: std::move(LoadDynLibrary), StaticVCRuntime, VCRuntimePath,
235 RuntimeAliases: std::move(RuntimeAliases));
236}
237
238Expected<MemoryBufferRef> COFFPlatform::getPerJDObjectFile() {
239 auto PerJDObj = OrcRuntimeArchive->findSym(name: "__orc_rt_coff_per_jd_marker");
240 if (!PerJDObj)
241 return PerJDObj.takeError();
242
243 if (!*PerJDObj)
244 return make_error<StringError>(Args: "Could not find per jd object file",
245 Args: inconvertibleErrorCode());
246
247 auto Buffer = (*PerJDObj)->getAsBinary();
248 if (!Buffer)
249 return Buffer.takeError();
250
251 return (*Buffer)->getMemoryBufferRef();
252}
253
254static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases,
255 ArrayRef<std::pair<const char *, const char *>> AL) {
256 for (auto &KV : AL) {
257 auto AliasName = ES.intern(SymName: KV.first);
258 assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map");
259 Aliases[std::move(AliasName)] = {ES.intern(SymName: KV.second),
260 JITSymbolFlags::Exported};
261 }
262}
263
264Error COFFPlatform::setupJITDylib(JITDylib &JD) {
265 if (auto Err = JD.define(MU: std::make_unique<COFFHeaderMaterializationUnit>(
266 args&: *this, args&: COFFHeaderStartSymbol)))
267 return Err;
268
269 if (auto Err = ES.lookup(SearchOrder: {&JD}, Symbol: COFFHeaderStartSymbol).takeError())
270 return Err;
271
272 // Define the CXX aliases.
273 SymbolAliasMap CXXAliases;
274 addAliases(ES, Aliases&: CXXAliases, AL: requiredCXXAliases());
275 if (auto Err = JD.define(MU: symbolAliases(Aliases: std::move(CXXAliases))))
276 return Err;
277
278 auto PerJDObj = getPerJDObjectFile();
279 if (!PerJDObj)
280 return PerJDObj.takeError();
281
282 auto I = getObjectFileInterface(ES, ObjBuffer: *PerJDObj);
283 if (!I)
284 return I.takeError();
285
286 if (auto Err = ObjLinkingLayer.add(
287 JD, O: MemoryBuffer::getMemBuffer(Ref: *PerJDObj, RequiresNullTerminator: false), I: std::move(*I)))
288 return Err;
289
290 if (!Bootstrapping) {
291 auto ImportedLibs = StaticVCRuntime
292 ? VCRuntimeBootstrap->loadStaticVCRuntime(JD)
293 : VCRuntimeBootstrap->loadDynamicVCRuntime(JD);
294 if (!ImportedLibs)
295 return ImportedLibs.takeError();
296 for (auto &Lib : *ImportedLibs)
297 if (auto Err = LoadDynLibrary(JD, Lib))
298 return Err;
299 if (StaticVCRuntime)
300 if (auto Err = VCRuntimeBootstrap->initializeStaticVCRuntime(JD))
301 return Err;
302 }
303
304 JD.addGenerator(DefGenerator: DLLImportDefinitionGenerator::Create(ES, L&: ObjLinkingLayer));
305 return Error::success();
306}
307
308Error COFFPlatform::teardownJITDylib(JITDylib &JD) {
309 std::lock_guard<std::mutex> Lock(PlatformMutex);
310 auto I = JITDylibToHeaderAddr.find(Val: &JD);
311 if (I != JITDylibToHeaderAddr.end()) {
312 assert(HeaderAddrToJITDylib.count(I->second) &&
313 "HeaderAddrToJITDylib missing entry");
314 HeaderAddrToJITDylib.erase(Val: I->second);
315 JITDylibToHeaderAddr.erase(I);
316 }
317 return Error::success();
318}
319
320Error COFFPlatform::notifyAdding(ResourceTracker &RT,
321 const MaterializationUnit &MU) {
322 auto &JD = RT.getJITDylib();
323 const auto &InitSym = MU.getInitializerSymbol();
324 if (!InitSym)
325 return Error::success();
326
327 RegisteredInitSymbols[&JD].add(Name: InitSym,
328 Flags: SymbolLookupFlags::WeaklyReferencedSymbol);
329
330 LLVM_DEBUG({
331 dbgs() << "COFFPlatform: Registered init symbol " << *InitSym << " for MU "
332 << MU.getName() << "\n";
333 });
334 return Error::success();
335}
336
337Error COFFPlatform::notifyRemoving(ResourceTracker &RT) {
338 llvm_unreachable("Not supported yet");
339}
340
341SymbolAliasMap COFFPlatform::standardPlatformAliases(ExecutionSession &ES) {
342 SymbolAliasMap Aliases;
343 addAliases(ES, Aliases, AL: standardRuntimeUtilityAliases());
344 return Aliases;
345}
346
347ArrayRef<std::pair<const char *, const char *>>
348COFFPlatform::requiredCXXAliases() {
349 static const std::pair<const char *, const char *> RequiredCXXAliases[] = {
350 {"_CxxThrowException", "__orc_rt_coff_cxx_throw_exception"},
351 {"_onexit", "__orc_rt_coff_onexit_per_jd"},
352 {"atexit", "__orc_rt_coff_atexit_per_jd"}};
353
354 return ArrayRef<std::pair<const char *, const char *>>(RequiredCXXAliases);
355}
356
357ArrayRef<std::pair<const char *, const char *>>
358COFFPlatform::standardRuntimeUtilityAliases() {
359 static const std::pair<const char *, const char *>
360 StandardRuntimeUtilityAliases[] = {
361 {"__orc_rt_run_program", "__orc_rt_coff_run_program"},
362 {"__orc_rt_jit_dlerror", "__orc_rt_coff_jit_dlerror"},
363 {"__orc_rt_jit_dlopen", "__orc_rt_coff_jit_dlopen"},
364 {"__orc_rt_jit_dlclose", "__orc_rt_coff_jit_dlclose"},
365 {"__orc_rt_jit_dlsym", "__orc_rt_coff_jit_dlsym"},
366 {"__orc_rt_log_error", "__orc_rt_log_error_to_stderr"}};
367
368 return ArrayRef<std::pair<const char *, const char *>>(
369 StandardRuntimeUtilityAliases);
370}
371
372bool COFFPlatform::supportedTarget(const Triple &TT) {
373 switch (TT.getArch()) {
374 case Triple::x86_64:
375 return true;
376 default:
377 return false;
378 }
379}
380
381COFFPlatform::COFFPlatform(
382 ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD,
383 std::unique_ptr<StaticLibraryDefinitionGenerator> OrcRuntimeGenerator,
384 std::set<std::string> DylibsToPreload,
385 std::unique_ptr<MemoryBuffer> OrcRuntimeArchiveBuffer,
386 std::unique_ptr<object::Archive> OrcRuntimeArchive,
387 LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime,
388 const char *VCRuntimePath, Error &Err)
389 : ES(ObjLinkingLayer.getExecutionSession()),
390 ObjLinkingLayer(ObjLinkingLayer),
391 LoadDynLibrary(std::move(LoadDynLibrary)),
392 OrcRuntimeArchiveBuffer(std::move(OrcRuntimeArchiveBuffer)),
393 OrcRuntimeArchive(std::move(OrcRuntimeArchive)),
394 StaticVCRuntime(StaticVCRuntime),
395 COFFHeaderStartSymbol(ES.intern(SymName: "__ImageBase")) {
396 ErrorAsOutParameter _(Err);
397
398 Bootstrapping.store(i: true);
399 ObjLinkingLayer.addPlugin(P: std::make_unique<COFFPlatformPlugin>(args&: *this));
400
401 // Load vc runtime
402 auto VCRT =
403 COFFVCRuntimeBootstrapper::Create(ES, ObjLinkingLayer, RuntimePath: VCRuntimePath);
404 if (!VCRT) {
405 Err = VCRT.takeError();
406 return;
407 }
408 VCRuntimeBootstrap = std::move(*VCRT);
409
410 auto ImportedLibs =
411 StaticVCRuntime ? VCRuntimeBootstrap->loadStaticVCRuntime(JD&: PlatformJD)
412 : VCRuntimeBootstrap->loadDynamicVCRuntime(JD&: PlatformJD);
413 if (!ImportedLibs) {
414 Err = ImportedLibs.takeError();
415 return;
416 }
417
418 for (auto &Lib : *ImportedLibs)
419 DylibsToPreload.insert(x: Lib);
420
421 PlatformJD.addGenerator(DefGenerator: std::move(OrcRuntimeGenerator));
422
423 // PlatformJD hasn't been set up by the platform yet (since we're creating
424 // the platform now), so set it up.
425 if (auto E2 = setupJITDylib(PlatformJD)) {
426 Err = std::move(E2);
427 return;
428 }
429
430 for (auto& Lib : DylibsToPreload)
431 if (auto E2 = this->LoadDynLibrary(PlatformJD, Lib)) {
432 Err = std::move(E2);
433 return;
434 }
435
436 if (StaticVCRuntime)
437 if (auto E2 = VCRuntimeBootstrap->initializeStaticVCRuntime(JD&: PlatformJD)) {
438 Err = std::move(E2);
439 return;
440 }
441
442 // Associate wrapper function tags with JIT-side function implementations.
443 if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) {
444 Err = std::move(E2);
445 return;
446 }
447
448 // Lookup addresses of runtime functions callable by the platform,
449 // call the platform bootstrap function to initialize the platform-state
450 // object in the executor.
451 if (auto E2 = bootstrapCOFFRuntime(PlatformJD)) {
452 Err = std::move(E2);
453 return;
454 }
455
456 Bootstrapping.store(i: false);
457 JDBootstrapStates.clear();
458}
459
460Expected<COFFPlatform::JITDylibDepMap>
461COFFPlatform::buildJDDepMap(JITDylib &JD) {
462 return ES.runSessionLocked(F: [&]() -> Expected<JITDylibDepMap> {
463 JITDylibDepMap JDDepMap;
464
465 SmallVector<JITDylib *, 16> Worklist({&JD});
466 while (!Worklist.empty()) {
467 auto CurJD = Worklist.back();
468 Worklist.pop_back();
469
470 auto &DM = JDDepMap[CurJD];
471 CurJD->withLinkOrderDo(F: [&](const JITDylibSearchOrder &O) {
472 DM.reserve(N: O.size());
473 for (auto &KV : O) {
474 if (KV.first == CurJD)
475 continue;
476 {
477 // Bare jitdylibs not known to the platform
478 std::lock_guard<std::mutex> Lock(PlatformMutex);
479 if (!JITDylibToHeaderAddr.count(Val: KV.first)) {
480 LLVM_DEBUG({
481 dbgs() << "JITDylib unregistered to COFFPlatform detected in "
482 "LinkOrder: "
483 << CurJD->getName() << "\n";
484 });
485 continue;
486 }
487 }
488 DM.push_back(Elt: KV.first);
489 // Push unvisited entry.
490 if (JDDepMap.try_emplace(Key: KV.first).second)
491 Worklist.push_back(Elt: KV.first);
492 }
493 });
494 }
495 return std::move(JDDepMap);
496 });
497}
498
499void COFFPlatform::pushInitializersLoop(PushInitializersSendResultFn SendResult,
500 JITDylibSP JD,
501 JITDylibDepMap &JDDepMap) {
502 SmallVector<JITDylib *, 16> Worklist({JD.get()});
503 DenseSet<JITDylib *> Visited({JD.get()});
504 DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols;
505 ES.runSessionLocked(F: [&]() {
506 while (!Worklist.empty()) {
507 auto CurJD = Worklist.back();
508 Worklist.pop_back();
509
510 auto RISItr = RegisteredInitSymbols.find(Val: CurJD);
511 if (RISItr != RegisteredInitSymbols.end()) {
512 NewInitSymbols[CurJD] = std::move(RISItr->second);
513 RegisteredInitSymbols.erase(I: RISItr);
514 }
515
516 for (auto *DepJD : JDDepMap[CurJD])
517 if (Visited.insert(V: DepJD).second)
518 Worklist.push_back(Elt: DepJD);
519 }
520 });
521
522 // If there are no further init symbols to look up then send the link order
523 // (as a list of header addresses) to the caller.
524 if (NewInitSymbols.empty()) {
525 // Build the dep info map to return.
526 COFFJITDylibDepInfoMap DIM;
527 DIM.reserve(n: JDDepMap.size());
528 for (auto &KV : JDDepMap) {
529 std::lock_guard<std::mutex> Lock(PlatformMutex);
530 COFFJITDylibDepInfo DepInfo;
531 DepInfo.reserve(n: KV.second.size());
532 for (auto &Dep : KV.second) {
533 DepInfo.push_back(x: JITDylibToHeaderAddr[Dep]);
534 }
535 auto H = JITDylibToHeaderAddr[KV.first];
536 DIM.push_back(x: std::make_pair(x&: H, y: std::move(DepInfo)));
537 }
538 SendResult(DIM);
539 return;
540 }
541
542 // Otherwise issue a lookup and re-run this phase when it completes.
543 lookupInitSymbolsAsync(
544 OnComplete: [this, SendResult = std::move(SendResult), &JD,
545 JDDepMap = std::move(JDDepMap)](Error Err) mutable {
546 if (Err)
547 SendResult(std::move(Err));
548 else
549 pushInitializersLoop(SendResult: std::move(SendResult), JD, JDDepMap);
550 },
551 ES, InitSyms: std::move(NewInitSymbols));
552}
553
554void COFFPlatform::rt_pushInitializers(PushInitializersSendResultFn SendResult,
555 ExecutorAddr JDHeaderAddr) {
556 JITDylibSP JD;
557 {
558 std::lock_guard<std::mutex> Lock(PlatformMutex);
559 auto I = HeaderAddrToJITDylib.find(Val: JDHeaderAddr);
560 if (I != HeaderAddrToJITDylib.end())
561 JD = I->second;
562 }
563
564 LLVM_DEBUG({
565 dbgs() << "COFFPlatform::rt_pushInitializers(" << JDHeaderAddr << ") ";
566 if (JD)
567 dbgs() << "pushing initializers for " << JD->getName() << "\n";
568 else
569 dbgs() << "No JITDylib for header address.\n";
570 });
571
572 if (!JD) {
573 SendResult(make_error<StringError>(Args: "No JITDylib with header addr " +
574 formatv(Fmt: "{0:x}", Vals&: JDHeaderAddr),
575 Args: inconvertibleErrorCode()));
576 return;
577 }
578
579 auto JDDepMap = buildJDDepMap(JD&: *JD);
580 if (!JDDepMap) {
581 SendResult(JDDepMap.takeError());
582 return;
583 }
584
585 pushInitializersLoop(SendResult: std::move(SendResult), JD, JDDepMap&: *JDDepMap);
586}
587
588void COFFPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult,
589 ExecutorAddr Handle, StringRef SymbolName) {
590 LLVM_DEBUG(dbgs() << "COFFPlatform::rt_lookupSymbol(\"" << Handle << "\")\n");
591
592 JITDylib *JD = nullptr;
593
594 {
595 std::lock_guard<std::mutex> Lock(PlatformMutex);
596 auto I = HeaderAddrToJITDylib.find(Val: Handle);
597 if (I != HeaderAddrToJITDylib.end())
598 JD = I->second;
599 }
600
601 if (!JD) {
602 LLVM_DEBUG(dbgs() << " No JITDylib for handle " << Handle << "\n");
603 SendResult(make_error<StringError>(Args: "No JITDylib associated with handle " +
604 formatv(Fmt: "{0:x}", Vals&: Handle),
605 Args: inconvertibleErrorCode()));
606 return;
607 }
608
609 // Use functor class to work around XL build compiler issue on AIX.
610 class RtLookupNotifyComplete {
611 public:
612 RtLookupNotifyComplete(SendSymbolAddressFn &&SendResult)
613 : SendResult(std::move(SendResult)) {}
614 void operator()(Expected<SymbolMap> Result) {
615 if (Result) {
616 assert(Result->size() == 1 && "Unexpected result map count");
617 SendResult(Result->begin()->second.getAddress());
618 } else {
619 SendResult(Result.takeError());
620 }
621 }
622
623 private:
624 SendSymbolAddressFn SendResult;
625 };
626
627 ES.lookup(
628 K: LookupKind::DLSym, SearchOrder: {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}},
629 Symbols: SymbolLookupSet(ES.intern(SymName: SymbolName)), RequiredState: SymbolState::Ready,
630 NotifyComplete: RtLookupNotifyComplete(std::move(SendResult)), RegisterDependencies: NoDependenciesToRegister);
631}
632
633Error COFFPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) {
634 ExecutionSession::JITDispatchHandlerAssociationMap WFs;
635
636 using LookupSymbolSPSSig =
637 SPSExpected<SPSExecutorAddr>(SPSExecutorAddr, SPSString);
638 WFs[ES.intern(SymName: "__orc_rt_coff_symbol_lookup_tag")] =
639 ES.wrapAsyncWithSPS<LookupSymbolSPSSig>(Instance: this,
640 Method: &COFFPlatform::rt_lookupSymbol);
641 using PushInitializersSPSSig =
642 SPSExpected<SPSCOFFJITDylibDepInfoMap>(SPSExecutorAddr);
643 WFs[ES.intern(SymName: "__orc_rt_coff_push_initializers_tag")] =
644 ES.wrapAsyncWithSPS<PushInitializersSPSSig>(
645 Instance: this, Method: &COFFPlatform::rt_pushInitializers);
646
647 return ES.registerJITDispatchHandlers(JD&: PlatformJD, WFs: std::move(WFs));
648}
649
650Error COFFPlatform::runBootstrapInitializers(JDBootstrapState &BState) {
651 llvm::sort(C&: BState.Initializers);
652 if (auto Err =
653 runBootstrapSubsectionInitializers(BState, Start: ".CRT$XIA", End: ".CRT$XIZ"))
654 return Err;
655
656 if (auto Err = runSymbolIfExists(PlatformJD&: *BState.JD, SymbolName: "__run_after_c_init"))
657 return Err;
658
659 if (auto Err =
660 runBootstrapSubsectionInitializers(BState, Start: ".CRT$XCA", End: ".CRT$XCZ"))
661 return Err;
662 return Error::success();
663}
664
665Error COFFPlatform::runBootstrapSubsectionInitializers(JDBootstrapState &BState,
666 StringRef Start,
667 StringRef End) {
668 for (auto &Initializer : BState.Initializers)
669 if (Initializer.first >= Start && Initializer.first <= End &&
670 Initializer.second) {
671 auto Res =
672 ES.getExecutorProcessControl().runAsVoidFunction(VoidFnAddr: Initializer.second);
673 if (!Res)
674 return Res.takeError();
675 }
676 return Error::success();
677}
678
679Error COFFPlatform::bootstrapCOFFRuntime(JITDylib &PlatformJD) {
680 // Lookup of runtime symbols causes the collection of initializers if
681 // it's static linking setting.
682 if (auto Err = lookupAndRecordAddrs(
683 ES, K: LookupKind::Static, SearchOrder: makeJITDylibSearchOrder(JDs: &PlatformJD),
684 Pairs: {
685 {ES.intern(SymName: "__orc_rt_coff_platform_bootstrap"),
686 &orc_rt_coff_platform_bootstrap},
687 {ES.intern(SymName: "__orc_rt_coff_platform_shutdown"),
688 &orc_rt_coff_platform_shutdown},
689 {ES.intern(SymName: "__orc_rt_coff_register_jitdylib"),
690 &orc_rt_coff_register_jitdylib},
691 {ES.intern(SymName: "__orc_rt_coff_deregister_jitdylib"),
692 &orc_rt_coff_deregister_jitdylib},
693 {ES.intern(SymName: "__orc_rt_coff_register_object_sections"),
694 &orc_rt_coff_register_object_sections},
695 {ES.intern(SymName: "__orc_rt_coff_deregister_object_sections"),
696 &orc_rt_coff_deregister_object_sections},
697 }))
698 return Err;
699
700 // Call bootstrap functions
701 if (auto Err = ES.callSPSWrapper<void()>(WrapperFnAddr: orc_rt_coff_platform_bootstrap))
702 return Err;
703
704 // Do the pending jitdylib registration actions that we couldn't do
705 // because orc runtime was not linked fully.
706 for (auto KV : JDBootstrapStates) {
707 auto &JDBState = KV.second;
708 if (auto Err = ES.callSPSWrapper<void(SPSString, SPSExecutorAddr)>(
709 WrapperFnAddr: orc_rt_coff_register_jitdylib, WrapperCallArgs&: JDBState.JDName,
710 WrapperCallArgs&: JDBState.HeaderAddr))
711 return Err;
712
713 for (auto &ObjSectionMap : JDBState.ObjectSectionsMaps)
714 if (auto Err = ES.callSPSWrapper<void(SPSExecutorAddr,
715 SPSCOFFObjectSectionsMap, bool)>(
716 WrapperFnAddr: orc_rt_coff_register_object_sections, WrapperCallArgs&: JDBState.HeaderAddr,
717 WrapperCallArgs&: ObjSectionMap, WrapperCallArgs: false))
718 return Err;
719 }
720
721 // Run static initializers collected in bootstrap stage.
722 for (auto KV : JDBootstrapStates) {
723 auto &JDBState = KV.second;
724 if (auto Err = runBootstrapInitializers(BState&: JDBState))
725 return Err;
726 }
727
728 return Error::success();
729}
730
731Error COFFPlatform::runSymbolIfExists(JITDylib &PlatformJD,
732 StringRef SymbolName) {
733 ExecutorAddr jit_function;
734 auto AfterCLookupErr = lookupAndRecordAddrs(
735 ES, K: LookupKind::Static, SearchOrder: makeJITDylibSearchOrder(JDs: &PlatformJD),
736 Pairs: {{ES.intern(SymName: SymbolName), &jit_function}});
737 if (!AfterCLookupErr) {
738 auto Res = ES.getExecutorProcessControl().runAsVoidFunction(VoidFnAddr: jit_function);
739 if (!Res)
740 return Res.takeError();
741 return Error::success();
742 }
743 if (!AfterCLookupErr.isA<SymbolsNotFound>())
744 return AfterCLookupErr;
745 consumeError(Err: std::move(AfterCLookupErr));
746 return Error::success();
747}
748
749void COFFPlatform::COFFPlatformPlugin::modifyPassConfig(
750 MaterializationResponsibility &MR, jitlink::LinkGraph &LG,
751 jitlink::PassConfiguration &Config) {
752
753 bool IsBootstrapping = CP.Bootstrapping.load();
754
755 if (auto InitSymbol = MR.getInitializerSymbol()) {
756 if (InitSymbol == CP.COFFHeaderStartSymbol) {
757 Config.PostAllocationPasses.push_back(
758 x: [this, &MR, IsBootstrapping](jitlink::LinkGraph &G) {
759 return associateJITDylibHeaderSymbol(G, MR, Bootstrap: IsBootstrapping);
760 });
761 return;
762 }
763 Config.PrePrunePasses.push_back(x: [this, &MR](jitlink::LinkGraph &G) {
764 return preserveInitializerSections(G, MR);
765 });
766 }
767
768 if (!IsBootstrapping)
769 Config.PostFixupPasses.push_back(
770 x: [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
771 return registerObjectPlatformSections(G, JD);
772 });
773 else
774 Config.PostFixupPasses.push_back(
775 x: [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
776 return registerObjectPlatformSectionsInBootstrap(G, JD);
777 });
778}
779
780Error COFFPlatform::COFFPlatformPlugin::associateJITDylibHeaderSymbol(
781 jitlink::LinkGraph &G, MaterializationResponsibility &MR,
782 bool IsBootstraping) {
783 auto I = llvm::find_if(Range: G.defined_symbols(), P: [this](jitlink::Symbol *Sym) {
784 return *Sym->getName() == *CP.COFFHeaderStartSymbol;
785 });
786 assert(I != G.defined_symbols().end() && "Missing COFF header start symbol");
787
788 auto &JD = MR.getTargetJITDylib();
789 std::lock_guard<std::mutex> Lock(CP.PlatformMutex);
790 auto HeaderAddr = (*I)->getAddress();
791 CP.JITDylibToHeaderAddr[&JD] = HeaderAddr;
792 CP.HeaderAddrToJITDylib[HeaderAddr] = &JD;
793 if (!IsBootstraping) {
794 G.allocActions().push_back(
795 x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<
796 SPSArgList<SPSString, SPSExecutorAddr>>(
797 FnAddr: CP.orc_rt_coff_register_jitdylib, Args: JD.getName(), Args: HeaderAddr)),
798 .Dealloc: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
799 FnAddr: CP.orc_rt_coff_deregister_jitdylib, Args: HeaderAddr))});
800 } else {
801 G.allocActions().push_back(
802 x: {.Finalize: {},
803 .Dealloc: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
804 FnAddr: CP.orc_rt_coff_deregister_jitdylib, Args: HeaderAddr))});
805 JDBootstrapState BState;
806 BState.JD = &JD;
807 BState.JDName = JD.getName();
808 BState.HeaderAddr = HeaderAddr;
809 CP.JDBootstrapStates.emplace(args: &JD, args&: BState);
810 }
811
812 return Error::success();
813}
814
815Error COFFPlatform::COFFPlatformPlugin::registerObjectPlatformSections(
816 jitlink::LinkGraph &G, JITDylib &JD) {
817 COFFObjectSectionsMap ObjSecs;
818 auto HeaderAddr = CP.JITDylibToHeaderAddr[&JD];
819 assert(HeaderAddr && "Must be registered jitdylib");
820 for (auto &S : G.sections()) {
821 jitlink::SectionRange Range(S);
822 if (Range.getSize())
823 ObjSecs.push_back(Elt: std::make_pair(x: S.getName().str(), y: Range.getRange()));
824 }
825
826 G.allocActions().push_back(
827 x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSCOFFRegisterObjectSectionsArgs>(
828 FnAddr: CP.orc_rt_coff_register_object_sections, Args: HeaderAddr, Args: ObjSecs, Args: true)),
829 .Dealloc: cantFail(
830 ValOrErr: WrapperFunctionCall::Create<SPSCOFFDeregisterObjectSectionsArgs>(
831 FnAddr: CP.orc_rt_coff_deregister_object_sections, Args: HeaderAddr,
832 Args: ObjSecs))});
833
834 return Error::success();
835}
836
837Error COFFPlatform::COFFPlatformPlugin::preserveInitializerSections(
838 jitlink::LinkGraph &G, MaterializationResponsibility &MR) {
839
840 if (const auto &InitSymName = MR.getInitializerSymbol()) {
841
842 jitlink::Symbol *InitSym = nullptr;
843
844 for (auto &InitSection : G.sections()) {
845 // Skip non-init sections.
846 if (!isCOFFInitializerSection(Name: InitSection.getName()) ||
847 InitSection.empty())
848 continue;
849
850 // Create the init symbol if it has not been created already and attach it
851 // to the first block.
852 if (!InitSym) {
853 auto &B = **InitSection.blocks().begin();
854 InitSym = &G.addDefinedSymbol(
855 Content&: B, Offset: 0, Name: *InitSymName, Size: B.getSize(), L: jitlink::Linkage::Strong,
856 S: jitlink::Scope::SideEffectsOnly, IsCallable: false, IsLive: true);
857 }
858
859 // Add keep-alive edges to anonymous symbols in all other init blocks.
860 for (auto *B : InitSection.blocks()) {
861 if (B == &InitSym->getBlock())
862 continue;
863
864 auto &S = G.addAnonymousSymbol(Content&: *B, Offset: 0, Size: B->getSize(), IsCallable: false, IsLive: true);
865 InitSym->getBlock().addEdge(K: jitlink::Edge::KeepAlive, Offset: 0, Target&: S, Addend: 0);
866 }
867 }
868 }
869
870 return Error::success();
871}
872
873Error COFFPlatform::COFFPlatformPlugin::
874 registerObjectPlatformSectionsInBootstrap(jitlink::LinkGraph &G,
875 JITDylib &JD) {
876 std::lock_guard<std::mutex> Lock(CP.PlatformMutex);
877 auto HeaderAddr = CP.JITDylibToHeaderAddr[&JD];
878 COFFObjectSectionsMap ObjSecs;
879 for (auto &S : G.sections()) {
880 jitlink::SectionRange Range(S);
881 if (Range.getSize())
882 ObjSecs.push_back(Elt: std::make_pair(x: S.getName().str(), y: Range.getRange()));
883 }
884
885 G.allocActions().push_back(
886 x: {.Finalize: {},
887 .Dealloc: cantFail(
888 ValOrErr: WrapperFunctionCall::Create<SPSCOFFDeregisterObjectSectionsArgs>(
889 FnAddr: CP.orc_rt_coff_deregister_object_sections, Args: HeaderAddr,
890 Args: ObjSecs))});
891
892 auto &BState = CP.JDBootstrapStates[&JD];
893 BState.ObjectSectionsMaps.push_back(x: std::move(ObjSecs));
894
895 // Collect static initializers
896 for (auto &S : G.sections())
897 if (isCOFFInitializerSection(Name: S.getName()))
898 for (auto *B : S.blocks()) {
899 if (B->edges_empty())
900 continue;
901 for (auto &E : B->edges())
902 BState.Initializers.push_back(Elt: std::make_pair(
903 x: S.getName().str(), y: E.getTarget().getAddress() + E.getAddend()));
904 }
905
906 return Error::success();
907}
908
909} // End namespace orc.
910} // End namespace llvm.
911