1//===------- COFFPlatform.cpp - Utilities for executing COFF in Orc -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/Orc/COFFPlatform.h"
10
11#include "llvm/ExecutionEngine/Orc/AbsoluteSymbols.h"
12#include "llvm/ExecutionEngine/Orc/COFF.h"
13#include "llvm/ExecutionEngine/Orc/CallProxiesSPS.h"
14#include "llvm/ExecutionEngine/Orc/DebugUtils.h"
15#include "llvm/ExecutionEngine/Orc/LookupAndApply.h"
16#include "llvm/ExecutionEngine/Orc/ObjectFileInterface.h"
17#include "llvm/ExecutionEngine/Orc/RecordProxy.h"
18#include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h"
19#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
20#include "llvm/Object/COFF.h"
21
22#include "llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h"
23
24#include "llvm/ExecutionEngine/JITLink/x86_64.h"
25
26#define DEBUG_TYPE "orc"
27
28using namespace llvm;
29using namespace llvm::orc;
30using namespace llvm::orc::shared;
31
32namespace llvm {
33namespace orc {
34namespace shared {
35
36using SPSCOFFJITDylibDepInfo = SPSSequence<SPSExecutorAddr>;
37using SPSCOFFJITDylibDepInfoMap =
38 SPSSequence<SPSTuple<SPSExecutorAddr, SPSCOFFJITDylibDepInfo>>;
39using SPSCOFFObjectSectionsMap =
40 SPSSequence<SPSTuple<SPSString, SPSExecutorAddrRange>>;
41using SPSCOFFRegisterObjectSectionsArgs =
42 SPSArgList<SPSExecutorAddr, SPSCOFFObjectSectionsMap, bool>;
43using SPSCOFFDeregisterObjectSectionsArgs =
44 SPSArgList<SPSExecutorAddr, SPSCOFFObjectSectionsMap>;
45
46} // namespace shared
47} // namespace orc
48} // namespace llvm
49namespace {
50
51class COFFHeaderMaterializationUnit : public MaterializationUnit {
52public:
53 COFFHeaderMaterializationUnit(COFFPlatform &CP,
54 const SymbolStringPtr &HeaderStartSymbol)
55 : MaterializationUnit(createHeaderInterface(MOP&: CP, HeaderStartSymbol)),
56 CP(CP) {}
57
58 StringRef getName() const override { return "COFFHeaderMU"; }
59
60 void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
61 auto G = std::make_unique<jitlink::LinkGraph>(
62 args: "<COFFHeaderMU>", args: CP.getExecutionSession().getSymbolStringPool(),
63 args: CP.getExecutionSession().getTargetTriple(), args: SubtargetFeatures(),
64 args&: jitlink::getGenericEdgeKindName);
65 auto &HeaderSection = G->createSection(Name: "__header", Prot: MemProt::Read);
66 auto &HeaderBlock = createHeaderBlock(G&: *G, HeaderSection);
67
68 // Init symbol is __ImageBase symbol.
69 auto &ImageBaseSymbol = G->addDefinedSymbol(
70 Content&: HeaderBlock, Offset: 0, Name: *R->getInitializerSymbol(), Size: HeaderBlock.getSize(),
71 L: jitlink::Linkage::Strong, S: jitlink::Scope::Default, IsCallable: false, IsLive: true);
72
73 addImageBaseRelocationEdge(B&: HeaderBlock, ImageBase&: ImageBaseSymbol);
74
75 CP.getObjectLinkingLayer().emit(R: std::move(R), G: std::move(G));
76 }
77
78 void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
79
80private:
81 struct HeaderSymbol {
82 const char *Name;
83 uint64_t Offset;
84 };
85
86 struct NTHeader {
87 support::ulittle32_t PEMagic;
88 object::coff_file_header FileHeader;
89 struct PEHeader {
90 object::pe32plus_header Header;
91 object::data_directory DataDirectory[COFF::NUM_DATA_DIRECTORIES + 1];
92 } OptionalHeader;
93 };
94
95 struct HeaderBlockContent {
96 object::dos_header DOSHeader;
97 COFFHeaderMaterializationUnit::NTHeader NTHeader;
98 };
99
100 static jitlink::Block &createHeaderBlock(jitlink::LinkGraph &G,
101 jitlink::Section &HeaderSection) {
102 HeaderBlockContent Hdr = {};
103
104 // Set up magic
105 Hdr.DOSHeader.Magic[0] = 'M';
106 Hdr.DOSHeader.Magic[1] = 'Z';
107 Hdr.DOSHeader.AddressOfNewExeHeader =
108 offsetof(HeaderBlockContent, NTHeader);
109 uint32_t PEMagic = *reinterpret_cast<const uint32_t *>(COFF::PEMagic);
110 Hdr.NTHeader.PEMagic = PEMagic;
111 Hdr.NTHeader.OptionalHeader.Header.Magic = COFF::PE32Header::PE32_PLUS;
112
113 switch (G.getTargetTriple().getArch()) {
114 case Triple::x86_64:
115 Hdr.NTHeader.FileHeader.Machine = COFF::IMAGE_FILE_MACHINE_AMD64;
116 break;
117 default:
118 llvm_unreachable("Unrecognized architecture");
119 }
120
121 auto HeaderContent = G.allocateContent(
122 Source: ArrayRef<char>(reinterpret_cast<const char *>(&Hdr), sizeof(Hdr)));
123
124 return G.createContentBlock(Parent&: HeaderSection, Content: HeaderContent, Address: ExecutorAddr(), Alignment: 8,
125 AlignmentOffset: 0);
126 }
127
128 static void addImageBaseRelocationEdge(jitlink::Block &B,
129 jitlink::Symbol &ImageBase) {
130 auto ImageBaseOffset = offsetof(HeaderBlockContent, NTHeader) +
131 offsetof(NTHeader, OptionalHeader) +
132 offsetof(object::pe32plus_header, ImageBase);
133 B.addEdge(K: jitlink::x86_64::Pointer64, Offset: ImageBaseOffset, Target&: ImageBase, Addend: 0);
134 }
135
136 static MaterializationUnit::Interface
137 createHeaderInterface(COFFPlatform &MOP,
138 const SymbolStringPtr &HeaderStartSymbol) {
139 SymbolFlagsMap HeaderSymbolFlags;
140
141 HeaderSymbolFlags[HeaderStartSymbol] = JITSymbolFlags::Exported;
142
143 return MaterializationUnit::Interface(std::move(HeaderSymbolFlags),
144 HeaderStartSymbol);
145 }
146
147 COFFPlatform &CP;
148};
149
150} // end anonymous namespace
151
152namespace llvm {
153namespace orc {
154
155Expected<std::unique_ptr<COFFPlatform>>
156COFFPlatform::Create(ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD,
157 std::unique_ptr<MemoryBuffer> OrcRuntimeArchiveBuffer,
158 LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime,
159 const char *VCRuntimePath,
160 std::optional<SymbolAliasMap> RuntimeAliases) {
161
162 auto &ES = ObjLinkingLayer.getExecutionSession();
163
164 // If the target is not supported then bail out immediately.
165 if (!supportedTarget(TT: ES.getTargetTriple()))
166 return make_error<StringError>(Args: "Unsupported COFFPlatform triple: " +
167 ES.getTargetTriple().str(),
168 Args: inconvertibleErrorCode());
169
170 auto GeneratorArchive =
171 object::Archive::create(Source: OrcRuntimeArchiveBuffer->getMemBufferRef());
172 if (!GeneratorArchive)
173 return GeneratorArchive.takeError();
174
175 std::set<std::string> DylibsToPreload;
176 auto OrcRuntimeArchiveGenerator = StaticLibraryDefinitionGenerator::Create(
177 L&: ObjLinkingLayer, ArchiveBuffer: nullptr, Archive: std::move(*GeneratorArchive),
178 VisitMembers: COFFImportFileScanner(DylibsToPreload));
179 if (!OrcRuntimeArchiveGenerator)
180 return OrcRuntimeArchiveGenerator.takeError();
181
182 // We need a second instance of the archive (for now) for the Platform. We
183 // can `cantFail` this call, since if it were going to fail it would have
184 // failed above.
185 auto RuntimeArchive = cantFail(
186 ValOrErr: object::Archive::create(Source: OrcRuntimeArchiveBuffer->getMemBufferRef()));
187
188 // Create default aliases if the caller didn't supply any.
189 if (!RuntimeAliases)
190 RuntimeAliases = standardPlatformAliases(ES);
191
192 // Define the aliases.
193 if (auto Err = PlatformJD.define(MU: symbolAliases(Aliases: std::move(*RuntimeAliases))))
194 return std::move(Err);
195
196 {
197 // Add JIT dispatch reexports from bootstrap JITDylib.
198 auto Exports = buildSimpleReexportsAliasMap(
199 SourceJD&: ES.getBootstrapJITDylib(),
200 Symbols: {{ES.intern(SymName: rt::DispatchName), ES.intern(SymName: rt::DispatchCtxName)}});
201 if (!Exports)
202 return Exports.takeError();
203 if (auto Err =
204 PlatformJD.define(MU: reexports(SourceJD&: ES.getBootstrapJITDylib(), Aliases: *Exports)))
205 return Err;
206 }
207
208 // Create the instance.
209 Error Err = Error::success();
210 auto P = std::unique_ptr<COFFPlatform>(new COFFPlatform(
211 ObjLinkingLayer, PlatformJD, std::move(*OrcRuntimeArchiveGenerator),
212 std::move(DylibsToPreload), std::move(OrcRuntimeArchiveBuffer),
213 std::move(RuntimeArchive), std::move(LoadDynLibrary), StaticVCRuntime,
214 VCRuntimePath, Err));
215 if (Err)
216 return std::move(Err);
217 return std::move(P);
218}
219
220Expected<std::unique_ptr<COFFPlatform>>
221COFFPlatform::Create(ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD,
222 const char *OrcRuntimePath,
223 LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime,
224 const char *VCRuntimePath,
225 std::optional<SymbolAliasMap> RuntimeAliases) {
226
227 auto ArchiveBuffer = MemoryBuffer::getFile(Filename: OrcRuntimePath);
228 if (!ArchiveBuffer)
229 return createFileError(F: OrcRuntimePath, EC: ArchiveBuffer.getError());
230
231 return Create(ObjLinkingLayer, PlatformJD, OrcRuntimeArchiveBuffer: std::move(*ArchiveBuffer),
232 LoadDynLibrary: std::move(LoadDynLibrary), StaticVCRuntime, VCRuntimePath,
233 RuntimeAliases: std::move(RuntimeAliases));
234}
235
236Expected<MemoryBufferRef> COFFPlatform::getPerJDObjectFile() {
237 auto PerJDObj = OrcRuntimeArchive->findSym(name: "__orc_rt_coff_per_jd_marker");
238 if (!PerJDObj)
239 return PerJDObj.takeError();
240
241 if (!*PerJDObj)
242 return make_error<StringError>(Args: "Could not find per jd object file",
243 Args: inconvertibleErrorCode());
244
245 auto Buffer = (*PerJDObj)->getAsBinary();
246 if (!Buffer)
247 return Buffer.takeError();
248
249 return (*Buffer)->getMemoryBufferRef();
250}
251
252static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases,
253 ArrayRef<std::pair<const char *, const char *>> AL) {
254 for (auto &KV : AL) {
255 auto AliasName = ES.intern(SymName: KV.first);
256 assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map");
257 Aliases[std::move(AliasName)] = {ES.intern(SymName: KV.second),
258 JITSymbolFlags::Exported};
259 }
260}
261
262Error COFFPlatform::setupJITDylib(JITDylib &JD) {
263 if (auto Err = JD.define(MU: std::make_unique<COFFHeaderMaterializationUnit>(
264 args&: *this, args&: COFFHeaderStartSymbol)))
265 return Err;
266
267 if (auto Err = ES.lookup(SearchOrder: {&JD}, Symbol: COFFHeaderStartSymbol).takeError())
268 return Err;
269
270 // Define the CXX aliases.
271 SymbolAliasMap CXXAliases;
272 addAliases(ES, Aliases&: CXXAliases, AL: requiredCXXAliases());
273 if (auto Err = JD.define(MU: symbolAliases(Aliases: std::move(CXXAliases))))
274 return Err;
275
276 auto PerJDObj = getPerJDObjectFile();
277 if (!PerJDObj)
278 return PerJDObj.takeError();
279
280 auto I = getObjectFileInterface(ES, ObjBuffer: *PerJDObj);
281 if (!I)
282 return I.takeError();
283
284 if (auto Err = ObjLinkingLayer.add(
285 JD, O: MemoryBuffer::getMemBuffer(Ref: *PerJDObj, RequiresNullTerminator: false), I: std::move(*I)))
286 return Err;
287
288 if (!Bootstrapping) {
289 auto ImportedLibs = StaticVCRuntime
290 ? VCRuntimeBootstrap->loadStaticVCRuntime(JD)
291 : VCRuntimeBootstrap->loadDynamicVCRuntime(JD);
292 if (!ImportedLibs)
293 return ImportedLibs.takeError();
294 for (auto &Lib : *ImportedLibs)
295 if (auto Err = LoadDynLibrary(JD, Lib))
296 return Err;
297 if (StaticVCRuntime)
298 if (auto Err = VCRuntimeBootstrap->initializeStaticVCRuntime(JD))
299 return Err;
300 }
301
302 JD.addGenerator(DefGenerator: DLLImportDefinitionGenerator::Create(ES, L&: ObjLinkingLayer));
303 return Error::success();
304}
305
306Error COFFPlatform::teardownJITDylib(JITDylib &JD) {
307 std::lock_guard<std::mutex> Lock(PlatformMutex);
308 auto I = JITDylibToHeaderAddr.find(Val: &JD);
309 if (I != JITDylibToHeaderAddr.end()) {
310 assert(HeaderAddrToJITDylib.count(I->second) &&
311 "HeaderAddrToJITDylib missing entry");
312 HeaderAddrToJITDylib.erase(Val: I->second);
313 JITDylibToHeaderAddr.erase(I);
314 }
315 return Error::success();
316}
317
318Error COFFPlatform::notifyAdding(ResourceTracker &RT,
319 const MaterializationUnit &MU) {
320 auto &JD = RT.getJITDylib();
321 const auto &InitSym = MU.getInitializerSymbol();
322 if (!InitSym)
323 return Error::success();
324
325 RegisteredInitSymbols[&JD].add(Name: InitSym,
326 Flags: SymbolLookupFlags::WeaklyReferencedSymbol);
327
328 LLVM_DEBUG({
329 dbgs() << "COFFPlatform: Registered init symbol " << *InitSym << " for MU "
330 << MU.getName() << "\n";
331 });
332 return Error::success();
333}
334
335Error COFFPlatform::notifyRemoving(ResourceTracker &RT) {
336 llvm_unreachable("Not supported yet");
337}
338
339SymbolAliasMap COFFPlatform::standardPlatformAliases(ExecutionSession &ES) {
340 SymbolAliasMap Aliases;
341 addAliases(ES, Aliases, AL: standardRuntimeUtilityAliases());
342 return Aliases;
343}
344
345ArrayRef<std::pair<const char *, const char *>>
346COFFPlatform::requiredCXXAliases() {
347 static const std::pair<const char *, const char *> RequiredCXXAliases[] = {
348 {"_CxxThrowException", "__orc_rt_coff_cxx_throw_exception"},
349 {"_onexit", "__orc_rt_coff_onexit_per_jd"},
350 {"atexit", "__orc_rt_coff_atexit_per_jd"}};
351
352 return ArrayRef<std::pair<const char *, const char *>>(RequiredCXXAliases);
353}
354
355ArrayRef<std::pair<const char *, const char *>>
356COFFPlatform::standardRuntimeUtilityAliases() {
357 static const std::pair<const char *, const char *>
358 StandardRuntimeUtilityAliases[] = {
359 {"__orc_rt_run_program", "__orc_rt_coff_run_program"},
360 {"__orc_rt_jit_dlerror", "__orc_rt_coff_jit_dlerror"},
361 {"__orc_rt_jit_dlopen", "__orc_rt_coff_jit_dlopen"},
362 {"__orc_rt_jit_dlupdate", "__orc_rt_coff_jit_dlupdate"},
363 {"__orc_rt_jit_dlclose", "__orc_rt_coff_jit_dlclose"},
364 {"__orc_rt_jit_dlsym", "__orc_rt_coff_jit_dlsym"},
365 {"__orc_rt_log_error", "__orc_rt_log_error_to_stderr"}};
366
367 return ArrayRef<std::pair<const char *, const char *>>(
368 StandardRuntimeUtilityAliases);
369}
370
371bool COFFPlatform::supportedTarget(const Triple &TT) {
372 switch (TT.getArch()) {
373 case Triple::x86_64:
374 return true;
375 default:
376 return false;
377 }
378}
379
380COFFPlatform::COFFPlatform(
381 ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD,
382 std::unique_ptr<StaticLibraryDefinitionGenerator> OrcRuntimeGenerator,
383 std::set<std::string> DylibsToPreload,
384 std::unique_ptr<MemoryBuffer> OrcRuntimeArchiveBuffer,
385 std::unique_ptr<object::Archive> OrcRuntimeArchive,
386 LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime,
387 const char *VCRuntimePath, Error &Err)
388 : ES(ObjLinkingLayer.getExecutionSession()),
389 ObjLinkingLayer(ObjLinkingLayer),
390 LoadDynLibrary(std::move(LoadDynLibrary)),
391 OrcRuntimeArchiveBuffer(std::move(OrcRuntimeArchiveBuffer)),
392 OrcRuntimeArchive(std::move(OrcRuntimeArchive)),
393 StaticVCRuntime(StaticVCRuntime),
394 COFFHeaderStartSymbol(ES.intern(SymName: "__ImageBase")) {
395 ErrorAsOutParameter _(Err);
396
397 Bootstrapping.store(i: true);
398 ObjLinkingLayer.addPlugin(P: std::make_unique<COFFPlatformPlugin>(args&: *this));
399
400 // Load vc runtime
401 auto VCRT =
402 COFFVCRuntimeBootstrapper::Create(ES, ObjLinkingLayer, RuntimePath: VCRuntimePath);
403 if (!VCRT) {
404 Err = VCRT.takeError();
405 return;
406 }
407 VCRuntimeBootstrap = std::move(*VCRT);
408
409 auto ImportedLibs =
410 StaticVCRuntime ? VCRuntimeBootstrap->loadStaticVCRuntime(JD&: PlatformJD)
411 : VCRuntimeBootstrap->loadDynamicVCRuntime(JD&: PlatformJD);
412 if (!ImportedLibs) {
413 Err = ImportedLibs.takeError();
414 return;
415 }
416
417 for (auto &Lib : *ImportedLibs)
418 DylibsToPreload.insert(x: Lib);
419
420 PlatformJD.addGenerator(DefGenerator: std::move(OrcRuntimeGenerator));
421
422 // PlatformJD hasn't been set up by the platform yet (since we're creating
423 // the platform now), so set it up.
424 if (auto E2 = setupJITDylib(PlatformJD)) {
425 Err = std::move(E2);
426 return;
427 }
428
429 for (auto& Lib : DylibsToPreload)
430 if (auto E2 = this->LoadDynLibrary(PlatformJD, Lib)) {
431 Err = std::move(E2);
432 return;
433 }
434
435 if (StaticVCRuntime)
436 if (auto E2 = VCRuntimeBootstrap->initializeStaticVCRuntime(JD&: PlatformJD)) {
437 Err = std::move(E2);
438 return;
439 }
440
441 // Associate wrapper function tags with JIT-side function implementations.
442 if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) {
443 Err = std::move(E2);
444 return;
445 }
446
447 // Lookup addresses of runtime functions callable by the platform,
448 // call the platform bootstrap function to initialize the platform-state
449 // object in the executor.
450 if (auto E2 = bootstrapCOFFRuntime(PlatformJD)) {
451 Err = std::move(E2);
452 return;
453 }
454
455 Bootstrapping.store(i: false);
456 JDBootstrapStates.clear();
457}
458
459Expected<COFFPlatform::JITDylibDepMap>
460COFFPlatform::buildJDDepMap(JITDylib &JD) {
461 return ES.runSessionLocked(F: [&]() -> Expected<JITDylibDepMap> {
462 JITDylibDepMap JDDepMap;
463
464 SmallVector<JITDylib *, 16> Worklist({&JD});
465 while (!Worklist.empty()) {
466 auto CurJD = Worklist.back();
467 Worklist.pop_back();
468
469 auto &DM = JDDepMap[CurJD];
470 CurJD->withLinkOrderDo(F: [&](const JITDylibSearchOrder &O) {
471 DM.reserve(N: O.size());
472 for (auto &KV : O) {
473 if (KV.first == CurJD)
474 continue;
475 {
476 // Bare jitdylibs not known to the platform
477 std::lock_guard<std::mutex> Lock(PlatformMutex);
478 if (!JITDylibToHeaderAddr.count(Val: KV.first)) {
479 LLVM_DEBUG({
480 dbgs() << "JITDylib unregistered to COFFPlatform detected in "
481 "LinkOrder: "
482 << CurJD->getName() << "\n";
483 });
484 continue;
485 }
486 }
487 DM.push_back(Elt: KV.first);
488 // Push unvisited entry.
489 if (JDDepMap.try_emplace(Key: KV.first).second)
490 Worklist.push_back(Elt: KV.first);
491 }
492 });
493 }
494 return std::move(JDDepMap);
495 });
496}
497
498void COFFPlatform::pushInitializersLoop(PushInitializersSendResultFn SendResult,
499 JITDylibSP JD,
500 JITDylibDepMap &JDDepMap) {
501 SmallVector<JITDylib *, 16> Worklist({JD.get()});
502 DenseSet<JITDylib *> Visited({JD.get()});
503 DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols;
504 ES.runSessionLocked(F: [&]() {
505 while (!Worklist.empty()) {
506 auto CurJD = Worklist.back();
507 Worklist.pop_back();
508
509 auto RISItr = RegisteredInitSymbols.find(Val: CurJD);
510 if (RISItr != RegisteredInitSymbols.end()) {
511 NewInitSymbols[CurJD] = std::move(RISItr->second);
512 RegisteredInitSymbols.erase(I: RISItr);
513 }
514
515 for (auto *DepJD : JDDepMap[CurJD])
516 if (Visited.insert(V: DepJD).second)
517 Worklist.push_back(Elt: DepJD);
518 }
519 });
520
521 // If there are no further init symbols to look up then send the link order
522 // (as a list of header addresses) to the caller.
523 if (NewInitSymbols.empty()) {
524 // Build the dep info map to return.
525 COFFJITDylibDepInfoMap DIM;
526 DIM.reserve(n: JDDepMap.size());
527 for (auto &KV : JDDepMap) {
528 std::lock_guard<std::mutex> Lock(PlatformMutex);
529 COFFJITDylibDepInfo DepInfo;
530 DepInfo.reserve(n: KV.second.size());
531 for (auto &Dep : KV.second) {
532 DepInfo.push_back(x: JITDylibToHeaderAddr[Dep]);
533 }
534 auto H = JITDylibToHeaderAddr[KV.first];
535 DIM.push_back(x: std::make_pair(x&: H, y: std::move(DepInfo)));
536 }
537 SendResult(DIM);
538 return;
539 }
540
541 // Otherwise issue a lookup and re-run this phase when it completes.
542 lookupInitSymbolsAsync(
543 OnComplete: [this, SendResult = std::move(SendResult), &JD,
544 JDDepMap = std::move(JDDepMap)](Error Err) mutable {
545 if (Err)
546 SendResult(std::move(Err));
547 else
548 pushInitializersLoop(SendResult: std::move(SendResult), JD, JDDepMap);
549 },
550 ES, InitSyms: std::move(NewInitSymbols));
551}
552
553void COFFPlatform::rt_pushInitializers(PushInitializersSendResultFn SendResult,
554 ExecutorAddr JDHeaderAddr) {
555 JITDylibSP JD;
556 {
557 std::lock_guard<std::mutex> Lock(PlatformMutex);
558 auto I = HeaderAddrToJITDylib.find(Val: JDHeaderAddr);
559 if (I != HeaderAddrToJITDylib.end())
560 JD = I->second;
561 }
562
563 LLVM_DEBUG({
564 dbgs() << "COFFPlatform::rt_pushInitializers(" << JDHeaderAddr << ") ";
565 if (JD)
566 dbgs() << "pushing initializers for " << JD->getName() << "\n";
567 else
568 dbgs() << "No JITDylib for header address.\n";
569 });
570
571 if (!JD) {
572 SendResult(make_error<StringError>(Args: "No JITDylib with header addr " +
573 formatv(Fmt: "{0:x}", Vals&: JDHeaderAddr),
574 Args: inconvertibleErrorCode()));
575 return;
576 }
577
578 auto JDDepMap = buildJDDepMap(JD&: *JD);
579 if (!JDDepMap) {
580 SendResult(JDDepMap.takeError());
581 return;
582 }
583
584 pushInitializersLoop(SendResult: std::move(SendResult), JD, JDDepMap&: *JDDepMap);
585}
586
587void COFFPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult,
588 ExecutorAddr Handle, StringRef SymbolName) {
589 LLVM_DEBUG(dbgs() << "COFFPlatform::rt_lookupSymbol(\"" << Handle << "\")\n");
590
591 JITDylib *JD = nullptr;
592
593 {
594 std::lock_guard<std::mutex> Lock(PlatformMutex);
595 auto I = HeaderAddrToJITDylib.find(Val: Handle);
596 if (I != HeaderAddrToJITDylib.end())
597 JD = I->second;
598 }
599
600 if (!JD) {
601 LLVM_DEBUG(dbgs() << " No JITDylib for handle " << Handle << "\n");
602 SendResult(make_error<StringError>(Args: "No JITDylib associated with handle " +
603 formatv(Fmt: "{0:x}", Vals&: Handle),
604 Args: inconvertibleErrorCode()));
605 return;
606 }
607
608 // Use functor class to work around XL build compiler issue on AIX.
609 class RtLookupNotifyComplete {
610 public:
611 RtLookupNotifyComplete(SendSymbolAddressFn &&SendResult)
612 : SendResult(std::move(SendResult)) {}
613 void operator()(Expected<SymbolMap> Result) {
614 if (Result) {
615 assert(Result->size() == 1 && "Unexpected result map count");
616 SendResult(Result->begin()->second.getAddress());
617 } else {
618 SendResult(Result.takeError());
619 }
620 }
621
622 private:
623 SendSymbolAddressFn SendResult;
624 };
625
626 ES.lookup(
627 K: LookupKind::DLSym, SearchOrder: {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}},
628 Symbols: SymbolLookupSet(ES.intern(SymName: SymbolName)), RequiredState: SymbolState::Ready,
629 NotifyComplete: RtLookupNotifyComplete(std::move(SendResult)), RegisterDependencies: NoDependenciesToRegister);
630}
631
632Error COFFPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) {
633 ExecutionSession::JITDispatchHandlerAssociationMap WFs;
634
635 using LookupSymbolSPSSig =
636 SPSExpected<SPSExecutorAddr>(SPSExecutorAddr, SPSString);
637 WFs[ES.intern(SymName: "__orc_rt_coff_symbol_lookup_tag")] =
638 ES.wrapAsyncWithSPS<LookupSymbolSPSSig>(Instance: this,
639 Method: &COFFPlatform::rt_lookupSymbol);
640 using PushInitializersSPSSig =
641 SPSExpected<SPSCOFFJITDylibDepInfoMap>(SPSExecutorAddr);
642 WFs[ES.intern(SymName: "__orc_rt_coff_push_initializers_tag")] =
643 ES.wrapAsyncWithSPS<PushInitializersSPSSig>(
644 Instance: this, Method: &COFFPlatform::rt_pushInitializers);
645
646 return ES.registerJITDispatchHandlers(JD&: PlatformJD, WFs: std::move(WFs));
647}
648
649Error COFFPlatform::runBootstrapInitializers(JDBootstrapState &BState) {
650 llvm::sort(C&: BState.Initializers);
651 if (auto Err =
652 runBootstrapSubsectionInitializers(BState, Start: ".CRT$XIA", End: ".CRT$XIZ"))
653 return Err;
654
655 if (auto Err = runSymbolIfExists(PlatformJD&: *BState.JD, SymbolName: "__run_after_c_init"))
656 return Err;
657
658 if (auto Err =
659 runBootstrapSubsectionInitializers(BState, Start: ".CRT$XCA", End: ".CRT$XCZ"))
660 return Err;
661 return Error::success();
662}
663
664Error COFFPlatform::runBootstrapSubsectionInitializers(JDBootstrapState &BState,
665 StringRef Start,
666 StringRef End) {
667 CallInt32VoidProxy CallInitializer;
668 if (auto Err = lookupAndApply(
669 JD&: ES.getBootstrapJITDylib(),
670 PrepareFns: {recordProxy<sps::CallInt32VoidProxySpec>(P: &CallInitializer)}))
671 return Err;
672 for (auto &Initializer : BState.Initializers)
673 if (Initializer.first >= Start && Initializer.first <= End &&
674 Initializer.second) {
675 auto Res = CallInitializer(ES, Initializer.second);
676 if (!Res)
677 return Res.takeError();
678 }
679 return Error::success();
680}
681
682Error COFFPlatform::bootstrapCOFFRuntime(JITDylib &PlatformJD) {
683 // Lookup of runtime symbols causes the collection of initializers if
684 // it's static linking setting.
685 if (auto Err = lookupAndApply(
686 JD&: PlatformJD, PrepareFns: {recordAddr(Name: "__orc_rt_coff_platform_bootstrap",
687 A: &orc_rt_coff_platform_bootstrap),
688 recordAddr(Name: "__orc_rt_coff_platform_shutdown",
689 A: &orc_rt_coff_platform_shutdown),
690 recordAddr(Name: "__orc_rt_coff_register_jitdylib",
691 A: &orc_rt_coff_register_jitdylib),
692 recordAddr(Name: "__orc_rt_coff_deregister_jitdylib",
693 A: &orc_rt_coff_deregister_jitdylib),
694 recordAddr(Name: "__orc_rt_coff_register_object_sections",
695 A: &orc_rt_coff_register_object_sections),
696 recordAddr(Name: "__orc_rt_coff_deregister_object_sections",
697 A: &orc_rt_coff_deregister_object_sections)}))
698 return Err;
699
700 // Call bootstrap functions
701 if (auto Err = ES.callSPSWrapper<void()>(WrapperFnAddr: orc_rt_coff_platform_bootstrap))
702 return Err;
703
704 // Do the pending jitdylib registration actions that we couldn't do
705 // because orc runtime was not linked fully.
706 for (auto KV : JDBootstrapStates) {
707 auto &JDBState = KV.second;
708 if (auto Err = ES.callSPSWrapper<void(SPSString, SPSExecutorAddr)>(
709 WrapperFnAddr: orc_rt_coff_register_jitdylib, WrapperCallArgs&: JDBState.JDName,
710 WrapperCallArgs&: JDBState.HeaderAddr))
711 return Err;
712
713 for (auto &ObjSectionMap : JDBState.ObjectSectionsMaps)
714 if (auto Err = ES.callSPSWrapper<void(SPSExecutorAddr,
715 SPSCOFFObjectSectionsMap, bool)>(
716 WrapperFnAddr: orc_rt_coff_register_object_sections, WrapperCallArgs&: JDBState.HeaderAddr,
717 WrapperCallArgs&: ObjSectionMap, WrapperCallArgs: false))
718 return Err;
719 }
720
721 // Run static initializers collected in bootstrap stage.
722 for (auto KV : JDBootstrapStates) {
723 auto &JDBState = KV.second;
724 if (auto Err = runBootstrapInitializers(BState&: JDBState))
725 return Err;
726 }
727
728 return Error::success();
729}
730
731Error COFFPlatform::runSymbolIfExists(JITDylib &PlatformJD,
732 StringRef SymbolName) {
733 ExecutorAddr TargetFn;
734 if (auto Err = lookupAndApply(
735 JD&: PlatformJD, PrepareFns: {recordAddr(Name: SymbolName, A: &TargetFn,
736 LF: SymbolLookupFlags::WeaklyReferencedSymbol)}))
737 return Err;
738 if (!TargetFn)
739 return Error::success(); // No target function.
740
741 CallInt32VoidProxy CallFn;
742 if (auto Err =
743 lookupAndApply(JD&: ES.getBootstrapJITDylib(),
744 PrepareFns: {recordProxy<sps::CallInt32VoidProxySpec>(P: &CallFn)}))
745 return Err;
746
747 return CallFn(ES, TargetFn).takeError();
748}
749
750void COFFPlatform::COFFPlatformPlugin::modifyPassConfig(
751 MaterializationResponsibility &MR, jitlink::LinkGraph &LG,
752 jitlink::PassConfiguration &Config) {
753
754 bool IsBootstrapping = CP.Bootstrapping.load();
755
756 if (auto InitSymbol = MR.getInitializerSymbol()) {
757 if (InitSymbol == CP.COFFHeaderStartSymbol) {
758 Config.PostAllocationPasses.push_back(
759 x: [this, &MR, IsBootstrapping](jitlink::LinkGraph &G) {
760 return associateJITDylibHeaderSymbol(G, MR, Bootstrap: IsBootstrapping);
761 });
762 return;
763 }
764 Config.PrePrunePasses.push_back(x: [this, &MR](jitlink::LinkGraph &G) {
765 return preserveInitializerSections(G, MR);
766 });
767 }
768
769 if (!IsBootstrapping)
770 Config.PostFixupPasses.push_back(
771 x: [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
772 return registerObjectPlatformSections(G, JD);
773 });
774 else
775 Config.PostFixupPasses.push_back(
776 x: [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
777 return registerObjectPlatformSectionsInBootstrap(G, JD);
778 });
779}
780
781Error COFFPlatform::COFFPlatformPlugin::associateJITDylibHeaderSymbol(
782 jitlink::LinkGraph &G, MaterializationResponsibility &MR,
783 bool IsBootstraping) {
784 auto I = llvm::find_if(Range: G.defined_symbols(), P: [this](jitlink::Symbol *Sym) {
785 return *Sym->getName() == *CP.COFFHeaderStartSymbol;
786 });
787 assert(I != G.defined_symbols().end() && "Missing COFF header start symbol");
788
789 auto &JD = MR.getTargetJITDylib();
790 std::lock_guard<std::mutex> Lock(CP.PlatformMutex);
791 auto HeaderAddr = (*I)->getAddress();
792 CP.JITDylibToHeaderAddr[&JD] = HeaderAddr;
793 CP.HeaderAddrToJITDylib[HeaderAddr] = &JD;
794 if (!IsBootstraping) {
795 G.allocActions().push_back(
796 x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<
797 SPSArgList<SPSString, SPSExecutorAddr>>(
798 FnAddr: CP.orc_rt_coff_register_jitdylib, Args: JD.getName(), Args: HeaderAddr)),
799 .Dealloc: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
800 FnAddr: CP.orc_rt_coff_deregister_jitdylib, Args: HeaderAddr))});
801 } else {
802 G.allocActions().push_back(
803 x: {.Finalize: {},
804 .Dealloc: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
805 FnAddr: CP.orc_rt_coff_deregister_jitdylib, Args: HeaderAddr))});
806 JDBootstrapState BState;
807 BState.JD = &JD;
808 BState.JDName = JD.getName();
809 BState.HeaderAddr = HeaderAddr;
810 CP.JDBootstrapStates.emplace(args: &JD, args&: BState);
811 }
812
813 return Error::success();
814}
815
816Error COFFPlatform::COFFPlatformPlugin::registerObjectPlatformSections(
817 jitlink::LinkGraph &G, JITDylib &JD) {
818 COFFObjectSectionsMap ObjSecs;
819 auto HeaderAddr = CP.JITDylibToHeaderAddr[&JD];
820 assert(HeaderAddr && "Must be registered jitdylib");
821 for (auto &S : G.sections()) {
822 jitlink::SectionRange Range(S);
823 if (Range.getSize())
824 ObjSecs.push_back(Elt: std::make_pair(x: S.getName().str(), y: Range.getRange()));
825 }
826
827 G.allocActions().push_back(
828 x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSCOFFRegisterObjectSectionsArgs>(
829 FnAddr: CP.orc_rt_coff_register_object_sections, Args: HeaderAddr, Args: ObjSecs, Args: true)),
830 .Dealloc: cantFail(
831 ValOrErr: WrapperFunctionCall::Create<SPSCOFFDeregisterObjectSectionsArgs>(
832 FnAddr: CP.orc_rt_coff_deregister_object_sections, Args: HeaderAddr,
833 Args: ObjSecs))});
834
835 return Error::success();
836}
837
838Error COFFPlatform::COFFPlatformPlugin::preserveInitializerSections(
839 jitlink::LinkGraph &G, MaterializationResponsibility &MR) {
840
841 if (const auto &InitSymName = MR.getInitializerSymbol()) {
842
843 jitlink::Symbol *InitSym = nullptr;
844
845 for (auto &InitSection : G.sections()) {
846 // Skip non-init sections.
847 if (!isCOFFInitializerSection(Name: InitSection.getName()) ||
848 InitSection.empty())
849 continue;
850
851 // Create the init symbol if it has not been created already and attach it
852 // to the first block.
853 if (!InitSym) {
854 auto &B = **InitSection.blocks().begin();
855 InitSym = &G.addDefinedSymbol(
856 Content&: B, Offset: 0, Name: *InitSymName, Size: B.getSize(), L: jitlink::Linkage::Strong,
857 S: jitlink::Scope::SideEffectsOnly, IsCallable: false, IsLive: true);
858 }
859
860 // Add keep-alive edges to anonymous symbols in all other init blocks.
861 for (auto *B : InitSection.blocks()) {
862 if (B == &InitSym->getBlock())
863 continue;
864
865 auto &S = G.addAnonymousSymbol(Content&: *B, Offset: 0, Size: B->getSize(), IsCallable: false, IsLive: true);
866 InitSym->getBlock().addEdge(K: jitlink::Edge::KeepAlive, Offset: 0, Target&: S, Addend: 0);
867 }
868 }
869 }
870
871 return Error::success();
872}
873
874Error COFFPlatform::COFFPlatformPlugin::
875 registerObjectPlatformSectionsInBootstrap(jitlink::LinkGraph &G,
876 JITDylib &JD) {
877 std::lock_guard<std::mutex> Lock(CP.PlatformMutex);
878 auto HeaderAddr = CP.JITDylibToHeaderAddr[&JD];
879 COFFObjectSectionsMap ObjSecs;
880 for (auto &S : G.sections()) {
881 jitlink::SectionRange Range(S);
882 if (Range.getSize())
883 ObjSecs.push_back(Elt: std::make_pair(x: S.getName().str(), y: Range.getRange()));
884 }
885
886 G.allocActions().push_back(
887 x: {.Finalize: {},
888 .Dealloc: cantFail(
889 ValOrErr: WrapperFunctionCall::Create<SPSCOFFDeregisterObjectSectionsArgs>(
890 FnAddr: CP.orc_rt_coff_deregister_object_sections, Args: HeaderAddr,
891 Args: ObjSecs))});
892
893 auto &BState = CP.JDBootstrapStates[&JD];
894 BState.ObjectSectionsMaps.push_back(x: std::move(ObjSecs));
895
896 // Collect static initializers
897 for (auto &S : G.sections())
898 if (isCOFFInitializerSection(Name: S.getName()))
899 for (auto *B : S.blocks()) {
900 if (B->edges_empty())
901 continue;
902 for (auto &E : B->edges())
903 BState.Initializers.push_back(Elt: std::make_pair(
904 x: S.getName().str(), y: E.getTarget().getAddress() + E.getAddend()));
905 }
906
907 return Error::success();
908}
909
910} // End namespace orc.
911} // End namespace llvm.
912