1//===------- COFFPlatform.cpp - Utilities for executing COFF in Orc -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/Orc/COFFPlatform.h"
10#include "llvm/ExecutionEngine/Orc/Mangling.h"
11
12#include "llvm/ExecutionEngine/Orc/COFF.h"
13#include "llvm/ExecutionEngine/Orc/CallProxiesSPS.h"
14#include "llvm/ExecutionEngine/Orc/DebugUtils.h"
15#include "llvm/ExecutionEngine/Orc/LookupAndApply.h"
16#include "llvm/ExecutionEngine/Orc/ObjectFileInterface.h"
17#include "llvm/ExecutionEngine/Orc/RecordProxy.h"
18#include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h"
19#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
20#include "llvm/Object/COFF.h"
21
22#include "llvm/ExecutionEngine/JITLink/x86_64.h"
23
24#define DEBUG_TYPE "orc"
25
26using namespace llvm;
27using namespace llvm::orc;
28using namespace llvm::orc::shared;
29
30namespace llvm {
31namespace orc {
32namespace shared {
33
34using SPSCOFFJITDylibDepInfo = SPSSequence<SPSExecutorAddr>;
35using SPSCOFFJITDylibDepInfoMap =
36 SPSSequence<SPSTuple<SPSExecutorAddr, SPSCOFFJITDylibDepInfo>>;
37using SPSCOFFObjectSectionsMap =
38 SPSSequence<SPSTuple<SPSString, SPSExecutorAddrRange>>;
39using SPSCOFFRegisterObjectSectionsArgs =
40 SPSArgList<SPSExecutorAddr, SPSCOFFObjectSectionsMap, bool>;
41using SPSCOFFDeregisterObjectSectionsArgs =
42 SPSArgList<SPSExecutorAddr, SPSCOFFObjectSectionsMap>;
43
44} // namespace shared
45} // namespace orc
46} // namespace llvm
47namespace {
48
49class COFFHeaderMaterializationUnit : public MaterializationUnit {
50public:
51 COFFHeaderMaterializationUnit(COFFPlatform &CP,
52 const SymbolStringPtr &HeaderStartSymbol)
53 : MaterializationUnit(createHeaderInterface(MOP&: CP, HeaderStartSymbol)),
54 CP(CP) {}
55
56 StringRef getName() const override { return "COFFHeaderMU"; }
57
58 void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
59 auto G = std::make_unique<jitlink::LinkGraph>(
60 args: "<COFFHeaderMU>", args: CP.getExecutionSession().getSymbolStringPool(),
61 args: CP.getExecutionSession().getTargetTriple(), args: SubtargetFeatures(),
62 args&: jitlink::getGenericEdgeKindName);
63 auto &HeaderSection = G->createSection(Name: "__header", Prot: MemProt::Read);
64 auto &HeaderBlock = createHeaderBlock(G&: *G, HeaderSection);
65
66 // Init symbol is __ImageBase symbol.
67 auto &ImageBaseSymbol = G->addDefinedSymbol(
68 Content&: HeaderBlock, Offset: 0, Name: *R->getInitializerSymbol(), Size: HeaderBlock.getSize(),
69 L: jitlink::Linkage::Strong, S: jitlink::Scope::Default, IsCallable: false, IsLive: true);
70
71 addImageBaseRelocationEdge(B&: HeaderBlock, ImageBase&: ImageBaseSymbol);
72
73 CP.getObjectLinkingLayer().emit(R: std::move(R), G: std::move(G));
74 }
75
76 void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
77
78private:
79 struct HeaderSymbol {
80 const char *Name;
81 uint64_t Offset;
82 };
83
84 struct NTHeader {
85 support::ulittle32_t PEMagic;
86 object::coff_file_header FileHeader;
87 struct PEHeader {
88 object::pe32plus_header Header;
89 object::data_directory DataDirectory[COFF::NUM_DATA_DIRECTORIES + 1];
90 } OptionalHeader;
91 };
92
93 struct HeaderBlockContent {
94 object::dos_header DOSHeader;
95 COFFHeaderMaterializationUnit::NTHeader NTHeader;
96 };
97
98 static jitlink::Block &createHeaderBlock(jitlink::LinkGraph &G,
99 jitlink::Section &HeaderSection) {
100 HeaderBlockContent Hdr = {};
101
102 // Set up magic
103 Hdr.DOSHeader.Magic[0] = 'M';
104 Hdr.DOSHeader.Magic[1] = 'Z';
105 Hdr.DOSHeader.AddressOfNewExeHeader =
106 offsetof(HeaderBlockContent, NTHeader);
107 uint32_t PEMagic = *reinterpret_cast<const uint32_t *>(COFF::PEMagic);
108 Hdr.NTHeader.PEMagic = PEMagic;
109 Hdr.NTHeader.OptionalHeader.Header.Magic = COFF::PE32Header::PE32_PLUS;
110
111 switch (G.getTargetTriple().getArch()) {
112 case Triple::x86_64:
113 Hdr.NTHeader.FileHeader.Machine = COFF::IMAGE_FILE_MACHINE_AMD64;
114 break;
115 default:
116 llvm_unreachable("Unrecognized architecture");
117 }
118
119 auto HeaderContent = G.allocateContent(
120 Source: ArrayRef<char>(reinterpret_cast<const char *>(&Hdr), sizeof(Hdr)));
121
122 return G.createContentBlock(Parent&: HeaderSection, Content: HeaderContent, Address: ExecutorAddr(), Alignment: 8,
123 AlignmentOffset: 0);
124 }
125
126 static void addImageBaseRelocationEdge(jitlink::Block &B,
127 jitlink::Symbol &ImageBase) {
128 auto ImageBaseOffset = offsetof(HeaderBlockContent, NTHeader) +
129 offsetof(NTHeader, OptionalHeader) +
130 offsetof(object::pe32plus_header, ImageBase);
131 B.addEdge(K: jitlink::x86_64::Pointer64, Offset: ImageBaseOffset, Target&: ImageBase, Addend: 0);
132 }
133
134 static MaterializationUnit::Interface
135 createHeaderInterface(COFFPlatform &MOP,
136 const SymbolStringPtr &HeaderStartSymbol) {
137 SymbolFlagsMap HeaderSymbolFlags;
138
139 HeaderSymbolFlags[HeaderStartSymbol] = JITSymbolFlags::Exported;
140
141 return MaterializationUnit::Interface(std::move(HeaderSymbolFlags),
142 HeaderStartSymbol);
143 }
144
145 COFFPlatform &CP;
146};
147
148} // end anonymous namespace
149
150namespace llvm {
151namespace orc {
152
153Expected<std::unique_ptr<COFFPlatform>>
154COFFPlatform::Create(ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD,
155 std::unique_ptr<MemoryBuffer> OrcRuntimeArchiveBuffer,
156 LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime,
157 const char *VCRuntimePath,
158 std::optional<SymbolAliasMap> RuntimeAliases) {
159
160 auto &ES = ObjLinkingLayer.getExecutionSession();
161
162 // If the target is not supported then bail out immediately.
163 if (!supportedTarget(TT: ES.getTargetTriple()))
164 return make_error<StringError>(Args: "Unsupported COFFPlatform triple: " +
165 ES.getTargetTriple().str(),
166 Args: inconvertibleErrorCode());
167
168 auto GeneratorArchive =
169 object::Archive::create(Source: OrcRuntimeArchiveBuffer->getMemBufferRef());
170 if (!GeneratorArchive)
171 return GeneratorArchive.takeError();
172
173 std::set<std::string> DylibsToPreload;
174 auto OrcRuntimeArchiveGenerator = StaticLibraryDefinitionGenerator::Create(
175 L&: ObjLinkingLayer, ArchiveBuffer: nullptr, Archive: std::move(*GeneratorArchive),
176 VisitMembers: COFFImportFileScanner(DylibsToPreload));
177 if (!OrcRuntimeArchiveGenerator)
178 return OrcRuntimeArchiveGenerator.takeError();
179
180 // We need a second instance of the archive (for now) for the Platform. We
181 // can `cantFail` this call, since if it were going to fail it would have
182 // failed above.
183 auto RuntimeArchive = cantFail(
184 ValOrErr: object::Archive::create(Source: OrcRuntimeArchiveBuffer->getMemBufferRef()));
185
186 // Create default aliases if the caller didn't supply any.
187 if (!RuntimeAliases)
188 RuntimeAliases = standardPlatformAliases(ES);
189
190 // Define the aliases.
191 if (auto Err = PlatformJD.define(MU: symbolAliases(Aliases: std::move(*RuntimeAliases))))
192 return std::move(Err);
193
194 {
195 // Add JIT dispatch reexports from bootstrap JITDylib.
196 MangleAndInterner Mangle(ES);
197 auto Exports = buildSimpleReexportsAliasMap(
198 SourceJD&: ES.getBootstrapJITDylib(),
199 Symbols: {{Mangle(rt::DispatchName), Mangle(rt::DispatchCtxName)}});
200 if (!Exports)
201 return Exports.takeError();
202 if (auto Err =
203 PlatformJD.define(MU: reexports(SourceJD&: ES.getBootstrapJITDylib(), Aliases: *Exports)))
204 return Err;
205 }
206
207 // Create the instance.
208 Error Err = Error::success();
209 auto P = std::unique_ptr<COFFPlatform>(new COFFPlatform(
210 ObjLinkingLayer, PlatformJD, std::move(*OrcRuntimeArchiveGenerator),
211 std::move(DylibsToPreload), std::move(OrcRuntimeArchiveBuffer),
212 std::move(RuntimeArchive), std::move(LoadDynLibrary), StaticVCRuntime,
213 VCRuntimePath, Err));
214 if (Err)
215 return std::move(Err);
216 return std::move(P);
217}
218
219Expected<std::unique_ptr<COFFPlatform>>
220COFFPlatform::Create(ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD,
221 const char *OrcRuntimePath,
222 LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime,
223 const char *VCRuntimePath,
224 std::optional<SymbolAliasMap> RuntimeAliases) {
225
226 auto ArchiveBuffer = MemoryBuffer::getFile(Filename: OrcRuntimePath);
227 if (!ArchiveBuffer)
228 return createFileError(F: OrcRuntimePath, EC: ArchiveBuffer.getError());
229
230 return Create(ObjLinkingLayer, PlatformJD, OrcRuntimeArchiveBuffer: std::move(*ArchiveBuffer),
231 LoadDynLibrary: std::move(LoadDynLibrary), StaticVCRuntime, VCRuntimePath,
232 RuntimeAliases: std::move(RuntimeAliases));
233}
234
235Expected<MemoryBufferRef> COFFPlatform::getPerJDObjectFile() {
236 auto PerJDObj = OrcRuntimeArchive->findSym(name: "__orc_rt_coff_per_jd_marker");
237 if (!PerJDObj)
238 return PerJDObj.takeError();
239
240 if (!*PerJDObj)
241 return make_error<StringError>(Args: "Could not find per jd object file",
242 Args: inconvertibleErrorCode());
243
244 auto Buffer = (*PerJDObj)->getAsBinary();
245 if (!Buffer)
246 return Buffer.takeError();
247
248 return (*Buffer)->getMemoryBufferRef();
249}
250
251static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases,
252 ArrayRef<std::pair<const char *, const char *>> AL) {
253 for (auto &KV : AL) {
254 auto AliasName = ES.intern(SymName: KV.first);
255 assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map");
256 Aliases[std::move(AliasName)] = {ES.intern(SymName: KV.second),
257 JITSymbolFlags::Exported};
258 }
259}
260
261Error COFFPlatform::setupJITDylib(JITDylib &JD) {
262 if (auto Err = JD.define(MU: std::make_unique<COFFHeaderMaterializationUnit>(
263 args&: *this, args&: COFFHeaderStartSymbol)))
264 return Err;
265
266 if (auto Err = ES.lookup(SearchOrder: {&JD}, Symbol: COFFHeaderStartSymbol).takeError())
267 return Err;
268
269 // Define the CXX aliases.
270 SymbolAliasMap CXXAliases;
271 addAliases(ES, Aliases&: CXXAliases, AL: requiredCXXAliases());
272 if (auto Err = JD.define(MU: symbolAliases(Aliases: std::move(CXXAliases))))
273 return Err;
274
275 auto PerJDObj = getPerJDObjectFile();
276 if (!PerJDObj)
277 return PerJDObj.takeError();
278
279 auto I = getObjectFileInterface(ES, ObjBuffer: *PerJDObj);
280 if (!I)
281 return I.takeError();
282
283 if (auto Err = ObjLinkingLayer.add(
284 JD, O: MemoryBuffer::getMemBuffer(Ref: *PerJDObj, RequiresNullTerminator: false), I: std::move(*I)))
285 return Err;
286
287 if (!Bootstrapping) {
288 auto ImportedLibs = StaticVCRuntime
289 ? VCRuntimeBootstrap->loadStaticVCRuntime(JD)
290 : VCRuntimeBootstrap->loadDynamicVCRuntime(JD);
291 if (!ImportedLibs)
292 return ImportedLibs.takeError();
293 for (auto &Lib : *ImportedLibs)
294 if (auto Err = LoadDynLibrary(JD, Lib))
295 return Err;
296 if (StaticVCRuntime)
297 if (auto Err = VCRuntimeBootstrap->initializeStaticVCRuntime(JD))
298 return Err;
299 }
300
301 JD.addGenerator(DefGenerator: DLLImportDefinitionGenerator::Create(ES, L&: ObjLinkingLayer));
302 return Error::success();
303}
304
305Error COFFPlatform::teardownJITDylib(JITDylib &JD) {
306 std::lock_guard<std::mutex> Lock(PlatformMutex);
307 auto I = JITDylibToHeaderAddr.find(Val: &JD);
308 if (I != JITDylibToHeaderAddr.end()) {
309 assert(HeaderAddrToJITDylib.count(I->second) &&
310 "HeaderAddrToJITDylib missing entry");
311 HeaderAddrToJITDylib.erase(Val: I->second);
312 JITDylibToHeaderAddr.erase(I);
313 }
314 return Error::success();
315}
316
317Error COFFPlatform::notifyAdding(ResourceTracker &RT,
318 const MaterializationUnit &MU) {
319 auto &JD = RT.getJITDylib();
320 const auto &InitSym = MU.getInitializerSymbol();
321 if (!InitSym)
322 return Error::success();
323
324 RegisteredInitSymbols[&JD].add(Name: InitSym,
325 Flags: SymbolLookupFlags::WeaklyReferencedSymbol);
326
327 LLVM_DEBUG({
328 dbgs() << "COFFPlatform: Registered init symbol " << *InitSym << " for MU "
329 << MU.getName() << "\n";
330 });
331 return Error::success();
332}
333
334Error COFFPlatform::notifyRemoving(ResourceTracker &RT) {
335 llvm_unreachable("Not supported yet");
336}
337
338SymbolAliasMap COFFPlatform::standardPlatformAliases(ExecutionSession &ES) {
339 SymbolAliasMap Aliases;
340 addAliases(ES, Aliases, AL: standardRuntimeUtilityAliases());
341 return Aliases;
342}
343
344ArrayRef<std::pair<const char *, const char *>>
345COFFPlatform::requiredCXXAliases() {
346 static const std::pair<const char *, const char *> RequiredCXXAliases[] = {
347 {"_CxxThrowException", "__orc_rt_coff_cxx_throw_exception"},
348 {"_onexit", "__orc_rt_coff_onexit_per_jd"},
349 {"atexit", "__orc_rt_coff_atexit_per_jd"}};
350
351 return ArrayRef<std::pair<const char *, const char *>>(RequiredCXXAliases);
352}
353
354ArrayRef<std::pair<const char *, const char *>>
355COFFPlatform::standardRuntimeUtilityAliases() {
356 static const std::pair<const char *, const char *>
357 StandardRuntimeUtilityAliases[] = {
358 {"__orc_rt_run_program", "__orc_rt_coff_run_program"},
359 {"__orc_rt_jit_dlerror", "__orc_rt_coff_jit_dlerror"},
360 {"__orc_rt_jit_dlopen", "__orc_rt_coff_jit_dlopen"},
361 {"__orc_rt_jit_dlupdate", "__orc_rt_coff_jit_dlupdate"},
362 {"__orc_rt_jit_dlclose", "__orc_rt_coff_jit_dlclose"},
363 {"__orc_rt_jit_dlsym", "__orc_rt_coff_jit_dlsym"},
364 {"__orc_rt_log_error", "__orc_rt_log_error_to_stderr"}};
365
366 return ArrayRef<std::pair<const char *, const char *>>(
367 StandardRuntimeUtilityAliases);
368}
369
370bool COFFPlatform::supportedTarget(const Triple &TT) {
371 switch (TT.getArch()) {
372 case Triple::x86_64:
373 return true;
374 default:
375 return false;
376 }
377}
378
379COFFPlatform::COFFPlatform(
380 ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD,
381 std::unique_ptr<StaticLibraryDefinitionGenerator> OrcRuntimeGenerator,
382 std::set<std::string> DylibsToPreload,
383 std::unique_ptr<MemoryBuffer> OrcRuntimeArchiveBuffer,
384 std::unique_ptr<object::Archive> OrcRuntimeArchive,
385 LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime,
386 const char *VCRuntimePath, Error &Err)
387 : ES(ObjLinkingLayer.getExecutionSession()),
388 ObjLinkingLayer(ObjLinkingLayer),
389 LoadDynLibrary(std::move(LoadDynLibrary)),
390 OrcRuntimeArchiveBuffer(std::move(OrcRuntimeArchiveBuffer)),
391 OrcRuntimeArchive(std::move(OrcRuntimeArchive)),
392 StaticVCRuntime(StaticVCRuntime),
393 COFFHeaderStartSymbol(ES.intern(SymName: "__ImageBase")) {
394 ErrorAsOutParameter _(Err);
395
396 Bootstrapping.store(i: true);
397 ObjLinkingLayer.addPlugin(P: std::make_unique<COFFPlatformPlugin>(args&: *this));
398
399 // Load vc runtime
400 auto VCRT =
401 COFFVCRuntimeBootstrapper::Create(ES, ObjLinkingLayer, RuntimePath: VCRuntimePath);
402 if (!VCRT) {
403 Err = VCRT.takeError();
404 return;
405 }
406 VCRuntimeBootstrap = std::move(*VCRT);
407
408 auto ImportedLibs =
409 StaticVCRuntime ? VCRuntimeBootstrap->loadStaticVCRuntime(JD&: PlatformJD)
410 : VCRuntimeBootstrap->loadDynamicVCRuntime(JD&: PlatformJD);
411 if (!ImportedLibs) {
412 Err = ImportedLibs.takeError();
413 return;
414 }
415
416 for (auto &Lib : *ImportedLibs)
417 DylibsToPreload.insert(x: Lib);
418
419 PlatformJD.addGenerator(DefGenerator: std::move(OrcRuntimeGenerator));
420
421 // PlatformJD hasn't been set up by the platform yet (since we're creating
422 // the platform now), so set it up.
423 if (auto E2 = setupJITDylib(PlatformJD)) {
424 Err = std::move(E2);
425 return;
426 }
427
428 for (auto& Lib : DylibsToPreload)
429 if (auto E2 = this->LoadDynLibrary(PlatformJD, Lib)) {
430 Err = std::move(E2);
431 return;
432 }
433
434 if (StaticVCRuntime)
435 if (auto E2 = VCRuntimeBootstrap->initializeStaticVCRuntime(JD&: PlatformJD)) {
436 Err = std::move(E2);
437 return;
438 }
439
440 // Associate wrapper function tags with JIT-side function implementations.
441 if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) {
442 Err = std::move(E2);
443 return;
444 }
445
446 // Lookup addresses of runtime functions callable by the platform,
447 // call the platform bootstrap function to initialize the platform-state
448 // object in the executor.
449 if (auto E2 = bootstrapCOFFRuntime(PlatformJD)) {
450 Err = std::move(E2);
451 return;
452 }
453
454 Bootstrapping.store(i: false);
455 JDBootstrapStates.clear();
456}
457
458Expected<COFFPlatform::JITDylibDepMap>
459COFFPlatform::buildJDDepMap(JITDylib &JD) {
460 return ES.runSessionLocked(F: [&]() -> Expected<JITDylibDepMap> {
461 JITDylibDepMap JDDepMap;
462
463 SmallVector<JITDylib *, 16> Worklist({&JD});
464 while (!Worklist.empty()) {
465 auto CurJD = Worklist.back();
466 Worklist.pop_back();
467
468 auto &DM = JDDepMap[CurJD];
469 CurJD->withLinkOrderDo(F: [&](const JITDylibSearchOrder &O) {
470 DM.reserve(N: O.size());
471 for (auto &KV : O) {
472 if (KV.first == CurJD)
473 continue;
474 {
475 // Bare jitdylibs not known to the platform
476 std::lock_guard<std::mutex> Lock(PlatformMutex);
477 if (!JITDylibToHeaderAddr.count(Val: KV.first)) {
478 LLVM_DEBUG({
479 dbgs() << "JITDylib unregistered to COFFPlatform detected in "
480 "LinkOrder: "
481 << CurJD->getName() << "\n";
482 });
483 continue;
484 }
485 }
486 DM.push_back(Elt: KV.first);
487 // Push unvisited entry.
488 if (JDDepMap.try_emplace(Key: KV.first).second)
489 Worklist.push_back(Elt: KV.first);
490 }
491 });
492 }
493 return std::move(JDDepMap);
494 });
495}
496
497void COFFPlatform::pushInitializersLoop(PushInitializersSendResultFn SendResult,
498 JITDylibSP JD,
499 JITDylibDepMap &JDDepMap) {
500 SmallVector<JITDylib *, 16> Worklist({JD.get()});
501 DenseSet<JITDylib *> Visited({JD.get()});
502 DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols;
503 ES.runSessionLocked(F: [&]() {
504 while (!Worklist.empty()) {
505 auto CurJD = Worklist.back();
506 Worklist.pop_back();
507
508 auto RISItr = RegisteredInitSymbols.find(Val: CurJD);
509 if (RISItr != RegisteredInitSymbols.end()) {
510 NewInitSymbols[CurJD] = std::move(RISItr->second);
511 RegisteredInitSymbols.erase(I: RISItr);
512 }
513
514 for (auto *DepJD : JDDepMap[CurJD])
515 if (Visited.insert(V: DepJD).second)
516 Worklist.push_back(Elt: DepJD);
517 }
518 });
519
520 // If there are no further init symbols to look up then send the link order
521 // (as a list of header addresses) to the caller.
522 if (NewInitSymbols.empty()) {
523 // Build the dep info map to return.
524 COFFJITDylibDepInfoMap DIM;
525 DIM.reserve(n: JDDepMap.size());
526 for (auto &KV : JDDepMap) {
527 std::lock_guard<std::mutex> Lock(PlatformMutex);
528 COFFJITDylibDepInfo DepInfo;
529 DepInfo.reserve(n: KV.second.size());
530 for (auto &Dep : KV.second) {
531 DepInfo.push_back(x: JITDylibToHeaderAddr[Dep]);
532 }
533 auto H = JITDylibToHeaderAddr[KV.first];
534 DIM.push_back(x: std::make_pair(x&: H, y: std::move(DepInfo)));
535 }
536 SendResult(DIM);
537 return;
538 }
539
540 // Otherwise issue a lookup and re-run this phase when it completes.
541 lookupInitSymbolsAsync(
542 OnComplete: [this, SendResult = std::move(SendResult), &JD,
543 JDDepMap = std::move(JDDepMap)](Error Err) mutable {
544 if (Err)
545 SendResult(std::move(Err));
546 else
547 pushInitializersLoop(SendResult: std::move(SendResult), JD, JDDepMap);
548 },
549 ES, InitSyms: std::move(NewInitSymbols));
550}
551
552void COFFPlatform::rt_pushInitializers(PushInitializersSendResultFn SendResult,
553 ExecutorAddr JDHeaderAddr) {
554 JITDylibSP JD;
555 {
556 std::lock_guard<std::mutex> Lock(PlatformMutex);
557 auto I = HeaderAddrToJITDylib.find(Val: JDHeaderAddr);
558 if (I != HeaderAddrToJITDylib.end())
559 JD = I->second;
560 }
561
562 LLVM_DEBUG({
563 dbgs() << "COFFPlatform::rt_pushInitializers(" << JDHeaderAddr << ") ";
564 if (JD)
565 dbgs() << "pushing initializers for " << JD->getName() << "\n";
566 else
567 dbgs() << "No JITDylib for header address.\n";
568 });
569
570 if (!JD) {
571 SendResult(make_error<StringError>(Args: "No JITDylib with header addr " +
572 formatv(Fmt: "{0:x}", Vals&: JDHeaderAddr),
573 Args: inconvertibleErrorCode()));
574 return;
575 }
576
577 auto JDDepMap = buildJDDepMap(JD&: *JD);
578 if (!JDDepMap) {
579 SendResult(JDDepMap.takeError());
580 return;
581 }
582
583 pushInitializersLoop(SendResult: std::move(SendResult), JD, JDDepMap&: *JDDepMap);
584}
585
586void COFFPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult,
587 ExecutorAddr Handle, StringRef SymbolName) {
588 LLVM_DEBUG(dbgs() << "COFFPlatform::rt_lookupSymbol(\"" << Handle << "\")\n");
589
590 JITDylib *JD = nullptr;
591
592 {
593 std::lock_guard<std::mutex> Lock(PlatformMutex);
594 auto I = HeaderAddrToJITDylib.find(Val: Handle);
595 if (I != HeaderAddrToJITDylib.end())
596 JD = I->second;
597 }
598
599 if (!JD) {
600 LLVM_DEBUG(dbgs() << " No JITDylib for handle " << Handle << "\n");
601 SendResult(make_error<StringError>(Args: "No JITDylib associated with handle " +
602 formatv(Fmt: "{0:x}", Vals&: Handle),
603 Args: inconvertibleErrorCode()));
604 return;
605 }
606
607 // Use functor class to work around XL build compiler issue on AIX.
608 class RtLookupNotifyComplete {
609 public:
610 RtLookupNotifyComplete(SendSymbolAddressFn &&SendResult)
611 : SendResult(std::move(SendResult)) {}
612 void operator()(Expected<SymbolMap> Result) {
613 if (Result) {
614 assert(Result->size() == 1 && "Unexpected result map count");
615 SendResult(Result->begin()->second.getAddress());
616 } else {
617 SendResult(Result.takeError());
618 }
619 }
620
621 private:
622 SendSymbolAddressFn SendResult;
623 };
624
625 ES.lookup(
626 K: LookupKind::DLSym, SearchOrder: {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}},
627 Symbols: SymbolLookupSet(ES.intern(SymName: SymbolName)), RequiredState: SymbolState::Ready,
628 NotifyComplete: RtLookupNotifyComplete(std::move(SendResult)), RegisterDependencies: NoDependenciesToRegister);
629}
630
631Error COFFPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) {
632 ExecutionSession::JITDispatchHandlerAssociationMap WFs;
633
634 using LookupSymbolSPSSig =
635 SPSExpected<SPSExecutorAddr>(SPSExecutorAddr, SPSString);
636 WFs[ES.intern(SymName: "__orc_rt_coff_symbol_lookup_tag")] =
637 ES.wrapAsyncWithSPS<LookupSymbolSPSSig>(Instance: this,
638 Method: &COFFPlatform::rt_lookupSymbol);
639 using PushInitializersSPSSig =
640 SPSExpected<SPSCOFFJITDylibDepInfoMap>(SPSExecutorAddr);
641 WFs[ES.intern(SymName: "__orc_rt_coff_push_initializers_tag")] =
642 ES.wrapAsyncWithSPS<PushInitializersSPSSig>(
643 Instance: this, Method: &COFFPlatform::rt_pushInitializers);
644
645 return ES.registerJITDispatchHandlers(JD&: PlatformJD, WFs: std::move(WFs));
646}
647
648Error COFFPlatform::runBootstrapInitializers(JDBootstrapState &BState) {
649 llvm::sort(C&: BState.Initializers);
650 if (auto Err =
651 runBootstrapSubsectionInitializers(BState, Start: ".CRT$XIA", End: ".CRT$XIZ"))
652 return Err;
653
654 if (auto Err = runSymbolIfExists(PlatformJD&: *BState.JD, SymbolName: "__run_after_c_init"))
655 return Err;
656
657 if (auto Err =
658 runBootstrapSubsectionInitializers(BState, Start: ".CRT$XCA", End: ".CRT$XCZ"))
659 return Err;
660 return Error::success();
661}
662
663Error COFFPlatform::runBootstrapSubsectionInitializers(JDBootstrapState &BState,
664 StringRef Start,
665 StringRef End) {
666 CallInt32VoidProxy CallInitializer;
667 if (auto Err = lookupAndApply(
668 JD&: ES.getBootstrapJITDylib(),
669 PrepareFns: {recordProxy<sps::CallInt32VoidProxySpec>(P: &CallInitializer)}))
670 return Err;
671 for (auto &Initializer : BState.Initializers)
672 if (Initializer.first >= Start && Initializer.first <= End &&
673 Initializer.second) {
674 auto Res = CallInitializer(ES, Initializer.second);
675 if (!Res)
676 return Res.takeError();
677 }
678 return Error::success();
679}
680
681Error COFFPlatform::bootstrapCOFFRuntime(JITDylib &PlatformJD) {
682 // Lookup of runtime symbols causes the collection of initializers if
683 // it's static linking setting.
684 if (auto Err = lookupAndApply(
685 JD&: PlatformJD,
686 PrepareFns: {recordAddr(
687 Name: SymbolNameSpec::verbatim(Name: "__orc_rt_coff_platform_bootstrap"),
688 A: &orc_rt_coff_platform_bootstrap),
689 recordAddr(
690 Name: SymbolNameSpec::verbatim(Name: "__orc_rt_coff_platform_shutdown"),
691 A: &orc_rt_coff_platform_shutdown),
692 recordAddr(
693 Name: SymbolNameSpec::verbatim(Name: "__orc_rt_coff_register_jitdylib"),
694 A: &orc_rt_coff_register_jitdylib),
695 recordAddr(
696 Name: SymbolNameSpec::verbatim(Name: "__orc_rt_coff_deregister_jitdylib"),
697 A: &orc_rt_coff_deregister_jitdylib),
698 recordAddr(Name: SymbolNameSpec::verbatim(
699 Name: "__orc_rt_coff_register_object_sections"),
700 A: &orc_rt_coff_register_object_sections),
701 recordAddr(Name: SymbolNameSpec::verbatim(
702 Name: "__orc_rt_coff_deregister_object_sections"),
703 A: &orc_rt_coff_deregister_object_sections)}))
704 return Err;
705
706 // Call bootstrap functions
707 if (auto Err = ES.callSPSWrapper<void()>(WrapperFnAddr: orc_rt_coff_platform_bootstrap))
708 return Err;
709
710 // Do the pending jitdylib registration actions that we couldn't do
711 // because orc runtime was not linked fully.
712 for (auto KV : JDBootstrapStates) {
713 auto &JDBState = KV.second;
714 if (auto Err = ES.callSPSWrapper<void(SPSString, SPSExecutorAddr)>(
715 WrapperFnAddr: orc_rt_coff_register_jitdylib, WrapperCallArgs&: JDBState.JDName,
716 WrapperCallArgs&: JDBState.HeaderAddr))
717 return Err;
718
719 for (auto &ObjSectionMap : JDBState.ObjectSectionsMaps)
720 if (auto Err = ES.callSPSWrapper<void(SPSExecutorAddr,
721 SPSCOFFObjectSectionsMap, bool)>(
722 WrapperFnAddr: orc_rt_coff_register_object_sections, WrapperCallArgs&: JDBState.HeaderAddr,
723 WrapperCallArgs&: ObjSectionMap, WrapperCallArgs: false))
724 return Err;
725 }
726
727 // Run static initializers collected in bootstrap stage.
728 for (auto KV : JDBootstrapStates) {
729 auto &JDBState = KV.second;
730 if (auto Err = runBootstrapInitializers(BState&: JDBState))
731 return Err;
732 }
733
734 return Error::success();
735}
736
737Error COFFPlatform::runSymbolIfExists(JITDylib &PlatformJD,
738 StringRef SymbolName) {
739 ExecutorAddr TargetFn;
740 if (auto Err = lookupAndApply(
741 JD&: PlatformJD,
742 PrepareFns: {recordAddr(Name: SymbolNameSpec::verbatim(Name: SymbolName), A: &TargetFn,
743 LF: SymbolLookupFlags::WeaklyReferencedSymbol)}))
744 return Err;
745 if (!TargetFn)
746 return Error::success(); // No target function.
747
748 CallInt32VoidProxy CallFn;
749 if (auto Err =
750 lookupAndApply(JD&: ES.getBootstrapJITDylib(),
751 PrepareFns: {recordProxy<sps::CallInt32VoidProxySpec>(P: &CallFn)}))
752 return Err;
753
754 return CallFn(ES, TargetFn).takeError();
755}
756
757void COFFPlatform::COFFPlatformPlugin::modifyPassConfig(
758 MaterializationResponsibility &MR, jitlink::LinkGraph &LG,
759 jitlink::PassConfiguration &Config) {
760
761 bool IsBootstrapping = CP.Bootstrapping.load();
762
763 if (auto InitSymbol = MR.getInitializerSymbol()) {
764 if (InitSymbol == CP.COFFHeaderStartSymbol) {
765 Config.PostAllocationPasses.push_back(
766 x: [this, &MR, IsBootstrapping](jitlink::LinkGraph &G) {
767 return associateJITDylibHeaderSymbol(G, MR, Bootstrap: IsBootstrapping);
768 });
769 return;
770 }
771 Config.PrePrunePasses.push_back(x: [this, &MR](jitlink::LinkGraph &G) {
772 return preserveInitializerSections(G, MR);
773 });
774 }
775
776 if (!IsBootstrapping)
777 Config.PostFixupPasses.push_back(
778 x: [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
779 return registerObjectPlatformSections(G, JD);
780 });
781 else
782 Config.PostFixupPasses.push_back(
783 x: [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
784 return registerObjectPlatformSectionsInBootstrap(G, JD);
785 });
786}
787
788Error COFFPlatform::COFFPlatformPlugin::associateJITDylibHeaderSymbol(
789 jitlink::LinkGraph &G, MaterializationResponsibility &MR,
790 bool IsBootstraping) {
791 auto I = llvm::find_if(Range: G.defined_symbols(), P: [this](jitlink::Symbol *Sym) {
792 return *Sym->getName() == *CP.COFFHeaderStartSymbol;
793 });
794 assert(I != G.defined_symbols().end() && "Missing COFF header start symbol");
795
796 auto &JD = MR.getTargetJITDylib();
797 std::lock_guard<std::mutex> Lock(CP.PlatformMutex);
798 auto HeaderAddr = (*I)->getAddress();
799 CP.JITDylibToHeaderAddr[&JD] = HeaderAddr;
800 CP.HeaderAddrToJITDylib[HeaderAddr] = &JD;
801 if (!IsBootstraping) {
802 G.allocActions().push_back(
803 x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<
804 SPSArgList<SPSString, SPSExecutorAddr>>(
805 FnAddr: CP.orc_rt_coff_register_jitdylib, Args: JD.getName(), Args: HeaderAddr)),
806 .Dealloc: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
807 FnAddr: CP.orc_rt_coff_deregister_jitdylib, Args: HeaderAddr))});
808 } else {
809 G.allocActions().push_back(
810 x: {.Finalize: {},
811 .Dealloc: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
812 FnAddr: CP.orc_rt_coff_deregister_jitdylib, Args: HeaderAddr))});
813 JDBootstrapState BState;
814 BState.JD = &JD;
815 BState.JDName = JD.getName();
816 BState.HeaderAddr = HeaderAddr;
817 CP.JDBootstrapStates.emplace(args: &JD, args&: BState);
818 }
819
820 return Error::success();
821}
822
823Error COFFPlatform::COFFPlatformPlugin::registerObjectPlatformSections(
824 jitlink::LinkGraph &G, JITDylib &JD) {
825 COFFObjectSectionsMap ObjSecs;
826 auto HeaderAddr = CP.JITDylibToHeaderAddr[&JD];
827 assert(HeaderAddr && "Must be registered jitdylib");
828 for (auto &S : G.sections()) {
829 jitlink::SectionRange Range(S);
830 if (Range.getSize())
831 ObjSecs.push_back(Elt: std::make_pair(x: S.getName().str(), y: Range.getRange()));
832 }
833
834 G.allocActions().push_back(
835 x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSCOFFRegisterObjectSectionsArgs>(
836 FnAddr: CP.orc_rt_coff_register_object_sections, Args: HeaderAddr, Args: ObjSecs, Args: true)),
837 .Dealloc: cantFail(
838 ValOrErr: WrapperFunctionCall::Create<SPSCOFFDeregisterObjectSectionsArgs>(
839 FnAddr: CP.orc_rt_coff_deregister_object_sections, Args: HeaderAddr,
840 Args: ObjSecs))});
841
842 return Error::success();
843}
844
845Error COFFPlatform::COFFPlatformPlugin::preserveInitializerSections(
846 jitlink::LinkGraph &G, MaterializationResponsibility &MR) {
847
848 if (const auto &InitSymName = MR.getInitializerSymbol()) {
849
850 jitlink::Symbol *InitSym = nullptr;
851
852 for (auto &InitSection : G.sections()) {
853 // Skip non-init sections.
854 if (!isCOFFInitializerSection(Name: InitSection.getName()) ||
855 InitSection.empty())
856 continue;
857
858 // Create the init symbol if it has not been created already and attach it
859 // to the first block.
860 if (!InitSym) {
861 auto &B = **InitSection.blocks().begin();
862 InitSym = &G.addDefinedSymbol(
863 Content&: B, Offset: 0, Name: *InitSymName, Size: B.getSize(), L: jitlink::Linkage::Strong,
864 S: jitlink::Scope::SideEffectsOnly, IsCallable: false, IsLive: true);
865 }
866
867 // Add keep-alive edges to anonymous symbols in all other init blocks.
868 for (auto *B : InitSection.blocks()) {
869 if (B == &InitSym->getBlock())
870 continue;
871
872 auto &S = G.addAnonymousSymbol(Content&: *B, Offset: 0, Size: B->getSize(), IsCallable: false, IsLive: true);
873 InitSym->getBlock().addEdge(K: jitlink::Edge::KeepAlive, Offset: 0, Target&: S, Addend: 0);
874 }
875 }
876 }
877
878 return Error::success();
879}
880
881Error COFFPlatform::COFFPlatformPlugin::
882 registerObjectPlatformSectionsInBootstrap(jitlink::LinkGraph &G,
883 JITDylib &JD) {
884 std::lock_guard<std::mutex> Lock(CP.PlatformMutex);
885 auto HeaderAddr = CP.JITDylibToHeaderAddr[&JD];
886 COFFObjectSectionsMap ObjSecs;
887 for (auto &S : G.sections()) {
888 jitlink::SectionRange Range(S);
889 if (Range.getSize())
890 ObjSecs.push_back(Elt: std::make_pair(x: S.getName().str(), y: Range.getRange()));
891 }
892
893 G.allocActions().push_back(
894 x: {.Finalize: {},
895 .Dealloc: cantFail(
896 ValOrErr: WrapperFunctionCall::Create<SPSCOFFDeregisterObjectSectionsArgs>(
897 FnAddr: CP.orc_rt_coff_deregister_object_sections, Args: HeaderAddr,
898 Args: ObjSecs))});
899
900 auto &BState = CP.JDBootstrapStates[&JD];
901 BState.ObjectSectionsMaps.push_back(x: std::move(ObjSecs));
902
903 // Collect static initializers
904 for (auto &S : G.sections())
905 if (isCOFFInitializerSection(Name: S.getName()))
906 for (auto *B : S.blocks()) {
907 if (B->edges_empty())
908 continue;
909 for (auto &E : B->edges())
910 BState.Initializers.push_back(Elt: std::make_pair(
911 x: S.getName().str(), y: E.getTarget().getAddress() + E.getAddend()));
912 }
913
914 return Error::success();
915}
916
917} // End namespace orc.
918} // End namespace llvm.
919