1//===------- COFFPlatform.cpp - Utilities for executing COFF in Orc -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/Orc/COFFPlatform.h"
10
11#include "llvm/ExecutionEngine/Orc/AbsoluteSymbols.h"
12#include "llvm/ExecutionEngine/Orc/COFF.h"
13#include "llvm/ExecutionEngine/Orc/DebugUtils.h"
14#include "llvm/ExecutionEngine/Orc/LookupAndRecordAddrs.h"
15#include "llvm/ExecutionEngine/Orc/ObjectFileInterface.h"
16#include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h"
17
18#include "llvm/Object/COFF.h"
19
20#include "llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h"
21
22#include "llvm/ExecutionEngine/JITLink/x86_64.h"
23
24#define DEBUG_TYPE "orc"
25
26using namespace llvm;
27using namespace llvm::orc;
28using namespace llvm::orc::shared;
29
30namespace llvm {
31namespace orc {
32namespace shared {
33
34using SPSCOFFJITDylibDepInfo = SPSSequence<SPSExecutorAddr>;
35using SPSCOFFJITDylibDepInfoMap =
36 SPSSequence<SPSTuple<SPSExecutorAddr, SPSCOFFJITDylibDepInfo>>;
37using SPSCOFFObjectSectionsMap =
38 SPSSequence<SPSTuple<SPSString, SPSExecutorAddrRange>>;
39using SPSCOFFRegisterObjectSectionsArgs =
40 SPSArgList<SPSExecutorAddr, SPSCOFFObjectSectionsMap, bool>;
41using SPSCOFFDeregisterObjectSectionsArgs =
42 SPSArgList<SPSExecutorAddr, SPSCOFFObjectSectionsMap>;
43
44} // namespace shared
45} // namespace orc
46} // namespace llvm
47namespace {
48
49class COFFHeaderMaterializationUnit : public MaterializationUnit {
50public:
51 COFFHeaderMaterializationUnit(COFFPlatform &CP,
52 const SymbolStringPtr &HeaderStartSymbol)
53 : MaterializationUnit(createHeaderInterface(MOP&: CP, HeaderStartSymbol)),
54 CP(CP) {}
55
56 StringRef getName() const override { return "COFFHeaderMU"; }
57
58 void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
59 auto G = std::make_unique<jitlink::LinkGraph>(
60 args: "<COFFHeaderMU>", args: CP.getExecutionSession().getSymbolStringPool(),
61 args: CP.getExecutionSession().getTargetTriple(), args: SubtargetFeatures(),
62 args&: jitlink::getGenericEdgeKindName);
63 auto &HeaderSection = G->createSection(Name: "__header", Prot: MemProt::Read);
64 auto &HeaderBlock = createHeaderBlock(G&: *G, HeaderSection);
65
66 // Init symbol is __ImageBase symbol.
67 auto &ImageBaseSymbol = G->addDefinedSymbol(
68 Content&: HeaderBlock, Offset: 0, Name: *R->getInitializerSymbol(), Size: HeaderBlock.getSize(),
69 L: jitlink::Linkage::Strong, S: jitlink::Scope::Default, IsCallable: false, IsLive: true);
70
71 addImageBaseRelocationEdge(B&: HeaderBlock, ImageBase&: ImageBaseSymbol);
72
73 CP.getObjectLinkingLayer().emit(R: std::move(R), G: std::move(G));
74 }
75
76 void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
77
78private:
79 struct HeaderSymbol {
80 const char *Name;
81 uint64_t Offset;
82 };
83
84 struct NTHeader {
85 support::ulittle32_t PEMagic;
86 object::coff_file_header FileHeader;
87 struct PEHeader {
88 object::pe32plus_header Header;
89 object::data_directory DataDirectory[COFF::NUM_DATA_DIRECTORIES + 1];
90 } OptionalHeader;
91 };
92
93 struct HeaderBlockContent {
94 object::dos_header DOSHeader;
95 COFFHeaderMaterializationUnit::NTHeader NTHeader;
96 };
97
98 static jitlink::Block &createHeaderBlock(jitlink::LinkGraph &G,
99 jitlink::Section &HeaderSection) {
100 HeaderBlockContent Hdr = {};
101
102 // Set up magic
103 Hdr.DOSHeader.Magic[0] = 'M';
104 Hdr.DOSHeader.Magic[1] = 'Z';
105 Hdr.DOSHeader.AddressOfNewExeHeader =
106 offsetof(HeaderBlockContent, NTHeader);
107 uint32_t PEMagic = *reinterpret_cast<const uint32_t *>(COFF::PEMagic);
108 Hdr.NTHeader.PEMagic = PEMagic;
109 Hdr.NTHeader.OptionalHeader.Header.Magic = COFF::PE32Header::PE32_PLUS;
110
111 switch (G.getTargetTriple().getArch()) {
112 case Triple::x86_64:
113 Hdr.NTHeader.FileHeader.Machine = COFF::IMAGE_FILE_MACHINE_AMD64;
114 break;
115 default:
116 llvm_unreachable("Unrecognized architecture");
117 }
118
119 auto HeaderContent = G.allocateContent(
120 Source: ArrayRef<char>(reinterpret_cast<const char *>(&Hdr), sizeof(Hdr)));
121
122 return G.createContentBlock(Parent&: HeaderSection, Content: HeaderContent, Address: ExecutorAddr(), Alignment: 8,
123 AlignmentOffset: 0);
124 }
125
126 static void addImageBaseRelocationEdge(jitlink::Block &B,
127 jitlink::Symbol &ImageBase) {
128 auto ImageBaseOffset = offsetof(HeaderBlockContent, NTHeader) +
129 offsetof(NTHeader, OptionalHeader) +
130 offsetof(object::pe32plus_header, ImageBase);
131 B.addEdge(K: jitlink::x86_64::Pointer64, Offset: ImageBaseOffset, Target&: ImageBase, Addend: 0);
132 }
133
134 static MaterializationUnit::Interface
135 createHeaderInterface(COFFPlatform &MOP,
136 const SymbolStringPtr &HeaderStartSymbol) {
137 SymbolFlagsMap HeaderSymbolFlags;
138
139 HeaderSymbolFlags[HeaderStartSymbol] = JITSymbolFlags::Exported;
140
141 return MaterializationUnit::Interface(std::move(HeaderSymbolFlags),
142 HeaderStartSymbol);
143 }
144
145 COFFPlatform &CP;
146};
147
148} // end anonymous namespace
149
150namespace llvm {
151namespace orc {
152
153Expected<std::unique_ptr<COFFPlatform>>
154COFFPlatform::Create(ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD,
155 std::unique_ptr<MemoryBuffer> OrcRuntimeArchiveBuffer,
156 LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime,
157 const char *VCRuntimePath,
158 std::optional<SymbolAliasMap> RuntimeAliases) {
159
160 auto &ES = ObjLinkingLayer.getExecutionSession();
161
162 // If the target is not supported then bail out immediately.
163 if (!supportedTarget(TT: ES.getTargetTriple()))
164 return make_error<StringError>(Args: "Unsupported COFFPlatform triple: " +
165 ES.getTargetTriple().str(),
166 Args: inconvertibleErrorCode());
167
168 auto &EPC = ES.getExecutorProcessControl();
169
170 auto GeneratorArchive =
171 object::Archive::create(Source: OrcRuntimeArchiveBuffer->getMemBufferRef());
172 if (!GeneratorArchive)
173 return GeneratorArchive.takeError();
174
175 std::set<std::string> DylibsToPreload;
176 auto OrcRuntimeArchiveGenerator = StaticLibraryDefinitionGenerator::Create(
177 L&: ObjLinkingLayer, ArchiveBuffer: nullptr, Archive: std::move(*GeneratorArchive),
178 VisitMembers: COFFImportFileScanner(DylibsToPreload));
179 if (!OrcRuntimeArchiveGenerator)
180 return OrcRuntimeArchiveGenerator.takeError();
181
182 // We need a second instance of the archive (for now) for the Platform. We
183 // can `cantFail` this call, since if it were going to fail it would have
184 // failed above.
185 auto RuntimeArchive = cantFail(
186 ValOrErr: object::Archive::create(Source: OrcRuntimeArchiveBuffer->getMemBufferRef()));
187
188 // Create default aliases if the caller didn't supply any.
189 if (!RuntimeAliases)
190 RuntimeAliases = standardPlatformAliases(ES);
191
192 // Define the aliases.
193 if (auto Err = PlatformJD.define(MU: symbolAliases(Aliases: std::move(*RuntimeAliases))))
194 return std::move(Err);
195
196 auto &HostFuncJD = ES.createBareJITDylib(Name: "$<PlatformRuntimeHostFuncJD>");
197
198 // Add JIT-dispatch function support symbols.
199 if (auto Err = HostFuncJD.define(
200 MU: absoluteSymbols(Symbols: {{ES.intern(SymName: "__orc_rt_jit_dispatch"),
201 {EPC.getJITDispatchInfo().JITDispatchFunction,
202 JITSymbolFlags::Exported}},
203 {ES.intern(SymName: "__orc_rt_jit_dispatch_ctx"),
204 {EPC.getJITDispatchInfo().JITDispatchContext,
205 JITSymbolFlags::Exported}}})))
206 return std::move(Err);
207
208 PlatformJD.addToLinkOrder(JD&: HostFuncJD);
209
210 // Create the instance.
211 Error Err = Error::success();
212 auto P = std::unique_ptr<COFFPlatform>(new COFFPlatform(
213 ObjLinkingLayer, PlatformJD, std::move(*OrcRuntimeArchiveGenerator),
214 std::move(DylibsToPreload), std::move(OrcRuntimeArchiveBuffer),
215 std::move(RuntimeArchive), std::move(LoadDynLibrary), StaticVCRuntime,
216 VCRuntimePath, Err));
217 if (Err)
218 return std::move(Err);
219 return std::move(P);
220}
221
222Expected<std::unique_ptr<COFFPlatform>>
223COFFPlatform::Create(ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD,
224 const char *OrcRuntimePath,
225 LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime,
226 const char *VCRuntimePath,
227 std::optional<SymbolAliasMap> RuntimeAliases) {
228
229 auto ArchiveBuffer = MemoryBuffer::getFile(Filename: OrcRuntimePath);
230 if (!ArchiveBuffer)
231 return createFileError(F: OrcRuntimePath, EC: ArchiveBuffer.getError());
232
233 return Create(ObjLinkingLayer, PlatformJD, OrcRuntimeArchiveBuffer: std::move(*ArchiveBuffer),
234 LoadDynLibrary: std::move(LoadDynLibrary), StaticVCRuntime, VCRuntimePath,
235 RuntimeAliases: std::move(RuntimeAliases));
236}
237
238Expected<MemoryBufferRef> COFFPlatform::getPerJDObjectFile() {
239 auto PerJDObj = OrcRuntimeArchive->findSym(name: "__orc_rt_coff_per_jd_marker");
240 if (!PerJDObj)
241 return PerJDObj.takeError();
242
243 if (!*PerJDObj)
244 return make_error<StringError>(Args: "Could not find per jd object file",
245 Args: inconvertibleErrorCode());
246
247 auto Buffer = (*PerJDObj)->getAsBinary();
248 if (!Buffer)
249 return Buffer.takeError();
250
251 return (*Buffer)->getMemoryBufferRef();
252}
253
254static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases,
255 ArrayRef<std::pair<const char *, const char *>> AL) {
256 for (auto &KV : AL) {
257 auto AliasName = ES.intern(SymName: KV.first);
258 assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map");
259 Aliases[std::move(AliasName)] = {ES.intern(SymName: KV.second),
260 JITSymbolFlags::Exported};
261 }
262}
263
264Error COFFPlatform::setupJITDylib(JITDylib &JD) {
265 if (auto Err = JD.define(MU: std::make_unique<COFFHeaderMaterializationUnit>(
266 args&: *this, args&: COFFHeaderStartSymbol)))
267 return Err;
268
269 if (auto Err = ES.lookup(SearchOrder: {&JD}, Symbol: COFFHeaderStartSymbol).takeError())
270 return Err;
271
272 // Define the CXX aliases.
273 SymbolAliasMap CXXAliases;
274 addAliases(ES, Aliases&: CXXAliases, AL: requiredCXXAliases());
275 if (auto Err = JD.define(MU: symbolAliases(Aliases: std::move(CXXAliases))))
276 return Err;
277
278 auto PerJDObj = getPerJDObjectFile();
279 if (!PerJDObj)
280 return PerJDObj.takeError();
281
282 auto I = getObjectFileInterface(ES, ObjBuffer: *PerJDObj);
283 if (!I)
284 return I.takeError();
285
286 if (auto Err = ObjLinkingLayer.add(
287 JD, O: MemoryBuffer::getMemBuffer(Ref: *PerJDObj, RequiresNullTerminator: false), I: std::move(*I)))
288 return Err;
289
290 if (!Bootstrapping) {
291 auto ImportedLibs = StaticVCRuntime
292 ? VCRuntimeBootstrap->loadStaticVCRuntime(JD)
293 : VCRuntimeBootstrap->loadDynamicVCRuntime(JD);
294 if (!ImportedLibs)
295 return ImportedLibs.takeError();
296 for (auto &Lib : *ImportedLibs)
297 if (auto Err = LoadDynLibrary(JD, Lib))
298 return Err;
299 if (StaticVCRuntime)
300 if (auto Err = VCRuntimeBootstrap->initializeStaticVCRuntime(JD))
301 return Err;
302 }
303
304 JD.addGenerator(DefGenerator: DLLImportDefinitionGenerator::Create(ES, L&: ObjLinkingLayer));
305 return Error::success();
306}
307
308Error COFFPlatform::teardownJITDylib(JITDylib &JD) {
309 std::lock_guard<std::mutex> Lock(PlatformMutex);
310 auto I = JITDylibToHeaderAddr.find(Val: &JD);
311 if (I != JITDylibToHeaderAddr.end()) {
312 assert(HeaderAddrToJITDylib.count(I->second) &&
313 "HeaderAddrToJITDylib missing entry");
314 HeaderAddrToJITDylib.erase(Val: I->second);
315 JITDylibToHeaderAddr.erase(I);
316 }
317 return Error::success();
318}
319
320Error COFFPlatform::notifyAdding(ResourceTracker &RT,
321 const MaterializationUnit &MU) {
322 auto &JD = RT.getJITDylib();
323 const auto &InitSym = MU.getInitializerSymbol();
324 if (!InitSym)
325 return Error::success();
326
327 RegisteredInitSymbols[&JD].add(Name: InitSym,
328 Flags: SymbolLookupFlags::WeaklyReferencedSymbol);
329
330 LLVM_DEBUG({
331 dbgs() << "COFFPlatform: Registered init symbol " << *InitSym << " for MU "
332 << MU.getName() << "\n";
333 });
334 return Error::success();
335}
336
337Error COFFPlatform::notifyRemoving(ResourceTracker &RT) {
338 llvm_unreachable("Not supported yet");
339}
340
341SymbolAliasMap COFFPlatform::standardPlatformAliases(ExecutionSession &ES) {
342 SymbolAliasMap Aliases;
343 addAliases(ES, Aliases, AL: standardRuntimeUtilityAliases());
344 return Aliases;
345}
346
347ArrayRef<std::pair<const char *, const char *>>
348COFFPlatform::requiredCXXAliases() {
349 static const std::pair<const char *, const char *> RequiredCXXAliases[] = {
350 {"_CxxThrowException", "__orc_rt_coff_cxx_throw_exception"},
351 {"_onexit", "__orc_rt_coff_onexit_per_jd"},
352 {"atexit", "__orc_rt_coff_atexit_per_jd"}};
353
354 return ArrayRef<std::pair<const char *, const char *>>(RequiredCXXAliases);
355}
356
357ArrayRef<std::pair<const char *, const char *>>
358COFFPlatform::standardRuntimeUtilityAliases() {
359 static const std::pair<const char *, const char *>
360 StandardRuntimeUtilityAliases[] = {
361 {"__orc_rt_run_program", "__orc_rt_coff_run_program"},
362 {"__orc_rt_jit_dlerror", "__orc_rt_coff_jit_dlerror"},
363 {"__orc_rt_jit_dlopen", "__orc_rt_coff_jit_dlopen"},
364 {"__orc_rt_jit_dlupdate", "__orc_rt_coff_jit_dlupdate"},
365 {"__orc_rt_jit_dlclose", "__orc_rt_coff_jit_dlclose"},
366 {"__orc_rt_jit_dlsym", "__orc_rt_coff_jit_dlsym"},
367 {"__orc_rt_log_error", "__orc_rt_log_error_to_stderr"}};
368
369 return ArrayRef<std::pair<const char *, const char *>>(
370 StandardRuntimeUtilityAliases);
371}
372
373bool COFFPlatform::supportedTarget(const Triple &TT) {
374 switch (TT.getArch()) {
375 case Triple::x86_64:
376 return true;
377 default:
378 return false;
379 }
380}
381
382COFFPlatform::COFFPlatform(
383 ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD,
384 std::unique_ptr<StaticLibraryDefinitionGenerator> OrcRuntimeGenerator,
385 std::set<std::string> DylibsToPreload,
386 std::unique_ptr<MemoryBuffer> OrcRuntimeArchiveBuffer,
387 std::unique_ptr<object::Archive> OrcRuntimeArchive,
388 LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime,
389 const char *VCRuntimePath, Error &Err)
390 : ES(ObjLinkingLayer.getExecutionSession()),
391 ObjLinkingLayer(ObjLinkingLayer),
392 LoadDynLibrary(std::move(LoadDynLibrary)),
393 OrcRuntimeArchiveBuffer(std::move(OrcRuntimeArchiveBuffer)),
394 OrcRuntimeArchive(std::move(OrcRuntimeArchive)),
395 StaticVCRuntime(StaticVCRuntime),
396 COFFHeaderStartSymbol(ES.intern(SymName: "__ImageBase")) {
397 ErrorAsOutParameter _(Err);
398
399 Bootstrapping.store(i: true);
400 ObjLinkingLayer.addPlugin(P: std::make_unique<COFFPlatformPlugin>(args&: *this));
401
402 // Load vc runtime
403 auto VCRT =
404 COFFVCRuntimeBootstrapper::Create(ES, ObjLinkingLayer, RuntimePath: VCRuntimePath);
405 if (!VCRT) {
406 Err = VCRT.takeError();
407 return;
408 }
409 VCRuntimeBootstrap = std::move(*VCRT);
410
411 auto ImportedLibs =
412 StaticVCRuntime ? VCRuntimeBootstrap->loadStaticVCRuntime(JD&: PlatformJD)
413 : VCRuntimeBootstrap->loadDynamicVCRuntime(JD&: PlatformJD);
414 if (!ImportedLibs) {
415 Err = ImportedLibs.takeError();
416 return;
417 }
418
419 for (auto &Lib : *ImportedLibs)
420 DylibsToPreload.insert(x: Lib);
421
422 PlatformJD.addGenerator(DefGenerator: std::move(OrcRuntimeGenerator));
423
424 // PlatformJD hasn't been set up by the platform yet (since we're creating
425 // the platform now), so set it up.
426 if (auto E2 = setupJITDylib(PlatformJD)) {
427 Err = std::move(E2);
428 return;
429 }
430
431 for (auto& Lib : DylibsToPreload)
432 if (auto E2 = this->LoadDynLibrary(PlatformJD, Lib)) {
433 Err = std::move(E2);
434 return;
435 }
436
437 if (StaticVCRuntime)
438 if (auto E2 = VCRuntimeBootstrap->initializeStaticVCRuntime(JD&: PlatformJD)) {
439 Err = std::move(E2);
440 return;
441 }
442
443 // Associate wrapper function tags with JIT-side function implementations.
444 if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) {
445 Err = std::move(E2);
446 return;
447 }
448
449 // Lookup addresses of runtime functions callable by the platform,
450 // call the platform bootstrap function to initialize the platform-state
451 // object in the executor.
452 if (auto E2 = bootstrapCOFFRuntime(PlatformJD)) {
453 Err = std::move(E2);
454 return;
455 }
456
457 Bootstrapping.store(i: false);
458 JDBootstrapStates.clear();
459}
460
461Expected<COFFPlatform::JITDylibDepMap>
462COFFPlatform::buildJDDepMap(JITDylib &JD) {
463 return ES.runSessionLocked(F: [&]() -> Expected<JITDylibDepMap> {
464 JITDylibDepMap JDDepMap;
465
466 SmallVector<JITDylib *, 16> Worklist({&JD});
467 while (!Worklist.empty()) {
468 auto CurJD = Worklist.back();
469 Worklist.pop_back();
470
471 auto &DM = JDDepMap[CurJD];
472 CurJD->withLinkOrderDo(F: [&](const JITDylibSearchOrder &O) {
473 DM.reserve(N: O.size());
474 for (auto &KV : O) {
475 if (KV.first == CurJD)
476 continue;
477 {
478 // Bare jitdylibs not known to the platform
479 std::lock_guard<std::mutex> Lock(PlatformMutex);
480 if (!JITDylibToHeaderAddr.count(Val: KV.first)) {
481 LLVM_DEBUG({
482 dbgs() << "JITDylib unregistered to COFFPlatform detected in "
483 "LinkOrder: "
484 << CurJD->getName() << "\n";
485 });
486 continue;
487 }
488 }
489 DM.push_back(Elt: KV.first);
490 // Push unvisited entry.
491 if (JDDepMap.try_emplace(Key: KV.first).second)
492 Worklist.push_back(Elt: KV.first);
493 }
494 });
495 }
496 return std::move(JDDepMap);
497 });
498}
499
500void COFFPlatform::pushInitializersLoop(PushInitializersSendResultFn SendResult,
501 JITDylibSP JD,
502 JITDylibDepMap &JDDepMap) {
503 SmallVector<JITDylib *, 16> Worklist({JD.get()});
504 DenseSet<JITDylib *> Visited({JD.get()});
505 DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols;
506 ES.runSessionLocked(F: [&]() {
507 while (!Worklist.empty()) {
508 auto CurJD = Worklist.back();
509 Worklist.pop_back();
510
511 auto RISItr = RegisteredInitSymbols.find(Val: CurJD);
512 if (RISItr != RegisteredInitSymbols.end()) {
513 NewInitSymbols[CurJD] = std::move(RISItr->second);
514 RegisteredInitSymbols.erase(I: RISItr);
515 }
516
517 for (auto *DepJD : JDDepMap[CurJD])
518 if (Visited.insert(V: DepJD).second)
519 Worklist.push_back(Elt: DepJD);
520 }
521 });
522
523 // If there are no further init symbols to look up then send the link order
524 // (as a list of header addresses) to the caller.
525 if (NewInitSymbols.empty()) {
526 // Build the dep info map to return.
527 COFFJITDylibDepInfoMap DIM;
528 DIM.reserve(n: JDDepMap.size());
529 for (auto &KV : JDDepMap) {
530 std::lock_guard<std::mutex> Lock(PlatformMutex);
531 COFFJITDylibDepInfo DepInfo;
532 DepInfo.reserve(n: KV.second.size());
533 for (auto &Dep : KV.second) {
534 DepInfo.push_back(x: JITDylibToHeaderAddr[Dep]);
535 }
536 auto H = JITDylibToHeaderAddr[KV.first];
537 DIM.push_back(x: std::make_pair(x&: H, y: std::move(DepInfo)));
538 }
539 SendResult(DIM);
540 return;
541 }
542
543 // Otherwise issue a lookup and re-run this phase when it completes.
544 lookupInitSymbolsAsync(
545 OnComplete: [this, SendResult = std::move(SendResult), &JD,
546 JDDepMap = std::move(JDDepMap)](Error Err) mutable {
547 if (Err)
548 SendResult(std::move(Err));
549 else
550 pushInitializersLoop(SendResult: std::move(SendResult), JD, JDDepMap);
551 },
552 ES, InitSyms: std::move(NewInitSymbols));
553}
554
555void COFFPlatform::rt_pushInitializers(PushInitializersSendResultFn SendResult,
556 ExecutorAddr JDHeaderAddr) {
557 JITDylibSP JD;
558 {
559 std::lock_guard<std::mutex> Lock(PlatformMutex);
560 auto I = HeaderAddrToJITDylib.find(Val: JDHeaderAddr);
561 if (I != HeaderAddrToJITDylib.end())
562 JD = I->second;
563 }
564
565 LLVM_DEBUG({
566 dbgs() << "COFFPlatform::rt_pushInitializers(" << JDHeaderAddr << ") ";
567 if (JD)
568 dbgs() << "pushing initializers for " << JD->getName() << "\n";
569 else
570 dbgs() << "No JITDylib for header address.\n";
571 });
572
573 if (!JD) {
574 SendResult(make_error<StringError>(Args: "No JITDylib with header addr " +
575 formatv(Fmt: "{0:x}", Vals&: JDHeaderAddr),
576 Args: inconvertibleErrorCode()));
577 return;
578 }
579
580 auto JDDepMap = buildJDDepMap(JD&: *JD);
581 if (!JDDepMap) {
582 SendResult(JDDepMap.takeError());
583 return;
584 }
585
586 pushInitializersLoop(SendResult: std::move(SendResult), JD, JDDepMap&: *JDDepMap);
587}
588
589void COFFPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult,
590 ExecutorAddr Handle, StringRef SymbolName) {
591 LLVM_DEBUG(dbgs() << "COFFPlatform::rt_lookupSymbol(\"" << Handle << "\")\n");
592
593 JITDylib *JD = nullptr;
594
595 {
596 std::lock_guard<std::mutex> Lock(PlatformMutex);
597 auto I = HeaderAddrToJITDylib.find(Val: Handle);
598 if (I != HeaderAddrToJITDylib.end())
599 JD = I->second;
600 }
601
602 if (!JD) {
603 LLVM_DEBUG(dbgs() << " No JITDylib for handle " << Handle << "\n");
604 SendResult(make_error<StringError>(Args: "No JITDylib associated with handle " +
605 formatv(Fmt: "{0:x}", Vals&: Handle),
606 Args: inconvertibleErrorCode()));
607 return;
608 }
609
610 // Use functor class to work around XL build compiler issue on AIX.
611 class RtLookupNotifyComplete {
612 public:
613 RtLookupNotifyComplete(SendSymbolAddressFn &&SendResult)
614 : SendResult(std::move(SendResult)) {}
615 void operator()(Expected<SymbolMap> Result) {
616 if (Result) {
617 assert(Result->size() == 1 && "Unexpected result map count");
618 SendResult(Result->begin()->second.getAddress());
619 } else {
620 SendResult(Result.takeError());
621 }
622 }
623
624 private:
625 SendSymbolAddressFn SendResult;
626 };
627
628 ES.lookup(
629 K: LookupKind::DLSym, SearchOrder: {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}},
630 Symbols: SymbolLookupSet(ES.intern(SymName: SymbolName)), RequiredState: SymbolState::Ready,
631 NotifyComplete: RtLookupNotifyComplete(std::move(SendResult)), RegisterDependencies: NoDependenciesToRegister);
632}
633
634Error COFFPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) {
635 ExecutionSession::JITDispatchHandlerAssociationMap WFs;
636
637 using LookupSymbolSPSSig =
638 SPSExpected<SPSExecutorAddr>(SPSExecutorAddr, SPSString);
639 WFs[ES.intern(SymName: "__orc_rt_coff_symbol_lookup_tag")] =
640 ES.wrapAsyncWithSPS<LookupSymbolSPSSig>(Instance: this,
641 Method: &COFFPlatform::rt_lookupSymbol);
642 using PushInitializersSPSSig =
643 SPSExpected<SPSCOFFJITDylibDepInfoMap>(SPSExecutorAddr);
644 WFs[ES.intern(SymName: "__orc_rt_coff_push_initializers_tag")] =
645 ES.wrapAsyncWithSPS<PushInitializersSPSSig>(
646 Instance: this, Method: &COFFPlatform::rt_pushInitializers);
647
648 return ES.registerJITDispatchHandlers(JD&: PlatformJD, WFs: std::move(WFs));
649}
650
651Error COFFPlatform::runBootstrapInitializers(JDBootstrapState &BState) {
652 llvm::sort(C&: BState.Initializers);
653 if (auto Err =
654 runBootstrapSubsectionInitializers(BState, Start: ".CRT$XIA", End: ".CRT$XIZ"))
655 return Err;
656
657 if (auto Err = runSymbolIfExists(PlatformJD&: *BState.JD, SymbolName: "__run_after_c_init"))
658 return Err;
659
660 if (auto Err =
661 runBootstrapSubsectionInitializers(BState, Start: ".CRT$XCA", End: ".CRT$XCZ"))
662 return Err;
663 return Error::success();
664}
665
666Error COFFPlatform::runBootstrapSubsectionInitializers(JDBootstrapState &BState,
667 StringRef Start,
668 StringRef End) {
669 for (auto &Initializer : BState.Initializers)
670 if (Initializer.first >= Start && Initializer.first <= End &&
671 Initializer.second) {
672 auto Res =
673 ES.getExecutorProcessControl().runAsVoidFunction(VoidFnAddr: Initializer.second);
674 if (!Res)
675 return Res.takeError();
676 }
677 return Error::success();
678}
679
680Error COFFPlatform::bootstrapCOFFRuntime(JITDylib &PlatformJD) {
681 // Lookup of runtime symbols causes the collection of initializers if
682 // it's static linking setting.
683 if (auto Err = lookupAndRecordAddrs(
684 ES, K: LookupKind::Static, SearchOrder: makeJITDylibSearchOrder(JDs: &PlatformJD),
685 Pairs: {
686 {ES.intern(SymName: "__orc_rt_coff_platform_bootstrap"),
687 &orc_rt_coff_platform_bootstrap},
688 {ES.intern(SymName: "__orc_rt_coff_platform_shutdown"),
689 &orc_rt_coff_platform_shutdown},
690 {ES.intern(SymName: "__orc_rt_coff_register_jitdylib"),
691 &orc_rt_coff_register_jitdylib},
692 {ES.intern(SymName: "__orc_rt_coff_deregister_jitdylib"),
693 &orc_rt_coff_deregister_jitdylib},
694 {ES.intern(SymName: "__orc_rt_coff_register_object_sections"),
695 &orc_rt_coff_register_object_sections},
696 {ES.intern(SymName: "__orc_rt_coff_deregister_object_sections"),
697 &orc_rt_coff_deregister_object_sections},
698 }))
699 return Err;
700
701 // Call bootstrap functions
702 if (auto Err = ES.callSPSWrapper<void()>(WrapperFnAddr: orc_rt_coff_platform_bootstrap))
703 return Err;
704
705 // Do the pending jitdylib registration actions that we couldn't do
706 // because orc runtime was not linked fully.
707 for (auto KV : JDBootstrapStates) {
708 auto &JDBState = KV.second;
709 if (auto Err = ES.callSPSWrapper<void(SPSString, SPSExecutorAddr)>(
710 WrapperFnAddr: orc_rt_coff_register_jitdylib, WrapperCallArgs&: JDBState.JDName,
711 WrapperCallArgs&: JDBState.HeaderAddr))
712 return Err;
713
714 for (auto &ObjSectionMap : JDBState.ObjectSectionsMaps)
715 if (auto Err = ES.callSPSWrapper<void(SPSExecutorAddr,
716 SPSCOFFObjectSectionsMap, bool)>(
717 WrapperFnAddr: orc_rt_coff_register_object_sections, WrapperCallArgs&: JDBState.HeaderAddr,
718 WrapperCallArgs&: ObjSectionMap, WrapperCallArgs: false))
719 return Err;
720 }
721
722 // Run static initializers collected in bootstrap stage.
723 for (auto KV : JDBootstrapStates) {
724 auto &JDBState = KV.second;
725 if (auto Err = runBootstrapInitializers(BState&: JDBState))
726 return Err;
727 }
728
729 return Error::success();
730}
731
732Error COFFPlatform::runSymbolIfExists(JITDylib &PlatformJD,
733 StringRef SymbolName) {
734 ExecutorAddr jit_function;
735 auto AfterCLookupErr = lookupAndRecordAddrs(
736 ES, K: LookupKind::Static, SearchOrder: makeJITDylibSearchOrder(JDs: &PlatformJD),
737 Pairs: {{ES.intern(SymName: SymbolName), &jit_function}});
738 if (!AfterCLookupErr) {
739 auto Res = ES.getExecutorProcessControl().runAsVoidFunction(VoidFnAddr: jit_function);
740 if (!Res)
741 return Res.takeError();
742 return Error::success();
743 }
744 if (!AfterCLookupErr.isA<SymbolsNotFound>())
745 return AfterCLookupErr;
746 consumeError(Err: std::move(AfterCLookupErr));
747 return Error::success();
748}
749
750void COFFPlatform::COFFPlatformPlugin::modifyPassConfig(
751 MaterializationResponsibility &MR, jitlink::LinkGraph &LG,
752 jitlink::PassConfiguration &Config) {
753
754 bool IsBootstrapping = CP.Bootstrapping.load();
755
756 if (auto InitSymbol = MR.getInitializerSymbol()) {
757 if (InitSymbol == CP.COFFHeaderStartSymbol) {
758 Config.PostAllocationPasses.push_back(
759 x: [this, &MR, IsBootstrapping](jitlink::LinkGraph &G) {
760 return associateJITDylibHeaderSymbol(G, MR, Bootstrap: IsBootstrapping);
761 });
762 return;
763 }
764 Config.PrePrunePasses.push_back(x: [this, &MR](jitlink::LinkGraph &G) {
765 return preserveInitializerSections(G, MR);
766 });
767 }
768
769 if (!IsBootstrapping)
770 Config.PostFixupPasses.push_back(
771 x: [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
772 return registerObjectPlatformSections(G, JD);
773 });
774 else
775 Config.PostFixupPasses.push_back(
776 x: [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
777 return registerObjectPlatformSectionsInBootstrap(G, JD);
778 });
779}
780
781Error COFFPlatform::COFFPlatformPlugin::associateJITDylibHeaderSymbol(
782 jitlink::LinkGraph &G, MaterializationResponsibility &MR,
783 bool IsBootstraping) {
784 auto I = llvm::find_if(Range: G.defined_symbols(), P: [this](jitlink::Symbol *Sym) {
785 return *Sym->getName() == *CP.COFFHeaderStartSymbol;
786 });
787 assert(I != G.defined_symbols().end() && "Missing COFF header start symbol");
788
789 auto &JD = MR.getTargetJITDylib();
790 std::lock_guard<std::mutex> Lock(CP.PlatformMutex);
791 auto HeaderAddr = (*I)->getAddress();
792 CP.JITDylibToHeaderAddr[&JD] = HeaderAddr;
793 CP.HeaderAddrToJITDylib[HeaderAddr] = &JD;
794 if (!IsBootstraping) {
795 G.allocActions().push_back(
796 x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<
797 SPSArgList<SPSString, SPSExecutorAddr>>(
798 FnAddr: CP.orc_rt_coff_register_jitdylib, Args: JD.getName(), Args: HeaderAddr)),
799 .Dealloc: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
800 FnAddr: CP.orc_rt_coff_deregister_jitdylib, Args: HeaderAddr))});
801 } else {
802 G.allocActions().push_back(
803 x: {.Finalize: {},
804 .Dealloc: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
805 FnAddr: CP.orc_rt_coff_deregister_jitdylib, Args: HeaderAddr))});
806 JDBootstrapState BState;
807 BState.JD = &JD;
808 BState.JDName = JD.getName();
809 BState.HeaderAddr = HeaderAddr;
810 CP.JDBootstrapStates.emplace(args: &JD, args&: BState);
811 }
812
813 return Error::success();
814}
815
816Error COFFPlatform::COFFPlatformPlugin::registerObjectPlatformSections(
817 jitlink::LinkGraph &G, JITDylib &JD) {
818 COFFObjectSectionsMap ObjSecs;
819 auto HeaderAddr = CP.JITDylibToHeaderAddr[&JD];
820 assert(HeaderAddr && "Must be registered jitdylib");
821 for (auto &S : G.sections()) {
822 jitlink::SectionRange Range(S);
823 if (Range.getSize())
824 ObjSecs.push_back(Elt: std::make_pair(x: S.getName().str(), y: Range.getRange()));
825 }
826
827 G.allocActions().push_back(
828 x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSCOFFRegisterObjectSectionsArgs>(
829 FnAddr: CP.orc_rt_coff_register_object_sections, Args: HeaderAddr, Args: ObjSecs, Args: true)),
830 .Dealloc: cantFail(
831 ValOrErr: WrapperFunctionCall::Create<SPSCOFFDeregisterObjectSectionsArgs>(
832 FnAddr: CP.orc_rt_coff_deregister_object_sections, Args: HeaderAddr,
833 Args: ObjSecs))});
834
835 return Error::success();
836}
837
838Error COFFPlatform::COFFPlatformPlugin::preserveInitializerSections(
839 jitlink::LinkGraph &G, MaterializationResponsibility &MR) {
840
841 if (const auto &InitSymName = MR.getInitializerSymbol()) {
842
843 jitlink::Symbol *InitSym = nullptr;
844
845 for (auto &InitSection : G.sections()) {
846 // Skip non-init sections.
847 if (!isCOFFInitializerSection(Name: InitSection.getName()) ||
848 InitSection.empty())
849 continue;
850
851 // Create the init symbol if it has not been created already and attach it
852 // to the first block.
853 if (!InitSym) {
854 auto &B = **InitSection.blocks().begin();
855 InitSym = &G.addDefinedSymbol(
856 Content&: B, Offset: 0, Name: *InitSymName, Size: B.getSize(), L: jitlink::Linkage::Strong,
857 S: jitlink::Scope::SideEffectsOnly, IsCallable: false, IsLive: true);
858 }
859
860 // Add keep-alive edges to anonymous symbols in all other init blocks.
861 for (auto *B : InitSection.blocks()) {
862 if (B == &InitSym->getBlock())
863 continue;
864
865 auto &S = G.addAnonymousSymbol(Content&: *B, Offset: 0, Size: B->getSize(), IsCallable: false, IsLive: true);
866 InitSym->getBlock().addEdge(K: jitlink::Edge::KeepAlive, Offset: 0, Target&: S, Addend: 0);
867 }
868 }
869 }
870
871 return Error::success();
872}
873
874Error COFFPlatform::COFFPlatformPlugin::
875 registerObjectPlatformSectionsInBootstrap(jitlink::LinkGraph &G,
876 JITDylib &JD) {
877 std::lock_guard<std::mutex> Lock(CP.PlatformMutex);
878 auto HeaderAddr = CP.JITDylibToHeaderAddr[&JD];
879 COFFObjectSectionsMap ObjSecs;
880 for (auto &S : G.sections()) {
881 jitlink::SectionRange Range(S);
882 if (Range.getSize())
883 ObjSecs.push_back(Elt: std::make_pair(x: S.getName().str(), y: Range.getRange()));
884 }
885
886 G.allocActions().push_back(
887 x: {.Finalize: {},
888 .Dealloc: cantFail(
889 ValOrErr: WrapperFunctionCall::Create<SPSCOFFDeregisterObjectSectionsArgs>(
890 FnAddr: CP.orc_rt_coff_deregister_object_sections, Args: HeaderAddr,
891 Args: ObjSecs))});
892
893 auto &BState = CP.JDBootstrapStates[&JD];
894 BState.ObjectSectionsMaps.push_back(x: std::move(ObjSecs));
895
896 // Collect static initializers
897 for (auto &S : G.sections())
898 if (isCOFFInitializerSection(Name: S.getName()))
899 for (auto *B : S.blocks()) {
900 if (B->edges_empty())
901 continue;
902 for (auto &E : B->edges())
903 BState.Initializers.push_back(Elt: std::make_pair(
904 x: S.getName().str(), y: E.getTarget().getAddress() + E.getAddend()));
905 }
906
907 return Error::success();
908}
909
910} // End namespace orc.
911} // End namespace llvm.
912