1//===------- COFFPlatform.cpp - Utilities for executing COFF in Orc -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/Orc/COFFPlatform.h"
10#include "llvm/ExecutionEngine/Orc/Mangling.h"
11
12#include "llvm/ExecutionEngine/Orc/BindCallControllerHandlerSPS.h"
13#include "llvm/ExecutionEngine/Orc/COFF.h"
14#include "llvm/ExecutionEngine/Orc/CallProxiesSPS.h"
15#include "llvm/ExecutionEngine/Orc/DebugUtils.h"
16#include "llvm/ExecutionEngine/Orc/LookupAndApply.h"
17#include "llvm/ExecutionEngine/Orc/ObjectFileInterface.h"
18#include "llvm/ExecutionEngine/Orc/RecordProxy.h"
19#include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h"
20#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
21#include "llvm/Object/COFF.h"
22
23#include "llvm/ExecutionEngine/JITLink/x86_64.h"
24
25#define DEBUG_TYPE "orc"
26
27using namespace llvm;
28using namespace llvm::orc;
29using namespace llvm::orc::shared;
30
31namespace llvm {
32namespace orc {
33namespace shared {
34
35using SPSCOFFJITDylibDepInfo = SPSSequence<SPSExecutorAddr>;
36using SPSCOFFJITDylibDepInfoMap =
37 SPSSequence<SPSTuple<SPSExecutorAddr, SPSCOFFJITDylibDepInfo>>;
38using SPSCOFFObjectSectionsMap =
39 SPSSequence<SPSTuple<SPSString, SPSExecutorAddrRange>>;
40using SPSCOFFRegisterObjectSectionsArgs =
41 SPSArgList<SPSExecutorAddr, SPSCOFFObjectSectionsMap, bool>;
42using SPSCOFFDeregisterObjectSectionsArgs =
43 SPSArgList<SPSExecutorAddr, SPSCOFFObjectSectionsMap>;
44
45} // namespace shared
46} // namespace orc
47} // namespace llvm
48// Controller-interface descriptors for the COFF platform runtime's
49// bootstrap-time SPS wrapper calls. Kept in the .cpp (the COFF platform's
50// private contract with its runtime; the SPS arg types live here too), and in
51// a named namespace so the constexpr Name members -- read only as constants by
52// ProxySpec -- don't trip -Wunused-const-variable.
53namespace llvm::orc::coff_sps_ci {
54struct PlatformBootstrap {
55 static constexpr SymbolNameSpec Name =
56 SymbolNameSpec::c(Name: "__orc_rt_coff_platform_bootstrap");
57 using SPSSig = void();
58};
59struct RegisterJITDylib {
60 static constexpr SymbolNameSpec Name =
61 SymbolNameSpec::c(Name: "__orc_rt_coff_register_jitdylib");
62 using SPSSig = void(SPSString, SPSExecutorAddr);
63};
64struct RegisterObjectSections {
65 static constexpr SymbolNameSpec Name =
66 SymbolNameSpec::c(Name: "__orc_rt_coff_register_object_sections");
67 using SPSSig = void(SPSExecutorAddr, SPSCOFFObjectSectionsMap, bool);
68};
69} // namespace llvm::orc::coff_sps_ci
70
71namespace {
72
73class COFFHeaderMaterializationUnit : public MaterializationUnit {
74public:
75 COFFHeaderMaterializationUnit(COFFPlatform &CP,
76 const SymbolStringPtr &HeaderStartSymbol)
77 : MaterializationUnit(createHeaderInterface(MOP&: CP, HeaderStartSymbol)),
78 CP(CP) {}
79
80 StringRef getName() const override { return "COFFHeaderMU"; }
81
82 void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
83 auto G = std::make_unique<jitlink::LinkGraph>(
84 args: "<COFFHeaderMU>", args: CP.getExecutionSession().getSymbolStringPool(),
85 args: CP.getExecutionSession().getTargetTriple(),
86 args: CP.getExecutionSession().getTargetTriple().getArchPointerBitWidth() / 8,
87 args: SubtargetFeatures(), args&: jitlink::getGenericEdgeKindName);
88 auto &HeaderSection = G->createSection(Name: "__header", Prot: MemProt::Read);
89 auto &HeaderBlock = createHeaderBlock(G&: *G, HeaderSection);
90
91 // Init symbol is __ImageBase symbol.
92 auto &ImageBaseSymbol = G->addDefinedSymbol(
93 Content&: HeaderBlock, Offset: 0, Name: *R->getInitializerSymbol(), Size: HeaderBlock.getSize(),
94 L: jitlink::Linkage::Strong, S: jitlink::Scope::Default, IsCallable: false, IsLive: true);
95
96 addImageBaseRelocationEdge(B&: HeaderBlock, ImageBase&: ImageBaseSymbol);
97
98 CP.getObjectLinkingLayer().emit(R: std::move(R), G: std::move(G));
99 }
100
101 void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
102
103private:
104 struct HeaderSymbol {
105 const char *Name;
106 uint64_t Offset;
107 };
108
109 struct NTHeader {
110 support::ulittle32_t PEMagic;
111 object::coff_file_header FileHeader;
112 struct PEHeader {
113 object::pe32plus_header Header;
114 object::data_directory DataDirectory[COFF::NUM_DATA_DIRECTORIES + 1];
115 } OptionalHeader;
116 };
117
118 struct HeaderBlockContent {
119 object::dos_header DOSHeader;
120 COFFHeaderMaterializationUnit::NTHeader NTHeader;
121 };
122
123 static jitlink::Block &createHeaderBlock(jitlink::LinkGraph &G,
124 jitlink::Section &HeaderSection) {
125 HeaderBlockContent Hdr = {};
126
127 // Set up magic
128 Hdr.DOSHeader.Magic[0] = 'M';
129 Hdr.DOSHeader.Magic[1] = 'Z';
130 Hdr.DOSHeader.AddressOfNewExeHeader =
131 offsetof(HeaderBlockContent, NTHeader);
132 uint32_t PEMagic = *reinterpret_cast<const uint32_t *>(COFF::PEMagic);
133 Hdr.NTHeader.PEMagic = PEMagic;
134 Hdr.NTHeader.OptionalHeader.Header.Magic = COFF::PE32Header::PE32_PLUS;
135
136 switch (G.getTargetTriple().getArch()) {
137 case Triple::x86_64:
138 Hdr.NTHeader.FileHeader.Machine = COFF::IMAGE_FILE_MACHINE_AMD64;
139 break;
140 default:
141 llvm_unreachable("Unrecognized architecture");
142 }
143
144 auto HeaderContent = G.allocateContent(
145 Source: ArrayRef<char>(reinterpret_cast<const char *>(&Hdr), sizeof(Hdr)));
146
147 return G.createContentBlock(Parent&: HeaderSection, Content: HeaderContent, Address: ExecutorAddr(), Alignment: 8,
148 AlignmentOffset: 0);
149 }
150
151 static void addImageBaseRelocationEdge(jitlink::Block &B,
152 jitlink::Symbol &ImageBase) {
153 auto ImageBaseOffset = offsetof(HeaderBlockContent, NTHeader) +
154 offsetof(NTHeader, OptionalHeader) +
155 offsetof(object::pe32plus_header, ImageBase);
156 B.addEdge(K: jitlink::x86_64::Pointer64, Offset: ImageBaseOffset, Target&: ImageBase, Addend: 0);
157 }
158
159 static MaterializationUnit::Interface
160 createHeaderInterface(COFFPlatform &MOP,
161 const SymbolStringPtr &HeaderStartSymbol) {
162 SymbolFlagsMap HeaderSymbolFlags;
163
164 HeaderSymbolFlags[HeaderStartSymbol] = JITSymbolFlags::Exported;
165
166 return MaterializationUnit::Interface(std::move(HeaderSymbolFlags),
167 HeaderStartSymbol);
168 }
169
170 COFFPlatform &CP;
171};
172
173} // end anonymous namespace
174
175namespace llvm {
176namespace orc {
177
178Expected<std::unique_ptr<COFFPlatform>>
179COFFPlatform::Create(ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD,
180 std::unique_ptr<MemoryBuffer> OrcRuntimeArchiveBuffer,
181 LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime,
182 const char *VCRuntimePath,
183 std::optional<SymbolAliasMap> RuntimeAliases) {
184
185 auto &ES = ObjLinkingLayer.getExecutionSession();
186
187 // If the target is not supported then bail out immediately.
188 if (!supportedTarget(TT: ES.getTargetTriple()))
189 return make_error<StringError>(Args: "Unsupported COFFPlatform triple: " +
190 ES.getTargetTriple().str(),
191 Args: inconvertibleErrorCode());
192
193 auto GeneratorArchive =
194 object::Archive::create(Source: OrcRuntimeArchiveBuffer->getMemBufferRef());
195 if (!GeneratorArchive)
196 return GeneratorArchive.takeError();
197
198 std::set<std::string> DylibsToPreload;
199 auto OrcRuntimeArchiveGenerator = StaticLibraryDefinitionGenerator::Create(
200 L&: ObjLinkingLayer, ArchiveBuffer: nullptr, Archive: std::move(*GeneratorArchive),
201 VisitMembers: COFFImportFileScanner(DylibsToPreload));
202 if (!OrcRuntimeArchiveGenerator)
203 return OrcRuntimeArchiveGenerator.takeError();
204
205 // We need a second instance of the archive (for now) for the Platform. We
206 // can `cantFail` this call, since if it were going to fail it would have
207 // failed above.
208 auto RuntimeArchive = cantFail(
209 ValOrErr: object::Archive::create(Source: OrcRuntimeArchiveBuffer->getMemBufferRef()));
210
211 // Create default aliases if the caller didn't supply any.
212 if (!RuntimeAliases)
213 RuntimeAliases = standardPlatformAliases(ES);
214
215 // Define the aliases.
216 if (auto Err = PlatformJD.define(MU: symbolAliases(Aliases: std::move(*RuntimeAliases))))
217 return std::move(Err);
218
219 {
220 // Add JIT dispatch reexports from bootstrap JITDylib.
221 MangleAndInterner Mangle(ES);
222 auto Exports = buildSimpleReexportsAliasMap(
223 SourceJD&: ES.getBootstrapJITDylib(),
224 Symbols: {{Mangle(rt::DispatchName), Mangle(rt::DispatchCtxName)}});
225 if (!Exports)
226 return Exports.takeError();
227 if (auto Err =
228 PlatformJD.define(MU: reexports(SourceJD&: ES.getBootstrapJITDylib(), Aliases: *Exports)))
229 return Err;
230 }
231
232 // Create the instance.
233 Error Err = Error::success();
234 auto P = std::unique_ptr<COFFPlatform>(new COFFPlatform(
235 ObjLinkingLayer, PlatformJD, std::move(*OrcRuntimeArchiveGenerator),
236 std::move(DylibsToPreload), std::move(OrcRuntimeArchiveBuffer),
237 std::move(RuntimeArchive), std::move(LoadDynLibrary), StaticVCRuntime,
238 VCRuntimePath, Err));
239 if (Err)
240 return std::move(Err);
241 return std::move(P);
242}
243
244Expected<std::unique_ptr<COFFPlatform>>
245COFFPlatform::Create(ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD,
246 const char *OrcRuntimePath,
247 LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime,
248 const char *VCRuntimePath,
249 std::optional<SymbolAliasMap> RuntimeAliases) {
250
251 auto ArchiveBuffer = MemoryBuffer::getFile(Filename: OrcRuntimePath);
252 if (!ArchiveBuffer)
253 return createFileError(F: OrcRuntimePath, EC: ArchiveBuffer.getError());
254
255 return Create(ObjLinkingLayer, PlatformJD, OrcRuntimeArchiveBuffer: std::move(*ArchiveBuffer),
256 LoadDynLibrary: std::move(LoadDynLibrary), StaticVCRuntime, VCRuntimePath,
257 RuntimeAliases: std::move(RuntimeAliases));
258}
259
260Expected<MemoryBufferRef> COFFPlatform::getPerJDObjectFile() {
261 auto PerJDObj = OrcRuntimeArchive->findSym(name: "__orc_rt_coff_per_jd_marker");
262 if (!PerJDObj)
263 return PerJDObj.takeError();
264
265 if (!*PerJDObj)
266 return make_error<StringError>(Args: "Could not find per jd object file",
267 Args: inconvertibleErrorCode());
268
269 auto Buffer = (*PerJDObj)->getAsBinary();
270 if (!Buffer)
271 return Buffer.takeError();
272
273 return (*Buffer)->getMemoryBufferRef();
274}
275
276static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases,
277 ArrayRef<std::pair<const char *, const char *>> AL) {
278 for (auto &KV : AL) {
279 auto AliasName = ES.intern(SymName: KV.first);
280 assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map");
281 Aliases[std::move(AliasName)] = {ES.intern(SymName: KV.second),
282 JITSymbolFlags::Exported};
283 }
284}
285
286Error COFFPlatform::setupJITDylib(JITDylib &JD) {
287 if (auto Err = JD.define(MU: std::make_unique<COFFHeaderMaterializationUnit>(
288 args&: *this, args&: COFFHeaderStartSymbol)))
289 return Err;
290
291 if (auto Err = ES.lookup(SearchOrder: {&JD}, Symbol: COFFHeaderStartSymbol).takeError())
292 return Err;
293
294 // Define the CXX aliases.
295 SymbolAliasMap CXXAliases;
296 addAliases(ES, Aliases&: CXXAliases, AL: requiredCXXAliases());
297 if (auto Err = JD.define(MU: symbolAliases(Aliases: std::move(CXXAliases))))
298 return Err;
299
300 auto PerJDObj = getPerJDObjectFile();
301 if (!PerJDObj)
302 return PerJDObj.takeError();
303
304 auto I = getObjectFileInterface(ES, ObjBuffer: *PerJDObj);
305 if (!I)
306 return I.takeError();
307
308 if (auto Err = ObjLinkingLayer.add(
309 JD, O: MemoryBuffer::getMemBuffer(Ref: *PerJDObj, RequiresNullTerminator: false), I: std::move(*I)))
310 return Err;
311
312 if (!Bootstrapping) {
313 auto ImportedLibs = StaticVCRuntime
314 ? VCRuntimeBootstrap->loadStaticVCRuntime(JD)
315 : VCRuntimeBootstrap->loadDynamicVCRuntime(JD);
316 if (!ImportedLibs)
317 return ImportedLibs.takeError();
318 for (auto &Lib : *ImportedLibs)
319 if (auto Err = LoadDynLibrary(JD, Lib))
320 return Err;
321 if (StaticVCRuntime)
322 if (auto Err = VCRuntimeBootstrap->initializeStaticVCRuntime(JD))
323 return Err;
324 }
325
326 JD.addGenerator(DefGenerator: DLLImportDefinitionGenerator::Create(ES, L&: ObjLinkingLayer));
327 return Error::success();
328}
329
330Error COFFPlatform::teardownJITDylib(JITDylib &JD) {
331 std::lock_guard<std::mutex> Lock(PlatformMutex);
332 auto I = JITDylibToHeaderAddr.find(Val: &JD);
333 if (I != JITDylibToHeaderAddr.end()) {
334 assert(HeaderAddrToJITDylib.count(I->second) &&
335 "HeaderAddrToJITDylib missing entry");
336 HeaderAddrToJITDylib.erase(Val: I->second);
337 JITDylibToHeaderAddr.erase(I);
338 }
339 return Error::success();
340}
341
342Error COFFPlatform::notifyAdding(ResourceTracker &RT,
343 const MaterializationUnit &MU) {
344 auto &JD = RT.getJITDylib();
345 const auto &InitSym = MU.getInitializerSymbol();
346 if (!InitSym)
347 return Error::success();
348
349 RegisteredInitSymbols[&JD].add(Name: InitSym,
350 Flags: SymbolLookupFlags::WeaklyReferencedSymbol);
351
352 LLVM_DEBUG({
353 dbgs() << "COFFPlatform: Registered init symbol " << *InitSym << " for MU "
354 << MU.getName() << "\n";
355 });
356 return Error::success();
357}
358
359Error COFFPlatform::notifyRemoving(ResourceTracker &RT) {
360 llvm_unreachable("Not supported yet");
361}
362
363SymbolAliasMap COFFPlatform::standardPlatformAliases(ExecutionSession &ES) {
364 SymbolAliasMap Aliases;
365 addAliases(ES, Aliases, AL: standardRuntimeUtilityAliases());
366 return Aliases;
367}
368
369ArrayRef<std::pair<const char *, const char *>>
370COFFPlatform::requiredCXXAliases() {
371 static const std::pair<const char *, const char *> RequiredCXXAliases[] = {
372 {"_CxxThrowException", "__orc_rt_coff_cxx_throw_exception"},
373 {"_onexit", "__orc_rt_coff_onexit_per_jd"},
374 {"atexit", "__orc_rt_coff_atexit_per_jd"}};
375
376 return ArrayRef<std::pair<const char *, const char *>>(RequiredCXXAliases);
377}
378
379ArrayRef<std::pair<const char *, const char *>>
380COFFPlatform::standardRuntimeUtilityAliases() {
381 static const std::pair<const char *, const char *>
382 StandardRuntimeUtilityAliases[] = {
383 {"__orc_rt_run_program", "__orc_rt_coff_run_program"},
384 {"__orc_rt_jit_dlerror", "__orc_rt_coff_jit_dlerror"},
385 {"__orc_rt_jit_dlopen", "__orc_rt_coff_jit_dlopen"},
386 {"__orc_rt_jit_dlupdate", "__orc_rt_coff_jit_dlupdate"},
387 {"__orc_rt_jit_dlclose", "__orc_rt_coff_jit_dlclose"},
388 {"__orc_rt_jit_dlsym", "__orc_rt_coff_jit_dlsym"},
389 {"__orc_rt_log_error", "__orc_rt_log_error_to_stderr"}};
390
391 return ArrayRef<std::pair<const char *, const char *>>(
392 StandardRuntimeUtilityAliases);
393}
394
395bool COFFPlatform::supportedTarget(const Triple &TT) {
396 switch (TT.getArch()) {
397 case Triple::x86_64:
398 return true;
399 default:
400 return false;
401 }
402}
403
404COFFPlatform::COFFPlatform(
405 ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD,
406 std::unique_ptr<StaticLibraryDefinitionGenerator> OrcRuntimeGenerator,
407 std::set<std::string> DylibsToPreload,
408 std::unique_ptr<MemoryBuffer> OrcRuntimeArchiveBuffer,
409 std::unique_ptr<object::Archive> OrcRuntimeArchive,
410 LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime,
411 const char *VCRuntimePath, Error &Err)
412 : ES(ObjLinkingLayer.getExecutionSession()),
413 ObjLinkingLayer(ObjLinkingLayer),
414 LoadDynLibrary(std::move(LoadDynLibrary)),
415 OrcRuntimeArchiveBuffer(std::move(OrcRuntimeArchiveBuffer)),
416 OrcRuntimeArchive(std::move(OrcRuntimeArchive)),
417 StaticVCRuntime(StaticVCRuntime),
418 COFFHeaderStartSymbol(ES.intern(SymName: "__ImageBase")) {
419 ErrorAsOutParameter _(Err);
420
421 Bootstrapping.store(i: true);
422 ObjLinkingLayer.addPlugin(P: std::make_unique<COFFPlatformPlugin>(args&: *this));
423
424 // Load vc runtime
425 auto VCRT =
426 COFFVCRuntimeBootstrapper::Create(ES, ObjLinkingLayer, RuntimePath: VCRuntimePath);
427 if (!VCRT) {
428 Err = VCRT.takeError();
429 return;
430 }
431 VCRuntimeBootstrap = std::move(*VCRT);
432
433 auto ImportedLibs =
434 StaticVCRuntime ? VCRuntimeBootstrap->loadStaticVCRuntime(JD&: PlatformJD)
435 : VCRuntimeBootstrap->loadDynamicVCRuntime(JD&: PlatformJD);
436 if (!ImportedLibs) {
437 Err = ImportedLibs.takeError();
438 return;
439 }
440
441 for (auto &Lib : *ImportedLibs)
442 DylibsToPreload.insert(x: Lib);
443
444 PlatformJD.addGenerator(DefGenerator: std::move(OrcRuntimeGenerator));
445
446 // PlatformJD hasn't been set up by the platform yet (since we're creating
447 // the platform now), so set it up.
448 if (auto E2 = setupJITDylib(PlatformJD)) {
449 Err = std::move(E2);
450 return;
451 }
452
453 for (auto& Lib : DylibsToPreload)
454 if (auto E2 = this->LoadDynLibrary(PlatformJD, Lib)) {
455 Err = std::move(E2);
456 return;
457 }
458
459 if (StaticVCRuntime)
460 if (auto E2 = VCRuntimeBootstrap->initializeStaticVCRuntime(JD&: PlatformJD)) {
461 Err = std::move(E2);
462 return;
463 }
464
465 // Associate wrapper function tags with JIT-side function implementations.
466 if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) {
467 Err = std::move(E2);
468 return;
469 }
470
471 // Lookup addresses of runtime functions callable by the platform,
472 // call the platform bootstrap function to initialize the platform-state
473 // object in the executor.
474 if (auto E2 = bootstrapCOFFRuntime(PlatformJD)) {
475 Err = std::move(E2);
476 return;
477 }
478
479 Bootstrapping.store(i: false);
480 JDBootstrapStates.clear();
481}
482
483Expected<COFFPlatform::JITDylibDepMap>
484COFFPlatform::buildJDDepMap(JITDylib &JD) {
485 return ES.runSessionLocked(F: [&]() -> Expected<JITDylibDepMap> {
486 JITDylibDepMap JDDepMap;
487
488 SmallVector<JITDylib *, 16> Worklist({&JD});
489 while (!Worklist.empty()) {
490 auto CurJD = Worklist.back();
491 Worklist.pop_back();
492
493 auto &DM = JDDepMap[CurJD];
494 CurJD->withLinkOrderDo(F: [&](const JITDylibSearchOrder &O) {
495 DM.reserve(N: O.size());
496 for (auto &KV : O) {
497 if (KV.first == CurJD)
498 continue;
499 {
500 // Bare jitdylibs not known to the platform
501 std::lock_guard<std::mutex> Lock(PlatformMutex);
502 if (!JITDylibToHeaderAddr.count(Val: KV.first)) {
503 LLVM_DEBUG({
504 dbgs() << "JITDylib unregistered to COFFPlatform detected in "
505 "LinkOrder: "
506 << CurJD->getName() << "\n";
507 });
508 continue;
509 }
510 }
511 DM.push_back(Elt: KV.first);
512 // Push unvisited entry.
513 if (JDDepMap.try_emplace(Key: KV.first).second)
514 Worklist.push_back(Elt: KV.first);
515 }
516 });
517 }
518 return std::move(JDDepMap);
519 });
520}
521
522void COFFPlatform::pushInitializersLoop(PushInitializersSendResultFn SendResult,
523 JITDylibSP JD,
524 JITDylibDepMap &JDDepMap) {
525 SmallVector<JITDylib *, 16> Worklist({JD.get()});
526 DenseSet<JITDylib *> Visited({JD.get()});
527 DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols;
528 ES.runSessionLocked(F: [&]() {
529 while (!Worklist.empty()) {
530 auto CurJD = Worklist.back();
531 Worklist.pop_back();
532
533 auto RISItr = RegisteredInitSymbols.find(Val: CurJD);
534 if (RISItr != RegisteredInitSymbols.end()) {
535 NewInitSymbols[CurJD] = std::move(RISItr->second);
536 RegisteredInitSymbols.erase(I: RISItr);
537 }
538
539 for (auto *DepJD : JDDepMap[CurJD])
540 if (Visited.insert(V: DepJD).second)
541 Worklist.push_back(Elt: DepJD);
542 }
543 });
544
545 // If there are no further init symbols to look up then send the link order
546 // (as a list of header addresses) to the caller.
547 if (NewInitSymbols.empty()) {
548 // Build the dep info map to return.
549 COFFJITDylibDepInfoMap DIM;
550 DIM.reserve(n: JDDepMap.size());
551 for (auto &KV : JDDepMap) {
552 std::lock_guard<std::mutex> Lock(PlatformMutex);
553 COFFJITDylibDepInfo DepInfo;
554 DepInfo.reserve(n: KV.second.size());
555 for (auto &Dep : KV.second) {
556 DepInfo.push_back(x: JITDylibToHeaderAddr[Dep]);
557 }
558 auto H = JITDylibToHeaderAddr[KV.first];
559 DIM.push_back(x: std::make_pair(x&: H, y: std::move(DepInfo)));
560 }
561 SendResult(DIM);
562 return;
563 }
564
565 // Otherwise issue a lookup and re-run this phase when it completes.
566 lookupInitSymbolsAsync(
567 OnComplete: [this, SendResult = std::move(SendResult), &JD,
568 JDDepMap = std::move(JDDepMap)](Error Err) mutable {
569 if (Err)
570 SendResult(std::move(Err));
571 else
572 pushInitializersLoop(SendResult: std::move(SendResult), JD, JDDepMap);
573 },
574 ES, InitSyms: std::move(NewInitSymbols));
575}
576
577void COFFPlatform::rt_pushInitializers(PushInitializersSendResultFn SendResult,
578 ExecutorAddr JDHeaderAddr) {
579 JITDylibSP JD;
580 {
581 std::lock_guard<std::mutex> Lock(PlatformMutex);
582 auto I = HeaderAddrToJITDylib.find(Val: JDHeaderAddr);
583 if (I != HeaderAddrToJITDylib.end())
584 JD = I->second;
585 }
586
587 LLVM_DEBUG({
588 dbgs() << "COFFPlatform::rt_pushInitializers(" << JDHeaderAddr << ") ";
589 if (JD)
590 dbgs() << "pushing initializers for " << JD->getName() << "\n";
591 else
592 dbgs() << "No JITDylib for header address.\n";
593 });
594
595 if (!JD) {
596 SendResult(make_error<StringError>(Args: "No JITDylib with header addr " +
597 formatv(Fmt: "{0:x}", Vals&: JDHeaderAddr),
598 Args: inconvertibleErrorCode()));
599 return;
600 }
601
602 auto JDDepMap = buildJDDepMap(JD&: *JD);
603 if (!JDDepMap) {
604 SendResult(JDDepMap.takeError());
605 return;
606 }
607
608 pushInitializersLoop(SendResult: std::move(SendResult), JD, JDDepMap&: *JDDepMap);
609}
610
611void COFFPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult,
612 ExecutorAddr Handle, StringRef SymbolName) {
613 LLVM_DEBUG(dbgs() << "COFFPlatform::rt_lookupSymbol(\"" << Handle << "\")\n");
614
615 JITDylib *JD = nullptr;
616
617 {
618 std::lock_guard<std::mutex> Lock(PlatformMutex);
619 auto I = HeaderAddrToJITDylib.find(Val: Handle);
620 if (I != HeaderAddrToJITDylib.end())
621 JD = I->second;
622 }
623
624 if (!JD) {
625 LLVM_DEBUG(dbgs() << " No JITDylib for handle " << Handle << "\n");
626 SendResult(make_error<StringError>(Args: "No JITDylib associated with handle " +
627 formatv(Fmt: "{0:x}", Vals&: Handle),
628 Args: inconvertibleErrorCode()));
629 return;
630 }
631
632 // Use functor class to work around XL build compiler issue on AIX.
633 class RtLookupNotifyComplete {
634 public:
635 RtLookupNotifyComplete(SendSymbolAddressFn &&SendResult)
636 : SendResult(std::move(SendResult)) {}
637 void operator()(Expected<SymbolMap> Result) {
638 if (Result) {
639 assert(Result->size() == 1 && "Unexpected result map count");
640 SendResult(Result->begin()->second.getAddress());
641 } else {
642 SendResult(Result.takeError());
643 }
644 }
645
646 private:
647 SendSymbolAddressFn SendResult;
648 };
649
650 ES.lookup(
651 K: LookupKind::DLSym, SearchOrder: {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}},
652 Symbols: SymbolLookupSet(ES.intern(SymName: SymbolName)), RequiredState: SymbolState::Ready,
653 NotifyComplete: RtLookupNotifyComplete(std::move(SendResult)), RegisterDependencies: NoDependenciesToRegister);
654}
655
656Error COFFPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) {
657 using LookupSymbolSPSSig =
658 SPSExpected<SPSExecutorAddr>(SPSExecutorAddr, SPSString);
659
660 using PushInitializersSPSSig =
661 SPSExpected<SPSCOFFJITDylibDepInfoMap>(SPSExecutorAddr);
662
663 return ES.registerCallControllerHandlers(
664 JD&: PlatformJD,
665 Hs: bindCallControllerHandlerSPS<LookupSymbolSPSSig>(
666 Name: SymbolNameSpec::c(Name: "__orc_rt_coff_symbol_lookup_tag"), Instance: this,
667 Method: &COFFPlatform::rt_lookupSymbol),
668 Hs: bindCallControllerHandlerSPS<PushInitializersSPSSig>(
669 Name: SymbolNameSpec::c(Name: "__orc_rt_coff_push_initializers_tag"), Instance: this,
670 Method: &COFFPlatform::rt_pushInitializers));
671}
672
673Error COFFPlatform::runBootstrapInitializers(JDBootstrapState &BState) {
674 llvm::sort(C&: BState.Initializers);
675 if (auto Err =
676 runBootstrapSubsectionInitializers(BState, Start: ".CRT$XIA", End: ".CRT$XIZ"))
677 return Err;
678
679 if (auto Err = runSymbolIfExists(PlatformJD&: *BState.JD, SymbolName: "__run_after_c_init"))
680 return Err;
681
682 if (auto Err =
683 runBootstrapSubsectionInitializers(BState, Start: ".CRT$XCA", End: ".CRT$XCZ"))
684 return Err;
685 return Error::success();
686}
687
688Error COFFPlatform::runBootstrapSubsectionInitializers(JDBootstrapState &BState,
689 StringRef Start,
690 StringRef End) {
691 CallInt32VoidProxy CallInitializer;
692 if (auto Err = lookupAndApply(
693 JD&: ES.getBootstrapJITDylib(),
694 PrepareFns: {recordProxy<sps::CallInt32VoidProxySpec>(P: &CallInitializer)}))
695 return Err;
696 for (auto &Initializer : BState.Initializers)
697 if (Initializer.first >= Start && Initializer.first <= End &&
698 Initializer.second) {
699 auto Res = CallInitializer(ES, Initializer.second);
700 if (!Res)
701 return Res.takeError();
702 }
703 return Error::success();
704}
705
706Error COFFPlatform::bootstrapCOFFRuntime(JITDylib &PlatformJD) {
707 // Lookup of runtime symbols causes the collection of initializers if
708 // it's static linking setting.
709 if (auto Err = lookupAndApply(
710 JD&: PlatformJD,
711 PrepareFns: {recordAddr(Name: SymbolNameSpec::c(Name: "__orc_rt_coff_platform_bootstrap"),
712 A: &orc_rt_coff_platform_bootstrap),
713 recordAddr(Name: SymbolNameSpec::c(Name: "__orc_rt_coff_platform_shutdown"),
714 A: &orc_rt_coff_platform_shutdown),
715 recordAddr(Name: SymbolNameSpec::c(Name: "__orc_rt_coff_register_jitdylib"),
716 A: &orc_rt_coff_register_jitdylib),
717 recordAddr(Name: SymbolNameSpec::c(Name: "__orc_rt_coff_deregister_jitdylib"),
718 A: &orc_rt_coff_deregister_jitdylib),
719 recordAddr(
720 Name: SymbolNameSpec::c(Name: "__orc_rt_coff_register_object_sections"),
721 A: &orc_rt_coff_register_object_sections),
722 recordAddr(
723 Name: SymbolNameSpec::c(Name: "__orc_rt_coff_deregister_object_sections"),
724 A: &orc_rt_coff_deregister_object_sections)}))
725 return Err;
726
727 // These runtime entry points are held as addresses because their primary use
728 // is as alloc-action tags (see the register/deregister sites below). The
729 // direct dispatches here are a bootstrap-time artifact, so rather than
730 // holding proxies as members we build them over the resolved addresses.
731 // TODO: drop these dispatches once bootstrap no longer needs them.
732 using PlatformBootstrapProxy = Proxy<void()>;
733 using RegisterJITDylibProxy = Proxy<void(std::string, ExecutorAddr)>;
734 using RegisterObjectSectionsProxy =
735 Proxy<void(ExecutorAddr, COFFObjectSectionsMap, bool)>;
736 using sps::ProxySpec;
737
738 PlatformBootstrapProxy PlatformBootstrap(
739 ProxySpec<PlatformBootstrapProxy,
740 coff_sps_ci::PlatformBootstrap>::dispatch,
741 orc_rt_coff_platform_bootstrap);
742 RegisterJITDylibProxy RegisterJITDylib(
743 ProxySpec<RegisterJITDylibProxy, coff_sps_ci::RegisterJITDylib>::dispatch,
744 orc_rt_coff_register_jitdylib);
745 RegisterObjectSectionsProxy RegisterObjectSections(
746 ProxySpec<RegisterObjectSectionsProxy,
747 coff_sps_ci::RegisterObjectSections>::dispatch,
748 orc_rt_coff_register_object_sections);
749
750 // Call bootstrap functions
751 if (auto Err = PlatformBootstrap(ES))
752 return Err;
753
754 // Do the pending jitdylib registration actions that we couldn't do
755 // because orc runtime was not linked fully.
756 for (auto KV : JDBootstrapStates) {
757 auto &JDBState = KV.second;
758 if (auto Err = RegisterJITDylib(ES, JDBState.JDName, JDBState.HeaderAddr))
759 return Err;
760
761 for (auto &ObjSectionMap : JDBState.ObjectSectionsMaps)
762 if (auto Err = RegisterObjectSections(ES, JDBState.HeaderAddr,
763 ObjSectionMap, false))
764 return Err;
765 }
766
767 // Run static initializers collected in bootstrap stage.
768 for (auto KV : JDBootstrapStates) {
769 auto &JDBState = KV.second;
770 if (auto Err = runBootstrapInitializers(BState&: JDBState))
771 return Err;
772 }
773
774 return Error::success();
775}
776
777Error COFFPlatform::runSymbolIfExists(JITDylib &PlatformJD,
778 StringRef SymbolName) {
779 ExecutorAddr TargetFn;
780 if (auto Err = lookupAndApply(
781 JD&: PlatformJD, PrepareFns: {recordAddr(Name: SymbolNameSpec::c(Name: SymbolName), A: &TargetFn,
782 LF: SymbolLookupFlags::WeaklyReferencedSymbol)}))
783 return Err;
784 if (!TargetFn)
785 return Error::success(); // No target function.
786
787 CallInt32VoidProxy CallFn;
788 if (auto Err =
789 lookupAndApply(JD&: ES.getBootstrapJITDylib(),
790 PrepareFns: {recordProxy<sps::CallInt32VoidProxySpec>(P: &CallFn)}))
791 return Err;
792
793 return CallFn(ES, TargetFn).takeError();
794}
795
796void COFFPlatform::COFFPlatformPlugin::modifyPassConfig(
797 MaterializationResponsibility &MR, jitlink::LinkGraph &LG,
798 jitlink::PassConfiguration &Config) {
799
800 bool IsBootstrapping = CP.Bootstrapping.load();
801
802 if (auto InitSymbol = MR.getInitializerSymbol()) {
803 if (InitSymbol == CP.COFFHeaderStartSymbol) {
804 Config.PostAllocationPasses.push_back(
805 x: [this, &MR, IsBootstrapping](jitlink::LinkGraph &G) {
806 return associateJITDylibHeaderSymbol(G, MR, Bootstrap: IsBootstrapping);
807 });
808 return;
809 }
810 Config.PrePrunePasses.push_back(x: [this, &MR](jitlink::LinkGraph &G) {
811 return preserveInitializerSections(G, MR);
812 });
813 }
814
815 if (!IsBootstrapping)
816 Config.PostFixupPasses.push_back(
817 x: [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
818 return registerObjectPlatformSections(G, JD);
819 });
820 else
821 Config.PostFixupPasses.push_back(
822 x: [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
823 return registerObjectPlatformSectionsInBootstrap(G, JD);
824 });
825}
826
827Error COFFPlatform::COFFPlatformPlugin::associateJITDylibHeaderSymbol(
828 jitlink::LinkGraph &G, MaterializationResponsibility &MR,
829 bool IsBootstraping) {
830 auto I = llvm::find_if(Range: G.defined_symbols(), P: [this](jitlink::Symbol *Sym) {
831 return *Sym->getName() == *CP.COFFHeaderStartSymbol;
832 });
833 assert(I != G.defined_symbols().end() && "Missing COFF header start symbol");
834
835 auto &JD = MR.getTargetJITDylib();
836 std::lock_guard<std::mutex> Lock(CP.PlatformMutex);
837 auto HeaderAddr = (*I)->getAddress();
838 CP.JITDylibToHeaderAddr[&JD] = HeaderAddr;
839 CP.HeaderAddrToJITDylib[HeaderAddr] = &JD;
840 if (!IsBootstraping) {
841 G.allocActions().push_back(
842 x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<
843 SPSArgList<SPSString, SPSExecutorAddr>>(
844 FnAddr: CP.orc_rt_coff_register_jitdylib, Args: JD.getName(), Args: HeaderAddr)),
845 .Dealloc: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
846 FnAddr: CP.orc_rt_coff_deregister_jitdylib, Args: HeaderAddr))});
847 } else {
848 G.allocActions().push_back(
849 x: {.Finalize: {},
850 .Dealloc: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
851 FnAddr: CP.orc_rt_coff_deregister_jitdylib, Args: HeaderAddr))});
852 JDBootstrapState BState;
853 BState.JD = &JD;
854 BState.JDName = JD.getName();
855 BState.HeaderAddr = HeaderAddr;
856 CP.JDBootstrapStates.emplace(args: &JD, args&: BState);
857 }
858
859 return Error::success();
860}
861
862Error COFFPlatform::COFFPlatformPlugin::registerObjectPlatformSections(
863 jitlink::LinkGraph &G, JITDylib &JD) {
864 COFFObjectSectionsMap ObjSecs;
865 auto HeaderAddr = CP.JITDylibToHeaderAddr[&JD];
866 assert(HeaderAddr && "Must be registered jitdylib");
867 for (auto &S : G.sections()) {
868 jitlink::SectionRange Range(S);
869 if (Range.getSize())
870 ObjSecs.push_back(Elt: std::make_pair(x: S.getName().str(), y: Range.getRange()));
871 }
872
873 G.allocActions().push_back(
874 x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSCOFFRegisterObjectSectionsArgs>(
875 FnAddr: CP.orc_rt_coff_register_object_sections, Args: HeaderAddr, Args: ObjSecs, Args: true)),
876 .Dealloc: cantFail(
877 ValOrErr: WrapperFunctionCall::Create<SPSCOFFDeregisterObjectSectionsArgs>(
878 FnAddr: CP.orc_rt_coff_deregister_object_sections, Args: HeaderAddr,
879 Args: ObjSecs))});
880
881 return Error::success();
882}
883
884Error COFFPlatform::COFFPlatformPlugin::preserveInitializerSections(
885 jitlink::LinkGraph &G, MaterializationResponsibility &MR) {
886
887 if (const auto &InitSymName = MR.getInitializerSymbol()) {
888
889 jitlink::Symbol *InitSym = nullptr;
890
891 for (auto &InitSection : G.sections()) {
892 // Skip non-init sections.
893 if (!isCOFFInitializerSection(Name: InitSection.getName()) ||
894 InitSection.empty())
895 continue;
896
897 // Create the init symbol if it has not been created already and attach it
898 // to the first block.
899 if (!InitSym) {
900 auto &B = **InitSection.blocks().begin();
901 InitSym = &G.addDefinedSymbol(
902 Content&: B, Offset: 0, Name: *InitSymName, Size: B.getSize(), L: jitlink::Linkage::Strong,
903 S: jitlink::Scope::SideEffectsOnly, IsCallable: false, IsLive: true);
904 }
905
906 // Add keep-alive edges to anonymous symbols in all other init blocks.
907 for (auto *B : InitSection.blocks()) {
908 if (B == &InitSym->getBlock())
909 continue;
910
911 auto &S = G.addAnonymousSymbol(Content&: *B, Offset: 0, Size: B->getSize(), IsCallable: false, IsLive: true);
912 InitSym->getBlock().addEdge(K: jitlink::Edge::KeepAlive, Offset: 0, Target&: S, Addend: 0);
913 }
914 }
915 }
916
917 return Error::success();
918}
919
920Error COFFPlatform::COFFPlatformPlugin::
921 registerObjectPlatformSectionsInBootstrap(jitlink::LinkGraph &G,
922 JITDylib &JD) {
923 std::lock_guard<std::mutex> Lock(CP.PlatformMutex);
924 auto HeaderAddr = CP.JITDylibToHeaderAddr[&JD];
925 COFFObjectSectionsMap ObjSecs;
926 for (auto &S : G.sections()) {
927 jitlink::SectionRange Range(S);
928 if (Range.getSize())
929 ObjSecs.push_back(Elt: std::make_pair(x: S.getName().str(), y: Range.getRange()));
930 }
931
932 G.allocActions().push_back(
933 x: {.Finalize: {},
934 .Dealloc: cantFail(
935 ValOrErr: WrapperFunctionCall::Create<SPSCOFFDeregisterObjectSectionsArgs>(
936 FnAddr: CP.orc_rt_coff_deregister_object_sections, Args: HeaderAddr,
937 Args: ObjSecs))});
938
939 auto &BState = CP.JDBootstrapStates[&JD];
940 BState.ObjectSectionsMaps.push_back(x: std::move(ObjSecs));
941
942 // Collect static initializers
943 for (auto &S : G.sections())
944 if (isCOFFInitializerSection(Name: S.getName()))
945 for (auto *B : S.blocks()) {
946 if (B->edges_empty())
947 continue;
948 for (auto &E : B->edges())
949 BState.Initializers.push_back(Elt: std::make_pair(
950 x: S.getName().str(), y: E.getTarget().getAddress() + E.getAddend()));
951 }
952
953 return Error::success();
954}
955
956} // End namespace orc.
957} // End namespace llvm.
958