1//===------- COFFPlatform.cpp - Utilities for executing COFF in Orc -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/Orc/COFFPlatform.h"
10#include "llvm/ExecutionEngine/Orc/Mangling.h"
11
12#include "llvm/ExecutionEngine/Orc/BindCallControllerHandler.h"
13#include "llvm/ExecutionEngine/Orc/COFF.h"
14#include "llvm/ExecutionEngine/Orc/CallProxiesSPS.h"
15#include "llvm/ExecutionEngine/Orc/DebugUtils.h"
16#include "llvm/ExecutionEngine/Orc/LookupAndApply.h"
17#include "llvm/ExecutionEngine/Orc/ObjectFileInterface.h"
18#include "llvm/ExecutionEngine/Orc/RecordProxy.h"
19#include "llvm/ExecutionEngine/Orc/SPSHandlerSpec.h"
20#include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h"
21#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
22#include "llvm/Object/COFF.h"
23
24#include "llvm/ExecutionEngine/JITLink/x86_64.h"
25
26#define DEBUG_TYPE "orc"
27
28using namespace llvm;
29using namespace llvm::orc;
30using namespace llvm::orc::shared;
31
32namespace llvm {
33namespace orc {
34namespace shared {
35
36using SPSCOFFJITDylibDepInfo = SPSSequence<SPSExecutorAddr>;
37using SPSCOFFJITDylibDepInfoMap =
38 SPSSequence<SPSTuple<SPSExecutorAddr, SPSCOFFJITDylibDepInfo>>;
39using SPSCOFFObjectSectionsMap =
40 SPSSequence<SPSTuple<SPSString, SPSExecutorAddrRange>>;
41using SPSCOFFRegisterObjectSectionsArgs =
42 SPSArgList<SPSExecutorAddr, SPSCOFFObjectSectionsMap, bool>;
43using SPSCOFFDeregisterObjectSectionsArgs =
44 SPSArgList<SPSExecutorAddr, SPSCOFFObjectSectionsMap>;
45
46} // namespace shared
47} // namespace orc
48} // namespace llvm
49// Controller-interface descriptors for the COFF platform runtime's
50// bootstrap-time SPS wrapper calls. Kept in the .cpp (the COFF platform's
51// private contract with its runtime; the SPS arg types live here too), and in
52// a named namespace so the constexpr Name members -- read only as constants by
53// ProxySpec -- don't trip -Wunused-const-variable.
54namespace llvm::orc::coff_sps_ci {
55struct PlatformBootstrap {
56 static constexpr SymbolNameSpec Name =
57 SymbolNameSpec::c(Name: "__orc_rt_coff_platform_bootstrap");
58 using SPSSig = void();
59};
60struct RegisterJITDylib {
61 static constexpr SymbolNameSpec Name =
62 SymbolNameSpec::c(Name: "__orc_rt_coff_register_jitdylib");
63 using SPSSig = void(SPSString, SPSExecutorAddr);
64};
65struct RegisterObjectSections {
66 static constexpr SymbolNameSpec Name =
67 SymbolNameSpec::c(Name: "__orc_rt_coff_register_object_sections");
68 using SPSSig = void(SPSExecutorAddr, SPSCOFFObjectSectionsMap, bool);
69};
70} // namespace llvm::orc::coff_sps_ci
71
72// Specs for the call-controller handlers that the COFF platform runtime calls.
73namespace llvm::orc::coff_sps_handlers {
74struct LookupSymbol : sps::HandlerSpec<SPSExpected<SPSExecutorAddr>(
75 SPSExecutorAddr, SPSString)> {
76 static constexpr SymbolNameSpec Name =
77 SymbolNameSpec::c(Name: "__orc_rt_coff_symbol_lookup_tag");
78};
79struct PushInitializers
80 : sps::HandlerSpec<SPSExpected<SPSCOFFJITDylibDepInfoMap>(
81 SPSExecutorAddr)> {
82 static constexpr SymbolNameSpec Name =
83 SymbolNameSpec::c(Name: "__orc_rt_coff_push_initializers_tag");
84};
85} // namespace llvm::orc::coff_sps_handlers
86
87namespace {
88
89class COFFHeaderMaterializationUnit : public MaterializationUnit {
90public:
91 COFFHeaderMaterializationUnit(COFFPlatform &CP,
92 const SymbolStringPtr &HeaderStartSymbol)
93 : MaterializationUnit(createHeaderInterface(MOP&: CP, HeaderStartSymbol)),
94 CP(CP) {}
95
96 StringRef getName() const override { return "COFFHeaderMU"; }
97
98 void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
99 auto G = std::make_unique<jitlink::LinkGraph>(
100 args: "<COFFHeaderMU>", args: CP.getExecutionSession().getSymbolStringPool(),
101 args: CP.getExecutionSession().getTargetTriple(),
102 args: CP.getExecutionSession().getTargetTriple().getArchPointerBitWidth() / 8,
103 args: SubtargetFeatures(), args&: jitlink::getGenericEdgeKindName);
104 auto &HeaderSection = G->createSection(Name: "__header", Prot: MemProt::Read);
105 auto &HeaderBlock = createHeaderBlock(G&: *G, HeaderSection);
106
107 // Init symbol is __ImageBase symbol.
108 auto &ImageBaseSymbol = G->addDefinedSymbol(
109 Content&: HeaderBlock, Offset: 0, Name: *R->getInitializerSymbol(), Size: HeaderBlock.getSize(),
110 L: jitlink::Linkage::Strong, S: jitlink::Scope::Default, IsCallable: false, IsLive: true);
111
112 addImageBaseRelocationEdge(B&: HeaderBlock, ImageBase&: ImageBaseSymbol);
113
114 CP.getObjectLinkingLayer().emit(R: std::move(R), G: std::move(G));
115 }
116
117 void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
118
119private:
120 struct HeaderSymbol {
121 const char *Name;
122 uint64_t Offset;
123 };
124
125 struct NTHeader {
126 support::ulittle32_t PEMagic;
127 object::coff_file_header FileHeader;
128 struct PEHeader {
129 object::pe32plus_header Header;
130 object::data_directory DataDirectory[COFF::NUM_DATA_DIRECTORIES + 1];
131 } OptionalHeader;
132 };
133
134 struct HeaderBlockContent {
135 object::dos_header DOSHeader;
136 COFFHeaderMaterializationUnit::NTHeader NTHeader;
137 };
138
139 static jitlink::Block &createHeaderBlock(jitlink::LinkGraph &G,
140 jitlink::Section &HeaderSection) {
141 HeaderBlockContent Hdr = {};
142
143 // Set up magic
144 Hdr.DOSHeader.Magic[0] = 'M';
145 Hdr.DOSHeader.Magic[1] = 'Z';
146 Hdr.DOSHeader.AddressOfNewExeHeader =
147 offsetof(HeaderBlockContent, NTHeader);
148 uint32_t PEMagic = *reinterpret_cast<const uint32_t *>(COFF::PEMagic);
149 Hdr.NTHeader.PEMagic = PEMagic;
150 Hdr.NTHeader.OptionalHeader.Header.Magic = COFF::PE32Header::PE32_PLUS;
151
152 switch (G.getTargetTriple().getArch()) {
153 case Triple::x86_64:
154 Hdr.NTHeader.FileHeader.Machine = COFF::IMAGE_FILE_MACHINE_AMD64;
155 break;
156 default:
157 llvm_unreachable("Unrecognized architecture");
158 }
159
160 auto HeaderContent = G.allocateContent(
161 Source: ArrayRef<char>(reinterpret_cast<const char *>(&Hdr), sizeof(Hdr)));
162
163 return G.createContentBlock(Parent&: HeaderSection, Content: HeaderContent, Address: ExecutorAddr(), Alignment: 8,
164 AlignmentOffset: 0);
165 }
166
167 static void addImageBaseRelocationEdge(jitlink::Block &B,
168 jitlink::Symbol &ImageBase) {
169 auto ImageBaseOffset = offsetof(HeaderBlockContent, NTHeader) +
170 offsetof(NTHeader, OptionalHeader) +
171 offsetof(object::pe32plus_header, ImageBase);
172 B.addEdge(K: jitlink::x86_64::Pointer64, Offset: ImageBaseOffset, Target&: ImageBase, Addend: 0);
173 }
174
175 static MaterializationUnit::Interface
176 createHeaderInterface(COFFPlatform &MOP,
177 const SymbolStringPtr &HeaderStartSymbol) {
178 SymbolFlagsMap HeaderSymbolFlags;
179
180 HeaderSymbolFlags[HeaderStartSymbol] = JITSymbolFlags::Exported;
181
182 return MaterializationUnit::Interface(std::move(HeaderSymbolFlags),
183 HeaderStartSymbol);
184 }
185
186 COFFPlatform &CP;
187};
188
189} // end anonymous namespace
190
191namespace llvm {
192namespace orc {
193
194Expected<std::unique_ptr<COFFPlatform>>
195COFFPlatform::Create(ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD,
196 std::unique_ptr<MemoryBuffer> OrcRuntimeArchiveBuffer,
197 LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime,
198 const char *VCRuntimePath,
199 std::optional<SymbolAliasMap> RuntimeAliases) {
200
201 auto &ES = ObjLinkingLayer.getExecutionSession();
202
203 // If the target is not supported then bail out immediately.
204 if (!supportedTarget(TT: ES.getTargetTriple()))
205 return make_error<StringError>(Args: "Unsupported COFFPlatform triple: " +
206 ES.getTargetTriple().str(),
207 Args: inconvertibleErrorCode());
208
209 auto GeneratorArchive =
210 object::Archive::create(Source: OrcRuntimeArchiveBuffer->getMemBufferRef());
211 if (!GeneratorArchive)
212 return GeneratorArchive.takeError();
213
214 std::set<std::string> DylibsToPreload;
215 auto OrcRuntimeArchiveGenerator = StaticLibraryDefinitionGenerator::Create(
216 L&: ObjLinkingLayer, ArchiveBuffer: nullptr, Archive: std::move(*GeneratorArchive),
217 VisitMembers: COFFImportFileScanner(DylibsToPreload));
218 if (!OrcRuntimeArchiveGenerator)
219 return OrcRuntimeArchiveGenerator.takeError();
220
221 // We need a second instance of the archive (for now) for the Platform. We
222 // can `cantFail` this call, since if it were going to fail it would have
223 // failed above.
224 auto RuntimeArchive = cantFail(
225 ValOrErr: object::Archive::create(Source: OrcRuntimeArchiveBuffer->getMemBufferRef()));
226
227 // Create default aliases if the caller didn't supply any.
228 if (!RuntimeAliases)
229 RuntimeAliases = standardPlatformAliases(ES);
230
231 // Define the aliases.
232 if (auto Err = PlatformJD.define(MU: symbolAliases(Aliases: std::move(*RuntimeAliases))))
233 return std::move(Err);
234
235 {
236 // Add JIT dispatch reexports from bootstrap JITDylib.
237 MangleAndInterner Mangle(ES);
238 auto Exports = buildSimpleReexportsAliasMap(
239 SourceJD&: ES.getBootstrapJITDylib(),
240 Symbols: {{Mangle(rt::DispatchName), Mangle(rt::DispatchCtxName)}});
241 if (!Exports)
242 return Exports.takeError();
243 if (auto Err =
244 PlatformJD.define(MU: reexports(SourceJD&: ES.getBootstrapJITDylib(), Aliases: *Exports)))
245 return Err;
246 }
247
248 // Create the instance.
249 Error Err = Error::success();
250 auto P = std::unique_ptr<COFFPlatform>(new COFFPlatform(
251 ObjLinkingLayer, PlatformJD, std::move(*OrcRuntimeArchiveGenerator),
252 std::move(DylibsToPreload), std::move(OrcRuntimeArchiveBuffer),
253 std::move(RuntimeArchive), std::move(LoadDynLibrary), StaticVCRuntime,
254 VCRuntimePath, Err));
255 if (Err)
256 return std::move(Err);
257 return std::move(P);
258}
259
260Expected<std::unique_ptr<COFFPlatform>>
261COFFPlatform::Create(ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD,
262 const char *OrcRuntimePath,
263 LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime,
264 const char *VCRuntimePath,
265 std::optional<SymbolAliasMap> RuntimeAliases) {
266
267 auto ArchiveBuffer = MemoryBuffer::getFile(Filename: OrcRuntimePath);
268 if (!ArchiveBuffer)
269 return createFileError(F: OrcRuntimePath, EC: ArchiveBuffer.getError());
270
271 return Create(ObjLinkingLayer, PlatformJD, OrcRuntimeArchiveBuffer: std::move(*ArchiveBuffer),
272 LoadDynLibrary: std::move(LoadDynLibrary), StaticVCRuntime, VCRuntimePath,
273 RuntimeAliases: std::move(RuntimeAliases));
274}
275
276Expected<MemoryBufferRef> COFFPlatform::getPerJDObjectFile() {
277 auto PerJDObj = OrcRuntimeArchive->findSym(name: "__orc_rt_coff_per_jd_marker");
278 if (!PerJDObj)
279 return PerJDObj.takeError();
280
281 if (!*PerJDObj)
282 return make_error<StringError>(Args: "Could not find per jd object file",
283 Args: inconvertibleErrorCode());
284
285 auto Buffer = (*PerJDObj)->getAsBinary();
286 if (!Buffer)
287 return Buffer.takeError();
288
289 return (*Buffer)->getMemoryBufferRef();
290}
291
292static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases,
293 ArrayRef<std::pair<const char *, const char *>> AL) {
294 for (auto &KV : AL) {
295 auto AliasName = ES.intern(SymName: KV.first);
296 assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map");
297 Aliases[std::move(AliasName)] = {ES.intern(SymName: KV.second),
298 JITSymbolFlags::Exported};
299 }
300}
301
302Error COFFPlatform::setupJITDylib(JITDylib &JD) {
303 if (auto Err = JD.define(MU: std::make_unique<COFFHeaderMaterializationUnit>(
304 args&: *this, args&: COFFHeaderStartSymbol)))
305 return Err;
306
307 if (auto Err = ES.lookup(SearchOrder: {&JD}, Symbol: COFFHeaderStartSymbol).takeError())
308 return Err;
309
310 // Define the CXX aliases.
311 SymbolAliasMap CXXAliases;
312 addAliases(ES, Aliases&: CXXAliases, AL: requiredCXXAliases());
313 if (auto Err = JD.define(MU: symbolAliases(Aliases: std::move(CXXAliases))))
314 return Err;
315
316 auto PerJDObj = getPerJDObjectFile();
317 if (!PerJDObj)
318 return PerJDObj.takeError();
319
320 auto I = getObjectFileInterface(ES, ObjBuffer: *PerJDObj);
321 if (!I)
322 return I.takeError();
323
324 if (auto Err = ObjLinkingLayer.add(
325 JD, O: MemoryBuffer::getMemBuffer(Ref: *PerJDObj, RequiresNullTerminator: false), I: std::move(*I)))
326 return Err;
327
328 if (!Bootstrapping) {
329 auto ImportedLibs = StaticVCRuntime
330 ? VCRuntimeBootstrap->loadStaticVCRuntime(JD)
331 : VCRuntimeBootstrap->loadDynamicVCRuntime(JD);
332 if (!ImportedLibs)
333 return ImportedLibs.takeError();
334 for (auto &Lib : *ImportedLibs)
335 if (auto Err = LoadDynLibrary(JD, Lib))
336 return Err;
337 if (StaticVCRuntime)
338 if (auto Err = VCRuntimeBootstrap->initializeStaticVCRuntime(JD))
339 return Err;
340 }
341
342 JD.addGenerator(DefGenerator: DLLImportDefinitionGenerator::Create(ES, L&: ObjLinkingLayer));
343 return Error::success();
344}
345
346Error COFFPlatform::teardownJITDylib(JITDylib &JD) {
347 std::lock_guard<std::mutex> Lock(PlatformMutex);
348 auto I = JITDylibToHeaderAddr.find(Val: &JD);
349 if (I != JITDylibToHeaderAddr.end()) {
350 assert(HeaderAddrToJITDylib.count(I->second) &&
351 "HeaderAddrToJITDylib missing entry");
352 HeaderAddrToJITDylib.erase(Val: I->second);
353 JITDylibToHeaderAddr.erase(I);
354 }
355 return Error::success();
356}
357
358Error COFFPlatform::notifyAdding(ResourceTracker &RT,
359 const MaterializationUnit &MU) {
360 auto &JD = RT.getJITDylib();
361 const auto &InitSym = MU.getInitializerSymbol();
362 if (!InitSym)
363 return Error::success();
364
365 RegisteredInitSymbols[&JD].add(Name: InitSym,
366 Flags: SymbolLookupFlags::WeaklyReferencedSymbol);
367
368 LLVM_DEBUG({
369 dbgs() << "COFFPlatform: Registered init symbol " << *InitSym << " for MU "
370 << MU.getName() << "\n";
371 });
372 return Error::success();
373}
374
375Error COFFPlatform::notifyRemoving(ResourceTracker &RT) {
376 llvm_unreachable("Not supported yet");
377}
378
379SymbolAliasMap COFFPlatform::standardPlatformAliases(ExecutionSession &ES) {
380 SymbolAliasMap Aliases;
381 addAliases(ES, Aliases, AL: standardRuntimeUtilityAliases());
382 return Aliases;
383}
384
385ArrayRef<std::pair<const char *, const char *>>
386COFFPlatform::requiredCXXAliases() {
387 static const std::pair<const char *, const char *> RequiredCXXAliases[] = {
388 {"_CxxThrowException", "__orc_rt_coff_cxx_throw_exception"},
389 {"_onexit", "__orc_rt_coff_onexit_per_jd"},
390 {"atexit", "__orc_rt_coff_atexit_per_jd"}};
391
392 return ArrayRef<std::pair<const char *, const char *>>(RequiredCXXAliases);
393}
394
395ArrayRef<std::pair<const char *, const char *>>
396COFFPlatform::standardRuntimeUtilityAliases() {
397 static const std::pair<const char *, const char *>
398 StandardRuntimeUtilityAliases[] = {
399 {"__orc_rt_run_program", "__orc_rt_coff_run_program"},
400 {"__orc_rt_jit_dlerror", "__orc_rt_coff_jit_dlerror"},
401 {"__orc_rt_jit_dlopen", "__orc_rt_coff_jit_dlopen"},
402 {"__orc_rt_jit_dlupdate", "__orc_rt_coff_jit_dlupdate"},
403 {"__orc_rt_jit_dlclose", "__orc_rt_coff_jit_dlclose"},
404 {"__orc_rt_jit_dlsym", "__orc_rt_coff_jit_dlsym"},
405 {"__orc_rt_log_error", "__orc_rt_log_error_to_stderr"}};
406
407 return ArrayRef<std::pair<const char *, const char *>>(
408 StandardRuntimeUtilityAliases);
409}
410
411bool COFFPlatform::supportedTarget(const Triple &TT) {
412 switch (TT.getArch()) {
413 case Triple::x86_64:
414 return true;
415 default:
416 return false;
417 }
418}
419
420COFFPlatform::COFFPlatform(
421 ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD,
422 std::unique_ptr<StaticLibraryDefinitionGenerator> OrcRuntimeGenerator,
423 std::set<std::string> DylibsToPreload,
424 std::unique_ptr<MemoryBuffer> OrcRuntimeArchiveBuffer,
425 std::unique_ptr<object::Archive> OrcRuntimeArchive,
426 LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime,
427 const char *VCRuntimePath, Error &Err)
428 : ES(ObjLinkingLayer.getExecutionSession()),
429 ObjLinkingLayer(ObjLinkingLayer),
430 LoadDynLibrary(std::move(LoadDynLibrary)),
431 OrcRuntimeArchiveBuffer(std::move(OrcRuntimeArchiveBuffer)),
432 OrcRuntimeArchive(std::move(OrcRuntimeArchive)),
433 StaticVCRuntime(StaticVCRuntime),
434 COFFHeaderStartSymbol(ES.intern(SymName: "__ImageBase")) {
435 ErrorAsOutParameter _(Err);
436
437 Bootstrapping.store(i: true);
438 ObjLinkingLayer.addPlugin(P: std::make_unique<COFFPlatformPlugin>(args&: *this));
439
440 // Load vc runtime
441 auto VCRT =
442 COFFVCRuntimeBootstrapper::Create(ES, ObjLinkingLayer, RuntimePath: VCRuntimePath);
443 if (!VCRT) {
444 Err = VCRT.takeError();
445 return;
446 }
447 VCRuntimeBootstrap = std::move(*VCRT);
448
449 auto ImportedLibs =
450 StaticVCRuntime ? VCRuntimeBootstrap->loadStaticVCRuntime(JD&: PlatformJD)
451 : VCRuntimeBootstrap->loadDynamicVCRuntime(JD&: PlatformJD);
452 if (!ImportedLibs) {
453 Err = ImportedLibs.takeError();
454 return;
455 }
456
457 for (auto &Lib : *ImportedLibs)
458 DylibsToPreload.insert(x: Lib);
459
460 PlatformJD.addGenerator(DefGenerator: std::move(OrcRuntimeGenerator));
461
462 // PlatformJD hasn't been set up by the platform yet (since we're creating
463 // the platform now), so set it up.
464 if (auto E2 = setupJITDylib(PlatformJD)) {
465 Err = std::move(E2);
466 return;
467 }
468
469 for (auto& Lib : DylibsToPreload)
470 if (auto E2 = this->LoadDynLibrary(PlatformJD, Lib)) {
471 Err = std::move(E2);
472 return;
473 }
474
475 if (StaticVCRuntime)
476 if (auto E2 = VCRuntimeBootstrap->initializeStaticVCRuntime(JD&: PlatformJD)) {
477 Err = std::move(E2);
478 return;
479 }
480
481 // Associate wrapper function tags with JIT-side function implementations.
482 if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) {
483 Err = std::move(E2);
484 return;
485 }
486
487 // Lookup addresses of runtime functions callable by the platform,
488 // call the platform bootstrap function to initialize the platform-state
489 // object in the executor.
490 if (auto E2 = bootstrapCOFFRuntime(PlatformJD)) {
491 Err = std::move(E2);
492 return;
493 }
494
495 Bootstrapping.store(i: false);
496 JDBootstrapStates.clear();
497}
498
499Expected<COFFPlatform::JITDylibDepMap>
500COFFPlatform::buildJDDepMap(JITDylib &JD) {
501 return ES.runSessionLocked(F: [&]() -> Expected<JITDylibDepMap> {
502 JITDylibDepMap JDDepMap;
503
504 SmallVector<JITDylib *, 16> Worklist({&JD});
505 while (!Worklist.empty()) {
506 auto CurJD = Worklist.back();
507 Worklist.pop_back();
508
509 auto &DM = JDDepMap[CurJD];
510 CurJD->withLinkOrderDo(F: [&](const JITDylibSearchOrder &O) {
511 DM.reserve(N: O.size());
512 for (auto &KV : O) {
513 if (KV.first == CurJD)
514 continue;
515 {
516 // Bare jitdylibs not known to the platform
517 std::lock_guard<std::mutex> Lock(PlatformMutex);
518 if (!JITDylibToHeaderAddr.count(Val: KV.first)) {
519 LLVM_DEBUG({
520 dbgs() << "JITDylib unregistered to COFFPlatform detected in "
521 "LinkOrder: "
522 << CurJD->getName() << "\n";
523 });
524 continue;
525 }
526 }
527 DM.push_back(Elt: KV.first);
528 // Push unvisited entry.
529 if (JDDepMap.try_emplace(Key: KV.first).second)
530 Worklist.push_back(Elt: KV.first);
531 }
532 });
533 }
534 return std::move(JDDepMap);
535 });
536}
537
538void COFFPlatform::pushInitializersLoop(PushInitializersSendResultFn SendResult,
539 JITDylibSP JD,
540 JITDylibDepMap &JDDepMap) {
541 SmallVector<JITDylib *, 16> Worklist({JD.get()});
542 DenseSet<JITDylib *> Visited({JD.get()});
543 DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols;
544 ES.runSessionLocked(F: [&]() {
545 while (!Worklist.empty()) {
546 auto CurJD = Worklist.back();
547 Worklist.pop_back();
548
549 auto RISItr = RegisteredInitSymbols.find(Val: CurJD);
550 if (RISItr != RegisteredInitSymbols.end()) {
551 NewInitSymbols[CurJD] = std::move(RISItr->second);
552 RegisteredInitSymbols.erase(I: RISItr);
553 }
554
555 for (auto *DepJD : JDDepMap[CurJD])
556 if (Visited.insert(V: DepJD).second)
557 Worklist.push_back(Elt: DepJD);
558 }
559 });
560
561 // If there are no further init symbols to look up then send the link order
562 // (as a list of header addresses) to the caller.
563 if (NewInitSymbols.empty()) {
564 // Build the dep info map to return.
565 COFFJITDylibDepInfoMap DIM;
566 DIM.reserve(n: JDDepMap.size());
567 for (auto &KV : JDDepMap) {
568 std::lock_guard<std::mutex> Lock(PlatformMutex);
569 COFFJITDylibDepInfo DepInfo;
570 DepInfo.reserve(n: KV.second.size());
571 for (auto &Dep : KV.second) {
572 DepInfo.push_back(x: JITDylibToHeaderAddr[Dep]);
573 }
574 auto H = JITDylibToHeaderAddr[KV.first];
575 DIM.push_back(x: std::make_pair(x&: H, y: std::move(DepInfo)));
576 }
577 SendResult(DIM);
578 return;
579 }
580
581 // Otherwise issue a lookup and re-run this phase when it completes.
582 lookupInitSymbolsAsync(
583 OnComplete: [this, SendResult = std::move(SendResult), &JD,
584 JDDepMap = std::move(JDDepMap)](Error Err) mutable {
585 if (Err)
586 SendResult(std::move(Err));
587 else
588 pushInitializersLoop(SendResult: std::move(SendResult), JD, JDDepMap);
589 },
590 ES, InitSyms: std::move(NewInitSymbols));
591}
592
593void COFFPlatform::rt_pushInitializers(PushInitializersSendResultFn SendResult,
594 ExecutorAddr JDHeaderAddr) {
595 JITDylibSP JD;
596 {
597 std::lock_guard<std::mutex> Lock(PlatformMutex);
598 auto I = HeaderAddrToJITDylib.find(Val: JDHeaderAddr);
599 if (I != HeaderAddrToJITDylib.end())
600 JD = I->second;
601 }
602
603 LLVM_DEBUG({
604 dbgs() << "COFFPlatform::rt_pushInitializers(" << JDHeaderAddr << ") ";
605 if (JD)
606 dbgs() << "pushing initializers for " << JD->getName() << "\n";
607 else
608 dbgs() << "No JITDylib for header address.\n";
609 });
610
611 if (!JD) {
612 SendResult(make_error<StringError>(Args: "No JITDylib with header addr " +
613 formatv(Fmt: "{0:x}", Vals&: JDHeaderAddr),
614 Args: inconvertibleErrorCode()));
615 return;
616 }
617
618 auto JDDepMap = buildJDDepMap(JD&: *JD);
619 if (!JDDepMap) {
620 SendResult(JDDepMap.takeError());
621 return;
622 }
623
624 pushInitializersLoop(SendResult: std::move(SendResult), JD, JDDepMap&: *JDDepMap);
625}
626
627void COFFPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult,
628 ExecutorAddr Handle, StringRef SymbolName) {
629 LLVM_DEBUG(dbgs() << "COFFPlatform::rt_lookupSymbol(\"" << Handle << "\")\n");
630
631 JITDylib *JD = nullptr;
632
633 {
634 std::lock_guard<std::mutex> Lock(PlatformMutex);
635 auto I = HeaderAddrToJITDylib.find(Val: Handle);
636 if (I != HeaderAddrToJITDylib.end())
637 JD = I->second;
638 }
639
640 if (!JD) {
641 LLVM_DEBUG(dbgs() << " No JITDylib for handle " << Handle << "\n");
642 SendResult(make_error<StringError>(Args: "No JITDylib associated with handle " +
643 formatv(Fmt: "{0:x}", Vals&: Handle),
644 Args: inconvertibleErrorCode()));
645 return;
646 }
647
648 // Use functor class to work around XL build compiler issue on AIX.
649 class RtLookupNotifyComplete {
650 public:
651 RtLookupNotifyComplete(SendSymbolAddressFn &&SendResult)
652 : SendResult(std::move(SendResult)) {}
653 void operator()(Expected<SymbolMap> Result) {
654 if (Result) {
655 assert(Result->size() == 1 && "Unexpected result map count");
656 SendResult(Result->begin()->second.getAddress());
657 } else {
658 SendResult(Result.takeError());
659 }
660 }
661
662 private:
663 SendSymbolAddressFn SendResult;
664 };
665
666 ES.lookup(
667 K: LookupKind::DLSym, SearchOrder: {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}},
668 Symbols: SymbolLookupSet(ES.intern(SymName: SymbolName)), RequiredState: SymbolState::Ready,
669 NotifyComplete: RtLookupNotifyComplete(std::move(SendResult)), RegisterDependencies: NoDependenciesToRegister);
670}
671
672Error COFFPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) {
673 return ES.registerCallControllerHandlers(
674 JD&: PlatformJD,
675 Hs: bindCallControllerHandler<coff_sps_handlers::LookupSymbol>(
676 Instance: this, Method: &COFFPlatform::rt_lookupSymbol),
677 Hs: bindCallControllerHandler<coff_sps_handlers::PushInitializers>(
678 Instance: this, Method: &COFFPlatform::rt_pushInitializers));
679}
680
681Error COFFPlatform::runBootstrapInitializers(JDBootstrapState &BState) {
682 llvm::sort(C&: BState.Initializers);
683 if (auto Err =
684 runBootstrapSubsectionInitializers(BState, Start: ".CRT$XIA", End: ".CRT$XIZ"))
685 return Err;
686
687 if (auto Err = runSymbolIfExists(PlatformJD&: *BState.JD, SymbolName: "__run_after_c_init"))
688 return Err;
689
690 if (auto Err =
691 runBootstrapSubsectionInitializers(BState, Start: ".CRT$XCA", End: ".CRT$XCZ"))
692 return Err;
693 return Error::success();
694}
695
696Error COFFPlatform::runBootstrapSubsectionInitializers(JDBootstrapState &BState,
697 StringRef Start,
698 StringRef End) {
699 CallInt32VoidProxy CallInitializer;
700 if (auto Err = lookupAndApply(
701 JD&: ES.getBootstrapJITDylib(),
702 PrepareFns: {recordProxy<sps::CallInt32VoidProxySpec>(P: &CallInitializer)}))
703 return Err;
704 for (auto &Initializer : BState.Initializers)
705 if (Initializer.first >= Start && Initializer.first <= End &&
706 Initializer.second) {
707 auto Res = CallInitializer(ES, Initializer.second);
708 if (!Res)
709 return Res.takeError();
710 }
711 return Error::success();
712}
713
714Error COFFPlatform::bootstrapCOFFRuntime(JITDylib &PlatformJD) {
715 // Lookup of runtime symbols causes the collection of initializers if
716 // it's static linking setting.
717 if (auto Err = lookupAndApply(
718 JD&: PlatformJD,
719 PrepareFns: {recordAddr(Name: SymbolNameSpec::c(Name: "__orc_rt_coff_platform_bootstrap"),
720 A: &orc_rt_coff_platform_bootstrap),
721 recordAddr(Name: SymbolNameSpec::c(Name: "__orc_rt_coff_platform_shutdown"),
722 A: &orc_rt_coff_platform_shutdown),
723 recordAddr(Name: SymbolNameSpec::c(Name: "__orc_rt_coff_register_jitdylib"),
724 A: &orc_rt_coff_register_jitdylib),
725 recordAddr(Name: SymbolNameSpec::c(Name: "__orc_rt_coff_deregister_jitdylib"),
726 A: &orc_rt_coff_deregister_jitdylib),
727 recordAddr(
728 Name: SymbolNameSpec::c(Name: "__orc_rt_coff_register_object_sections"),
729 A: &orc_rt_coff_register_object_sections),
730 recordAddr(
731 Name: SymbolNameSpec::c(Name: "__orc_rt_coff_deregister_object_sections"),
732 A: &orc_rt_coff_deregister_object_sections)}))
733 return Err;
734
735 // These runtime entry points are held as addresses because their primary use
736 // is as alloc-action tags (see the register/deregister sites below). The
737 // direct dispatches here are a bootstrap-time artifact, so rather than
738 // holding proxies as members we build them over the resolved addresses.
739 // TODO: drop these dispatches once bootstrap no longer needs them.
740 using PlatformBootstrapProxy = Proxy<void()>;
741 using RegisterJITDylibProxy = Proxy<void(std::string, ExecutorAddr)>;
742 using RegisterObjectSectionsProxy =
743 Proxy<void(ExecutorAddr, COFFObjectSectionsMap, bool)>;
744 using sps::ProxySpec;
745
746 PlatformBootstrapProxy PlatformBootstrap(
747 ProxySpec<coff_sps_ci::PlatformBootstrap>::dispatch,
748 orc_rt_coff_platform_bootstrap);
749 RegisterJITDylibProxy RegisterJITDylib(
750 ProxySpec<coff_sps_ci::RegisterJITDylib>::dispatch,
751 orc_rt_coff_register_jitdylib);
752 RegisterObjectSectionsProxy RegisterObjectSections(
753 ProxySpec<coff_sps_ci::RegisterObjectSections>::dispatch,
754 orc_rt_coff_register_object_sections);
755
756 // Call bootstrap functions
757 if (auto Err = PlatformBootstrap(ES))
758 return Err;
759
760 // Do the pending jitdylib registration actions that we couldn't do
761 // because orc runtime was not linked fully.
762 for (auto KV : JDBootstrapStates) {
763 auto &JDBState = KV.second;
764 if (auto Err = RegisterJITDylib(ES, JDBState.JDName, JDBState.HeaderAddr))
765 return Err;
766
767 for (auto &ObjSectionMap : JDBState.ObjectSectionsMaps)
768 if (auto Err = RegisterObjectSections(ES, JDBState.HeaderAddr,
769 ObjSectionMap, false))
770 return Err;
771 }
772
773 // Run static initializers collected in bootstrap stage.
774 for (auto KV : JDBootstrapStates) {
775 auto &JDBState = KV.second;
776 if (auto Err = runBootstrapInitializers(BState&: JDBState))
777 return Err;
778 }
779
780 return Error::success();
781}
782
783Error COFFPlatform::runSymbolIfExists(JITDylib &PlatformJD,
784 StringRef SymbolName) {
785 ExecutorAddr TargetFn;
786 if (auto Err = lookupAndApply(
787 JD&: PlatformJD, PrepareFns: {recordAddr(Name: SymbolNameSpec::c(Name: SymbolName), A: &TargetFn,
788 LF: SymbolLookupFlags::WeaklyReferencedSymbol)}))
789 return Err;
790 if (!TargetFn)
791 return Error::success(); // No target function.
792
793 CallInt32VoidProxy CallFn;
794 if (auto Err =
795 lookupAndApply(JD&: ES.getBootstrapJITDylib(),
796 PrepareFns: {recordProxy<sps::CallInt32VoidProxySpec>(P: &CallFn)}))
797 return Err;
798
799 return CallFn(ES, TargetFn).takeError();
800}
801
802void COFFPlatform::COFFPlatformPlugin::modifyPassConfig(
803 MaterializationResponsibility &MR, jitlink::LinkGraph &LG,
804 jitlink::PassConfiguration &Config) {
805
806 bool IsBootstrapping = CP.Bootstrapping.load();
807
808 if (auto InitSymbol = MR.getInitializerSymbol()) {
809 if (InitSymbol == CP.COFFHeaderStartSymbol) {
810 Config.PostAllocationPasses.push_back(
811 x: [this, &MR, IsBootstrapping](jitlink::LinkGraph &G) {
812 return associateJITDylibHeaderSymbol(G, MR, Bootstrap: IsBootstrapping);
813 });
814 return;
815 }
816 Config.PrePrunePasses.push_back(x: [this, &MR](jitlink::LinkGraph &G) {
817 return preserveInitializerSections(G, MR);
818 });
819 }
820
821 if (!IsBootstrapping)
822 Config.PostFixupPasses.push_back(
823 x: [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
824 return registerObjectPlatformSections(G, JD);
825 });
826 else
827 Config.PostFixupPasses.push_back(
828 x: [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
829 return registerObjectPlatformSectionsInBootstrap(G, JD);
830 });
831}
832
833Error COFFPlatform::COFFPlatformPlugin::associateJITDylibHeaderSymbol(
834 jitlink::LinkGraph &G, MaterializationResponsibility &MR,
835 bool IsBootstraping) {
836 auto I = llvm::find_if(Range: G.defined_symbols(), P: [this](jitlink::Symbol *Sym) {
837 return *Sym->getName() == *CP.COFFHeaderStartSymbol;
838 });
839 assert(I != G.defined_symbols().end() && "Missing COFF header start symbol");
840
841 auto &JD = MR.getTargetJITDylib();
842 std::lock_guard<std::mutex> Lock(CP.PlatformMutex);
843 auto HeaderAddr = (*I)->getAddress();
844 CP.JITDylibToHeaderAddr[&JD] = HeaderAddr;
845 CP.HeaderAddrToJITDylib[HeaderAddr] = &JD;
846 if (!IsBootstraping) {
847 G.allocActions().push_back(
848 x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<
849 SPSArgList<SPSString, SPSExecutorAddr>>(
850 FnAddr: CP.orc_rt_coff_register_jitdylib, Args: JD.getName(), Args: HeaderAddr)),
851 .Dealloc: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
852 FnAddr: CP.orc_rt_coff_deregister_jitdylib, Args: HeaderAddr))});
853 } else {
854 G.allocActions().push_back(
855 x: {.Finalize: {},
856 .Dealloc: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
857 FnAddr: CP.orc_rt_coff_deregister_jitdylib, Args: HeaderAddr))});
858 JDBootstrapState BState;
859 BState.JD = &JD;
860 BState.JDName = JD.getName();
861 BState.HeaderAddr = HeaderAddr;
862 CP.JDBootstrapStates.emplace(args: &JD, args&: BState);
863 }
864
865 return Error::success();
866}
867
868Error COFFPlatform::COFFPlatformPlugin::registerObjectPlatformSections(
869 jitlink::LinkGraph &G, JITDylib &JD) {
870 COFFObjectSectionsMap ObjSecs;
871 auto HeaderAddr = CP.JITDylibToHeaderAddr[&JD];
872 assert(HeaderAddr && "Must be registered jitdylib");
873 for (auto &S : G.sections()) {
874 jitlink::SectionRange Range(S);
875 if (Range.getSize())
876 ObjSecs.push_back(Elt: std::make_pair(x: S.getName().str(), y: Range.getRange()));
877 }
878
879 G.allocActions().push_back(
880 x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSCOFFRegisterObjectSectionsArgs>(
881 FnAddr: CP.orc_rt_coff_register_object_sections, Args: HeaderAddr, Args: ObjSecs, Args: true)),
882 .Dealloc: cantFail(
883 ValOrErr: WrapperFunctionCall::Create<SPSCOFFDeregisterObjectSectionsArgs>(
884 FnAddr: CP.orc_rt_coff_deregister_object_sections, Args: HeaderAddr,
885 Args: ObjSecs))});
886
887 return Error::success();
888}
889
890Error COFFPlatform::COFFPlatformPlugin::preserveInitializerSections(
891 jitlink::LinkGraph &G, MaterializationResponsibility &MR) {
892
893 if (const auto &InitSymName = MR.getInitializerSymbol()) {
894
895 jitlink::Symbol *InitSym = nullptr;
896
897 for (auto &InitSection : G.sections()) {
898 // Skip non-init sections.
899 if (!isCOFFInitializerSection(Name: InitSection.getName()) ||
900 InitSection.empty())
901 continue;
902
903 // Create the init symbol if it has not been created already and attach it
904 // to the first block.
905 if (!InitSym) {
906 auto &B = **InitSection.blocks().begin();
907 InitSym = &G.addDefinedSymbol(
908 Content&: B, Offset: 0, Name: *InitSymName, Size: B.getSize(), L: jitlink::Linkage::Strong,
909 S: jitlink::Scope::SideEffectsOnly, IsCallable: false, IsLive: true);
910 }
911
912 // Add keep-alive edges to anonymous symbols in all other init blocks.
913 for (auto *B : InitSection.blocks()) {
914 if (B == &InitSym->getBlock())
915 continue;
916
917 auto &S = G.addAnonymousSymbol(Content&: *B, Offset: 0, Size: B->getSize(), IsCallable: false, IsLive: true);
918 InitSym->getBlock().addEdge(K: jitlink::Edge::KeepAlive, Offset: 0, Target&: S, Addend: 0);
919 }
920 }
921 }
922
923 return Error::success();
924}
925
926Error COFFPlatform::COFFPlatformPlugin::
927 registerObjectPlatformSectionsInBootstrap(jitlink::LinkGraph &G,
928 JITDylib &JD) {
929 std::lock_guard<std::mutex> Lock(CP.PlatformMutex);
930 auto HeaderAddr = CP.JITDylibToHeaderAddr[&JD];
931 COFFObjectSectionsMap ObjSecs;
932 for (auto &S : G.sections()) {
933 jitlink::SectionRange Range(S);
934 if (Range.getSize())
935 ObjSecs.push_back(Elt: std::make_pair(x: S.getName().str(), y: Range.getRange()));
936 }
937
938 G.allocActions().push_back(
939 x: {.Finalize: {},
940 .Dealloc: cantFail(
941 ValOrErr: WrapperFunctionCall::Create<SPSCOFFDeregisterObjectSectionsArgs>(
942 FnAddr: CP.orc_rt_coff_deregister_object_sections, Args: HeaderAddr,
943 Args: ObjSecs))});
944
945 auto &BState = CP.JDBootstrapStates[&JD];
946 BState.ObjectSectionsMaps.push_back(x: std::move(ObjSecs));
947
948 // Collect static initializers
949 for (auto &S : G.sections())
950 if (isCOFFInitializerSection(Name: S.getName()))
951 for (auto *B : S.blocks()) {
952 if (B->edges_empty())
953 continue;
954 for (auto &E : B->edges())
955 BState.Initializers.push_back(Elt: std::make_pair(
956 x: S.getName().str(), y: E.getTarget().getAddress() + E.getAddend()));
957 }
958
959 return Error::success();
960}
961
962} // End namespace orc.
963} // End namespace llvm.
964