1//===--------- LLJIT.cpp - An ORC-based JIT for compiling LLVM IR ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/Orc/LLJIT.h"
10
11#include "llvm/Analysis/TargetLibraryInfo.h"
12#include "llvm/Config/llvm-config.h" // for LLVM_ENABLE_THREADS
13#include "llvm/ExecutionEngine/Orc/COFFPlatform.h"
14#include "llvm/ExecutionEngine/Orc/EHFrameRegistrationPlugin.h"
15#include "llvm/ExecutionEngine/Orc/ELFNixPlatform.h"
16#include "llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h"
17#include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
18#include "llvm/ExecutionEngine/Orc/MachOPlatform.h"
19#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
20#include "llvm/ExecutionEngine/Orc/ObjectTransformLayer.h"
21#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
22#include "llvm/ExecutionEngine/Orc/SelfExecutorProcessControl.h"
23#include "llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h"
24#include "llvm/ExecutionEngine/Orc/UnwindInfoRegistrationPlugin.h"
25#include "llvm/ExecutionEngine/SectionMemoryManager.h"
26#include "llvm/IR/GlobalVariable.h"
27#include "llvm/IR/IRBuilder.h"
28#include "llvm/IR/Mangler.h"
29#include "llvm/IR/Module.h"
30#include "llvm/Support/DynamicLibrary.h"
31
32#define DEBUG_TYPE "orc"
33
34using namespace llvm;
35using namespace llvm::orc;
36
37namespace {
38
39/// Adds helper function decls and wrapper functions that call the helper with
40/// some additional prefix arguments.
41///
42/// E.g. For wrapper "foo" with type i8(i8, i64), helper "bar", and prefix
43/// args i32 4 and i16 12345, this function will add:
44///
45/// declare i8 @bar(i32, i16, i8, i64)
46///
47/// define i8 @foo(i8, i64) {
48/// entry:
49/// %2 = call i8 @bar(i32 4, i16 12345, i8 %0, i64 %1)
50/// ret i8 %2
51/// }
52///
53Function *addHelperAndWrapper(Module &M, StringRef WrapperName,
54 FunctionType *WrapperFnType,
55 GlobalValue::VisibilityTypes WrapperVisibility,
56 StringRef HelperName,
57 ArrayRef<Value *> HelperPrefixArgs) {
58 std::vector<Type *> HelperArgTypes;
59 for (auto *Arg : HelperPrefixArgs)
60 HelperArgTypes.push_back(x: Arg->getType());
61 llvm::append_range(C&: HelperArgTypes, R: WrapperFnType->params());
62 auto *HelperFnType =
63 FunctionType::get(Result: WrapperFnType->getReturnType(), Params: HelperArgTypes, isVarArg: false);
64 auto *HelperFn = Function::Create(Ty: HelperFnType, Linkage: GlobalValue::ExternalLinkage,
65 N: HelperName, M);
66
67 auto *WrapperFn = Function::Create(
68 Ty: WrapperFnType, Linkage: GlobalValue::ExternalLinkage, N: WrapperName, M);
69 WrapperFn->setVisibility(WrapperVisibility);
70
71 auto *EntryBlock = BasicBlock::Create(Context&: M.getContext(), Name: "entry", Parent: WrapperFn);
72 IRBuilder<> IB(EntryBlock);
73
74 std::vector<Value *> HelperArgs;
75 llvm::append_range(C&: HelperArgs, R&: HelperPrefixArgs);
76 for (auto &Arg : WrapperFn->args())
77 HelperArgs.push_back(x: &Arg);
78 auto *HelperResult = IB.CreateCall(Callee: HelperFn, Args: HelperArgs);
79 if (HelperFn->getReturnType()->isVoidTy())
80 IB.CreateRetVoid();
81 else
82 IB.CreateRet(V: HelperResult);
83
84 return WrapperFn;
85}
86
87class GenericLLVMIRPlatformSupport;
88
89/// orc::Platform component of Generic LLVM IR Platform support.
90/// Just forwards calls to the GenericLLVMIRPlatformSupport class below.
91class GenericLLVMIRPlatform : public Platform {
92public:
93 GenericLLVMIRPlatform(GenericLLVMIRPlatformSupport &S) : S(S) {}
94 Error setupJITDylib(JITDylib &JD) override;
95 Error teardownJITDylib(JITDylib &JD) override;
96 Error notifyAdding(ResourceTracker &RT,
97 const MaterializationUnit &MU) override;
98 Error notifyRemoving(ResourceTracker &RT) override {
99 // Noop -- Nothing to do (yet).
100 return Error::success();
101 }
102
103private:
104 GenericLLVMIRPlatformSupport &S;
105};
106
107/// This transform parses llvm.global_ctors to produce a single initialization
108/// function for the module, records the function, then deletes
109/// llvm.global_ctors.
110class GlobalCtorDtorScraper {
111public:
112 GlobalCtorDtorScraper(GenericLLVMIRPlatformSupport &PS,
113 StringRef InitFunctionPrefix,
114 StringRef DeInitFunctionPrefix)
115 : PS(PS), InitFunctionPrefix(InitFunctionPrefix),
116 DeInitFunctionPrefix(DeInitFunctionPrefix) {}
117 Expected<ThreadSafeModule> operator()(ThreadSafeModule TSM,
118 MaterializationResponsibility &R);
119
120private:
121 GenericLLVMIRPlatformSupport &PS;
122 StringRef InitFunctionPrefix;
123 StringRef DeInitFunctionPrefix;
124};
125
126/// Generic IR Platform Support
127///
128/// Scrapes llvm.global_ctors and llvm.global_dtors and replaces them with
129/// specially named 'init' and 'deinit'. Injects definitions / interposes for
130/// some runtime API, including __cxa_atexit, dlopen, and dlclose.
131class GenericLLVMIRPlatformSupport : public LLJIT::PlatformSupport {
132public:
133 GenericLLVMIRPlatformSupport(LLJIT &J, JITDylib &PlatformJD)
134 : J(J), InitFunctionPrefix(J.mangle(UnmangledName: "__orc_init_func.")),
135 DeInitFunctionPrefix(J.mangle(UnmangledName: "__orc_deinit_func.")) {
136
137 getExecutionSession().setPlatform(
138 std::make_unique<GenericLLVMIRPlatform>(args&: *this));
139
140 setInitTransform(J, T: GlobalCtorDtorScraper(*this, InitFunctionPrefix,
141 DeInitFunctionPrefix));
142
143 SymbolMap StdInterposes;
144
145 StdInterposes[J.mangleAndIntern(UnmangledName: "__lljit.platform_support_instance")] = {
146 ExecutorAddr::fromPtr(Ptr: this), JITSymbolFlags::Exported};
147 StdInterposes[J.mangleAndIntern(UnmangledName: "__lljit.cxa_atexit_helper")] = {
148 ExecutorAddr::fromPtr(Ptr: registerCxaAtExitHelper), JITSymbolFlags()};
149
150 cantFail(Err: PlatformJD.define(MU: absoluteSymbols(Symbols: std::move(StdInterposes))));
151 cantFail(Err: setupJITDylib(PlatformJD));
152 cantFail(Err: J.addIRModule(JD&: PlatformJD, TSM: createPlatformRuntimeModule()));
153 }
154
155 ExecutionSession &getExecutionSession() { return J.getExecutionSession(); }
156
157 /// Adds a module that defines the __dso_handle global.
158 Error setupJITDylib(JITDylib &JD) {
159
160 // Add per-jitdylib standard interposes.
161 SymbolMap PerJDInterposes;
162 PerJDInterposes[J.mangleAndIntern(UnmangledName: "__lljit.run_atexits_helper")] = {
163 ExecutorAddr::fromPtr(Ptr: runAtExitsHelper), JITSymbolFlags()};
164 PerJDInterposes[J.mangleAndIntern(UnmangledName: "__lljit.atexit_helper")] = {
165 ExecutorAddr::fromPtr(Ptr: registerAtExitHelper), JITSymbolFlags()};
166 cantFail(Err: JD.define(MU: absoluteSymbols(Symbols: std::move(PerJDInterposes))));
167
168 auto Ctx = std::make_unique<LLVMContext>();
169 auto M = std::make_unique<Module>(args: "__standard_lib", args&: *Ctx);
170 M->setDataLayout(J.getDataLayout());
171
172 auto *Int64Ty = Type::getInt64Ty(C&: *Ctx);
173 auto *DSOHandle = new GlobalVariable(
174 *M, Int64Ty, true, GlobalValue::ExternalLinkage,
175 ConstantInt::get(Ty: Int64Ty, V: reinterpret_cast<uintptr_t>(&JD)),
176 "__dso_handle");
177 DSOHandle->setVisibility(GlobalValue::DefaultVisibility);
178 DSOHandle->setInitializer(
179 ConstantInt::get(Ty: Int64Ty, V: ExecutorAddr::fromPtr(Ptr: &JD).getValue()));
180
181 auto *GenericIRPlatformSupportTy =
182 StructType::create(Context&: *Ctx, Name: "lljit.GenericLLJITIRPlatformSupport");
183
184 auto *PlatformInstanceDecl = new GlobalVariable(
185 *M, GenericIRPlatformSupportTy, true, GlobalValue::ExternalLinkage,
186 nullptr, "__lljit.platform_support_instance");
187
188 auto *VoidTy = Type::getVoidTy(C&: *Ctx);
189 addHelperAndWrapper(
190 M&: *M, WrapperName: "__lljit_run_atexits", WrapperFnType: FunctionType::get(Result: VoidTy, Params: {}, isVarArg: false),
191 WrapperVisibility: GlobalValue::HiddenVisibility, HelperName: "__lljit.run_atexits_helper",
192 HelperPrefixArgs: {PlatformInstanceDecl, DSOHandle});
193
194 auto *IntTy = Type::getIntNTy(C&: *Ctx, N: sizeof(int) * CHAR_BIT);
195 auto *AtExitCallbackPtrTy = PointerType::getUnqual(C&: *Ctx);
196 auto *AtExit = addHelperAndWrapper(
197 M&: *M, WrapperName: "atexit", WrapperFnType: FunctionType::get(Result: IntTy, Params: {AtExitCallbackPtrTy}, isVarArg: false),
198 WrapperVisibility: GlobalValue::HiddenVisibility, HelperName: "__lljit.atexit_helper",
199 HelperPrefixArgs: {PlatformInstanceDecl, DSOHandle});
200 Attribute::AttrKind AtExitExtAttr =
201 TargetLibraryInfo::getExtAttrForI32Return(T: J.getTargetTriple());
202 if (AtExitExtAttr != Attribute::None)
203 AtExit->addRetAttr(Kind: AtExitExtAttr);
204
205 return J.addIRModule(JD, TSM: ThreadSafeModule(std::move(M), std::move(Ctx)));
206 }
207
208 Error notifyAdding(ResourceTracker &RT, const MaterializationUnit &MU) {
209 auto &JD = RT.getJITDylib();
210 if (auto &InitSym = MU.getInitializerSymbol())
211 InitSymbols[&JD].add(Name: InitSym, Flags: SymbolLookupFlags::WeaklyReferencedSymbol);
212 else {
213 // If there's no identified init symbol attached, but there is a symbol
214 // with the GenericIRPlatform::InitFunctionPrefix, then treat that as
215 // an init function. Add the symbol to both the InitSymbols map (which
216 // will trigger a lookup to materialize the module) and the InitFunctions
217 // map (which holds the names of the symbols to execute).
218 for (auto &KV : MU.getSymbols())
219 if ((*KV.first).starts_with(Prefix: InitFunctionPrefix)) {
220 InitSymbols[&JD].add(Name: KV.first,
221 Flags: SymbolLookupFlags::WeaklyReferencedSymbol);
222 InitFunctions[&JD].add(Name: KV.first);
223 } else if ((*KV.first).starts_with(Prefix: DeInitFunctionPrefix)) {
224 DeInitFunctions[&JD].add(Name: KV.first);
225 }
226 }
227 return Error::success();
228 }
229
230 Error initialize(JITDylib &JD) override {
231 LLVM_DEBUG({
232 dbgs() << "GenericLLVMIRPlatformSupport getting initializers to run\n";
233 });
234 if (auto Initializers = getInitializers(JD)) {
235 LLVM_DEBUG(
236 { dbgs() << "GenericLLVMIRPlatformSupport running initializers\n"; });
237 for (auto InitFnAddr : *Initializers) {
238 LLVM_DEBUG({
239 dbgs() << " Running init " << formatv("{0:x16}", InitFnAddr)
240 << "...\n";
241 });
242 auto *InitFn = InitFnAddr.toPtr<void (*)()>();
243 InitFn();
244 }
245 } else
246 return Initializers.takeError();
247 return Error::success();
248 }
249
250 Error deinitialize(JITDylib &JD) override {
251 LLVM_DEBUG({
252 dbgs() << "GenericLLVMIRPlatformSupport getting deinitializers to run\n";
253 });
254 if (auto Deinitializers = getDeinitializers(JD)) {
255 LLVM_DEBUG({
256 dbgs() << "GenericLLVMIRPlatformSupport running deinitializers\n";
257 });
258 for (auto DeinitFnAddr : *Deinitializers) {
259 LLVM_DEBUG({
260 dbgs() << " Running deinit " << formatv("{0:x16}", DeinitFnAddr)
261 << "...\n";
262 });
263 auto *DeinitFn = DeinitFnAddr.toPtr<void (*)()>();
264 DeinitFn();
265 }
266 } else
267 return Deinitializers.takeError();
268
269 return Error::success();
270 }
271
272 void registerInitFunc(JITDylib &JD, SymbolStringPtr InitName) {
273 getExecutionSession().runSessionLocked(
274 F: [&]() { InitFunctions[&JD].add(Name: InitName); });
275 }
276
277 void registerDeInitFunc(JITDylib &JD, SymbolStringPtr DeInitName) {
278 getExecutionSession().runSessionLocked(
279 F: [&]() { DeInitFunctions[&JD].add(Name: DeInitName); });
280 }
281
282private:
283 Expected<std::vector<ExecutorAddr>> getInitializers(JITDylib &JD) {
284 if (auto Err = issueInitLookups(JD))
285 return std::move(Err);
286
287 DenseMap<JITDylib *, SymbolLookupSet> LookupSymbols;
288 std::vector<JITDylibSP> DFSLinkOrder;
289
290 if (auto Err = getExecutionSession().runSessionLocked(F: [&]() -> Error {
291 if (auto DFSLinkOrderOrErr = JD.getDFSLinkOrder())
292 DFSLinkOrder = std::move(*DFSLinkOrderOrErr);
293 else
294 return DFSLinkOrderOrErr.takeError();
295
296 for (auto &NextJD : DFSLinkOrder) {
297 auto IFItr = InitFunctions.find(Val: NextJD.get());
298 if (IFItr != InitFunctions.end()) {
299 LookupSymbols[NextJD.get()] = std::move(IFItr->second);
300 InitFunctions.erase(I: IFItr);
301 }
302 }
303 return Error::success();
304 }))
305 return std::move(Err);
306
307 LLVM_DEBUG({
308 dbgs() << "JITDylib init order is [ ";
309 for (auto &JD : llvm::reverse(DFSLinkOrder))
310 dbgs() << "\"" << JD->getName() << "\" ";
311 dbgs() << "]\n";
312 dbgs() << "Looking up init functions:\n";
313 for (auto &KV : LookupSymbols)
314 dbgs() << " \"" << KV.first->getName() << "\": " << KV.second << "\n";
315 });
316
317 auto &ES = getExecutionSession();
318 auto LookupResult = Platform::lookupInitSymbols(ES, InitSyms: LookupSymbols);
319
320 if (!LookupResult)
321 return LookupResult.takeError();
322
323 std::vector<ExecutorAddr> Initializers;
324 while (!DFSLinkOrder.empty()) {
325 auto &NextJD = *DFSLinkOrder.back();
326 DFSLinkOrder.pop_back();
327 auto InitsItr = LookupResult->find(Val: &NextJD);
328 if (InitsItr == LookupResult->end())
329 continue;
330 for (auto &KV : InitsItr->second)
331 Initializers.push_back(x: KV.second.getAddress());
332 }
333
334 return Initializers;
335 }
336
337 Expected<std::vector<ExecutorAddr>> getDeinitializers(JITDylib &JD) {
338 auto &ES = getExecutionSession();
339
340 auto LLJITRunAtExits = J.mangleAndIntern(UnmangledName: "__lljit_run_atexits");
341
342 DenseMap<JITDylib *, SymbolLookupSet> LookupSymbols;
343 std::vector<JITDylibSP> DFSLinkOrder;
344
345 if (auto Err = ES.runSessionLocked(F: [&]() -> Error {
346 if (auto DFSLinkOrderOrErr = JD.getDFSLinkOrder())
347 DFSLinkOrder = std::move(*DFSLinkOrderOrErr);
348 else
349 return DFSLinkOrderOrErr.takeError();
350
351 for (auto &NextJD : DFSLinkOrder) {
352 auto &JDLookupSymbols = LookupSymbols[NextJD.get()];
353 auto DIFItr = DeInitFunctions.find(Val: NextJD.get());
354 if (DIFItr != DeInitFunctions.end()) {
355 LookupSymbols[NextJD.get()] = std::move(DIFItr->second);
356 DeInitFunctions.erase(I: DIFItr);
357 }
358 JDLookupSymbols.add(Name: LLJITRunAtExits,
359 Flags: SymbolLookupFlags::WeaklyReferencedSymbol);
360 }
361 return Error::success();
362 }))
363 return std::move(Err);
364
365 LLVM_DEBUG({
366 dbgs() << "JITDylib deinit order is [ ";
367 for (auto &JD : DFSLinkOrder)
368 dbgs() << "\"" << JD->getName() << "\" ";
369 dbgs() << "]\n";
370 dbgs() << "Looking up deinit functions:\n";
371 for (auto &KV : LookupSymbols)
372 dbgs() << " \"" << KV.first->getName() << "\": " << KV.second << "\n";
373 });
374
375 auto LookupResult = Platform::lookupInitSymbols(ES, InitSyms: LookupSymbols);
376
377 if (!LookupResult)
378 return LookupResult.takeError();
379
380 std::vector<ExecutorAddr> DeInitializers;
381 for (auto &NextJD : DFSLinkOrder) {
382 auto DeInitsItr = LookupResult->find(Val: NextJD.get());
383 assert(DeInitsItr != LookupResult->end() &&
384 "Every JD should have at least __lljit_run_atexits");
385
386 auto RunAtExitsItr = DeInitsItr->second.find(Val: LLJITRunAtExits);
387 if (RunAtExitsItr != DeInitsItr->second.end())
388 DeInitializers.push_back(x: RunAtExitsItr->second.getAddress());
389
390 for (auto &KV : DeInitsItr->second)
391 if (KV.first != LLJITRunAtExits)
392 DeInitializers.push_back(x: KV.second.getAddress());
393 }
394
395 return DeInitializers;
396 }
397
398 /// Issue lookups for all init symbols required to initialize JD (and any
399 /// JITDylibs that it depends on).
400 Error issueInitLookups(JITDylib &JD) {
401 DenseMap<JITDylib *, SymbolLookupSet> RequiredInitSymbols;
402 std::vector<JITDylibSP> DFSLinkOrder;
403
404 if (auto Err = getExecutionSession().runSessionLocked(F: [&]() -> Error {
405 if (auto DFSLinkOrderOrErr = JD.getDFSLinkOrder())
406 DFSLinkOrder = std::move(*DFSLinkOrderOrErr);
407 else
408 return DFSLinkOrderOrErr.takeError();
409
410 for (auto &NextJD : DFSLinkOrder) {
411 auto ISItr = InitSymbols.find(Val: NextJD.get());
412 if (ISItr != InitSymbols.end()) {
413 RequiredInitSymbols[NextJD.get()] = std::move(ISItr->second);
414 InitSymbols.erase(I: ISItr);
415 }
416 }
417 return Error::success();
418 }))
419 return Err;
420
421 return Platform::lookupInitSymbols(ES&: getExecutionSession(),
422 InitSyms: RequiredInitSymbols)
423 .takeError();
424 }
425
426 static void registerCxaAtExitHelper(void *Self, void (*F)(void *), void *Ctx,
427 void *DSOHandle) {
428 LLVM_DEBUG({
429 dbgs() << "Registering cxa atexit function " << (void *)F << " for JD "
430 << (*static_cast<JITDylib **>(DSOHandle))->getName() << "\n";
431 });
432 static_cast<GenericLLVMIRPlatformSupport *>(Self)->AtExitMgr.registerAtExit(
433 F, Ctx, DSOHandle);
434 }
435
436 static void registerAtExitHelper(void *Self, void *DSOHandle, void (*F)()) {
437 LLVM_DEBUG({
438 dbgs() << "Registering atexit function " << (void *)F << " for JD "
439 << (*static_cast<JITDylib **>(DSOHandle))->getName() << "\n";
440 });
441 static_cast<GenericLLVMIRPlatformSupport *>(Self)->AtExitMgr.registerAtExit(
442 F: reinterpret_cast<void (*)(void *)>(F), Ctx: nullptr, DSOHandle);
443 }
444
445 static void runAtExitsHelper(void *Self, void *DSOHandle) {
446 LLVM_DEBUG({
447 dbgs() << "Running atexit functions for JD "
448 << (*static_cast<JITDylib **>(DSOHandle))->getName() << "\n";
449 });
450 static_cast<GenericLLVMIRPlatformSupport *>(Self)->AtExitMgr.runAtExits(
451 DSOHandle);
452 }
453
454 // Constructs an LLVM IR module containing platform runtime globals,
455 // functions, and interposes.
456 ThreadSafeModule createPlatformRuntimeModule() {
457 auto Ctx = std::make_unique<LLVMContext>();
458 auto M = std::make_unique<Module>(args: "__standard_lib", args&: *Ctx);
459 M->setDataLayout(J.getDataLayout());
460
461 auto *GenericIRPlatformSupportTy =
462 StructType::create(Context&: *Ctx, Name: "lljit.GenericLLJITIRPlatformSupport");
463
464 auto *PlatformInstanceDecl = new GlobalVariable(
465 *M, GenericIRPlatformSupportTy, true, GlobalValue::ExternalLinkage,
466 nullptr, "__lljit.platform_support_instance");
467
468 auto *IntTy = Type::getIntNTy(C&: *Ctx, N: sizeof(int) * CHAR_BIT);
469 auto *BytePtrTy = PointerType::getUnqual(C&: *Ctx);
470 auto *CxaAtExitCallbackPtrTy = PointerType::getUnqual(C&: *Ctx);
471
472 auto *CxaAtExit = addHelperAndWrapper(
473 M&: *M, WrapperName: "__cxa_atexit",
474 WrapperFnType: FunctionType::get(Result: IntTy, Params: {CxaAtExitCallbackPtrTy, BytePtrTy, BytePtrTy},
475 isVarArg: false),
476 WrapperVisibility: GlobalValue::DefaultVisibility, HelperName: "__lljit.cxa_atexit_helper",
477 HelperPrefixArgs: {PlatformInstanceDecl});
478 Attribute::AttrKind CxaAtExitExtAttr =
479 TargetLibraryInfo::getExtAttrForI32Return(T: J.getTargetTriple());
480 if (CxaAtExitExtAttr != Attribute::None)
481 CxaAtExit->addRetAttr(Kind: CxaAtExitExtAttr);
482
483 return ThreadSafeModule(std::move(M), std::move(Ctx));
484 }
485
486 LLJIT &J;
487 std::string InitFunctionPrefix;
488 std::string DeInitFunctionPrefix;
489 DenseMap<JITDylib *, SymbolLookupSet> InitSymbols;
490 DenseMap<JITDylib *, SymbolLookupSet> InitFunctions;
491 DenseMap<JITDylib *, SymbolLookupSet> DeInitFunctions;
492 ItaniumCXAAtExitSupport AtExitMgr;
493};
494
495Error GenericLLVMIRPlatform::setupJITDylib(JITDylib &JD) {
496 return S.setupJITDylib(JD);
497}
498
499Error GenericLLVMIRPlatform::teardownJITDylib(JITDylib &JD) {
500 return Error::success();
501}
502
503Error GenericLLVMIRPlatform::notifyAdding(ResourceTracker &RT,
504 const MaterializationUnit &MU) {
505 return S.notifyAdding(RT, MU);
506}
507
508Expected<ThreadSafeModule>
509GlobalCtorDtorScraper::operator()(ThreadSafeModule TSM,
510 MaterializationResponsibility &R) {
511 auto Err = TSM.withModuleDo(F: [&](Module &M) -> Error {
512 auto &Ctx = M.getContext();
513 auto *GlobalCtors = M.getNamedGlobal(Name: "llvm.global_ctors");
514 auto *GlobalDtors = M.getNamedGlobal(Name: "llvm.global_dtors");
515
516 auto RegisterCOrDtors = [&](GlobalVariable *GlobalCOrDtors,
517 bool isCtor) -> Error {
518 // If there's no llvm.global_c/dtor or it's just a decl then skip.
519 if (!GlobalCOrDtors || GlobalCOrDtors->isDeclaration())
520 return Error::success();
521 std::string InitOrDeInitFunctionName;
522 if (isCtor)
523 raw_string_ostream(InitOrDeInitFunctionName)
524 << InitFunctionPrefix << M.getModuleIdentifier();
525 else
526 raw_string_ostream(InitOrDeInitFunctionName)
527 << DeInitFunctionPrefix << M.getModuleIdentifier();
528
529 MangleAndInterner Mangle(PS.getExecutionSession(), M.getDataLayout());
530 auto InternedInitOrDeInitName = Mangle(InitOrDeInitFunctionName);
531 if (auto Err = R.defineMaterializing(
532 SymbolFlags: {{InternedInitOrDeInitName, JITSymbolFlags::Callable}}))
533 return Err;
534
535 auto *InitOrDeInitFunc = Function::Create(
536 Ty: FunctionType::get(Result: Type::getVoidTy(C&: Ctx), Params: {}, isVarArg: false),
537 Linkage: GlobalValue::ExternalLinkage, N: InitOrDeInitFunctionName, M: &M);
538 InitOrDeInitFunc->setVisibility(GlobalValue::HiddenVisibility);
539 std::vector<std::pair<Function *, unsigned>> InitsOrDeInits;
540 auto COrDtors = isCtor ? getConstructors(M) : getDestructors(M);
541
542 for (auto E : COrDtors)
543 InitsOrDeInits.push_back(x: std::make_pair(x&: E.Func, y&: E.Priority));
544 llvm::stable_sort(Range&: InitsOrDeInits, C: llvm::less_second());
545
546 auto *InitOrDeInitFuncEntryBlock =
547 BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: InitOrDeInitFunc);
548 IRBuilder<> IB(InitOrDeInitFuncEntryBlock);
549 for (auto &KV : InitsOrDeInits)
550 IB.CreateCall(Callee: KV.first);
551 IB.CreateRetVoid();
552
553 if (isCtor)
554 PS.registerInitFunc(JD&: R.getTargetJITDylib(), InitName: InternedInitOrDeInitName);
555 else
556 PS.registerDeInitFunc(JD&: R.getTargetJITDylib(), DeInitName: InternedInitOrDeInitName);
557
558 GlobalCOrDtors->eraseFromParent();
559 return Error::success();
560 };
561
562 if (auto Err = RegisterCOrDtors(GlobalCtors, true))
563 return Err;
564 if (auto Err = RegisterCOrDtors(GlobalDtors, false))
565 return Err;
566
567 return Error::success();
568 });
569
570 if (Err)
571 return std::move(Err);
572
573 return std::move(TSM);
574}
575
576/// Inactive Platform Support
577///
578/// Explicitly disables platform support. JITDylibs are not scanned for special
579/// init/deinit symbols. No runtime API interposes are injected.
580class InactivePlatformSupport : public LLJIT::PlatformSupport {
581public:
582 InactivePlatformSupport() = default;
583
584 Error initialize(JITDylib &JD) override {
585 LLVM_DEBUG(dbgs() << "InactivePlatformSupport: no initializers running for "
586 << JD.getName() << "\n");
587 return Error::success();
588 }
589
590 Error deinitialize(JITDylib &JD) override {
591 LLVM_DEBUG(
592 dbgs() << "InactivePlatformSupport: no deinitializers running for "
593 << JD.getName() << "\n");
594 return Error::success();
595 }
596};
597
598} // end anonymous namespace
599
600namespace llvm {
601namespace orc {
602
603Error ORCPlatformSupport::initialize(orc::JITDylib &JD) {
604 using llvm::orc::shared::SPSExecutorAddr;
605 using llvm::orc::shared::SPSString;
606 using SPSDLOpenSig = SPSExecutorAddr(SPSString, int32_t);
607 using SPSDLUpdateSig = int32_t(SPSExecutorAddr);
608 enum dlopen_mode : int32_t {
609 ORC_RT_RTLD_LAZY = 0x1,
610 ORC_RT_RTLD_NOW = 0x2,
611 ORC_RT_RTLD_LOCAL = 0x4,
612 ORC_RT_RTLD_GLOBAL = 0x8
613 };
614
615 auto &ES = J.getExecutionSession();
616 auto MainSearchOrder = J.getMainJITDylib().withLinkOrderDo(
617 F: [](const JITDylibSearchOrder &SO) { return SO; });
618 StringRef WrapperToCall = "__orc_rt_jit_dlopen_wrapper";
619 bool dlupdate = false;
620 if (InitializedDylib.contains(Ptr: &JD)) {
621 WrapperToCall = "__orc_rt_jit_dlupdate_wrapper";
622 dlupdate = true;
623 } else
624 InitializedDylib.insert(Ptr: &JD);
625
626 if (auto WrapperAddr =
627 ES.lookup(SearchOrder: MainSearchOrder, Symbol: J.mangleAndIntern(UnmangledName: WrapperToCall))) {
628 if (dlupdate) {
629 int32_t result;
630 auto E = ES.callSPSWrapper<SPSDLUpdateSig>(WrapperFnAddr: WrapperAddr->getAddress(),
631 WrapperCallArgs&: result, WrapperCallArgs&: DSOHandles[&JD]);
632 if (E)
633 return E;
634 else if (result)
635 return make_error<StringError>(Args: "dlupdate failed",
636 Args: inconvertibleErrorCode());
637 } else
638 return ES.callSPSWrapper<SPSDLOpenSig>(WrapperFnAddr: WrapperAddr->getAddress(),
639 WrapperCallArgs&: DSOHandles[&JD], WrapperCallArgs: JD.getName(),
640 WrapperCallArgs: int32_t(ORC_RT_RTLD_LAZY));
641 } else
642 return WrapperAddr.takeError();
643
644 return Error::success();
645}
646
647Error ORCPlatformSupport::deinitialize(orc::JITDylib &JD) {
648 using llvm::orc::shared::SPSExecutorAddr;
649 using SPSDLCloseSig = int32_t(SPSExecutorAddr);
650
651 auto &ES = J.getExecutionSession();
652 auto MainSearchOrder = J.getMainJITDylib().withLinkOrderDo(
653 F: [](const JITDylibSearchOrder &SO) { return SO; });
654
655 if (auto WrapperAddr = ES.lookup(
656 SearchOrder: MainSearchOrder, Symbol: J.mangleAndIntern(UnmangledName: "__orc_rt_jit_dlclose_wrapper"))) {
657 int32_t result;
658 auto E = J.getExecutionSession().callSPSWrapper<SPSDLCloseSig>(
659 WrapperFnAddr: WrapperAddr->getAddress(), WrapperCallArgs&: result, WrapperCallArgs&: DSOHandles[&JD]);
660 if (E)
661 return E;
662 else if (result)
663 return make_error<StringError>(Args: "dlclose failed",
664 Args: inconvertibleErrorCode());
665 DSOHandles.erase(Val: &JD);
666 InitializedDylib.erase(Ptr: &JD);
667 } else
668 return WrapperAddr.takeError();
669 return Error::success();
670}
671
672void LLJIT::PlatformSupport::setInitTransform(
673 LLJIT &J, IRTransformLayer::TransformFunction T) {
674 J.InitHelperTransformLayer->setTransform(std::move(T));
675}
676
677LLJIT::PlatformSupport::~PlatformSupport() = default;
678
679Error LLJITBuilderState::prepareForConstruction() {
680
681 LLVM_DEBUG(dbgs() << "Preparing to create LLJIT instance...\n");
682
683 if (!JTMB) {
684 LLVM_DEBUG({
685 dbgs() << " No explicitly set JITTargetMachineBuilder. "
686 "Detecting host...\n";
687 });
688 if (auto JTMBOrErr = JITTargetMachineBuilder::detectHost())
689 JTMB = std::move(*JTMBOrErr);
690 else
691 return JTMBOrErr.takeError();
692 }
693
694 if ((ES || EPC) && NumCompileThreads)
695 return make_error<StringError>(
696 Args: "NumCompileThreads cannot be used with a custom ExecutionSession or "
697 "ExecutorProcessControl",
698 Args: inconvertibleErrorCode());
699
700#if !LLVM_ENABLE_THREADS
701 if (NumCompileThreads)
702 return make_error<StringError>(
703 "LLJIT num-compile-threads is " + Twine(NumCompileThreads) +
704 " but LLVM was compiled with LLVM_ENABLE_THREADS=Off",
705 inconvertibleErrorCode());
706#endif // !LLVM_ENABLE_THREADS
707
708 // Only used in debug builds.
709 [[maybe_unused]] bool ConcurrentCompilationSettingDefaulted =
710 !SupportConcurrentCompilation;
711
712 if (!SupportConcurrentCompilation) {
713#if LLVM_ENABLE_THREADS
714 SupportConcurrentCompilation = NumCompileThreads || ES || EPC;
715#else
716 SupportConcurrentCompilation = false;
717#endif // LLVM_ENABLE_THREADS
718 } else {
719#if !LLVM_ENABLE_THREADS
720 if (*SupportConcurrentCompilation)
721 return make_error<StringError>(
722 "LLJIT concurrent compilation support requested, but LLVM was built "
723 "with LLVM_ENABLE_THREADS=Off",
724 inconvertibleErrorCode());
725#endif // !LLVM_ENABLE_THREADS
726 }
727
728 LLVM_DEBUG({
729 dbgs() << " JITTargetMachineBuilder is "
730 << JITTargetMachineBuilderPrinter(*JTMB, " ")
731 << " Pre-constructed ExecutionSession: " << (ES ? "Yes" : "No")
732 << "\n"
733 << " DataLayout: ";
734 if (DL)
735 dbgs() << DL->getStringRepresentation() << "\n";
736 else
737 dbgs() << "None (will be created by JITTargetMachineBuilder)\n";
738
739 dbgs() << " Custom object-linking-layer creator: "
740 << (CreateObjectLinkingLayer ? "Yes" : "No") << "\n"
741 << " Custom compile-function creator: "
742 << (CreateCompileFunction ? "Yes" : "No") << "\n"
743 << " Custom platform-setup function: "
744 << (SetUpPlatform ? "Yes" : "No") << "\n"
745 << " Support concurrent compilation: "
746 << (*SupportConcurrentCompilation ? "Yes" : "No");
747 if (ConcurrentCompilationSettingDefaulted)
748 dbgs() << " (defaulted based on ES / EPC / NumCompileThreads)\n";
749 else
750 dbgs() << "\n";
751 dbgs() << " Number of compile threads: " << NumCompileThreads << "\n";
752 });
753
754 // Create DL if not specified.
755 if (!DL) {
756 if (auto DLOrErr = JTMB->getDefaultDataLayoutForTarget())
757 DL = std::move(*DLOrErr);
758 else
759 return DLOrErr.takeError();
760 }
761
762 // If neither ES nor EPC has been set then create an EPC instance.
763 if (!ES && !EPC) {
764 LLVM_DEBUG({
765 dbgs() << "ExecutorProcessControl not specified, "
766 "Creating SelfExecutorProcessControl instance\n";
767 });
768
769 std::unique_ptr<TaskDispatcher> D = nullptr;
770#if LLVM_ENABLE_THREADS
771 if (*SupportConcurrentCompilation) {
772 std::optional<size_t> NumThreads = std ::nullopt;
773 if (NumCompileThreads)
774 NumThreads = NumCompileThreads;
775 D = std::make_unique<DynamicThreadPoolTaskDispatcher>(args&: NumThreads);
776 } else
777 D = std::make_unique<InPlaceTaskDispatcher>();
778#endif // LLVM_ENABLE_THREADS
779 if (auto EPCOrErr =
780 SelfExecutorProcessControl::Create(SSP: nullptr, D: std::move(D)))
781 EPC = std::move(*EPCOrErr);
782 else
783 return EPCOrErr.takeError();
784 } else if (EPC) {
785 LLVM_DEBUG({
786 dbgs() << "Using explicitly specified ExecutorProcessControl instance "
787 << EPC.get() << "\n";
788 });
789 } else {
790 LLVM_DEBUG({
791 dbgs() << "Using explicitly specified ExecutionSession instance "
792 << ES.get() << "\n";
793 });
794 }
795
796 // If the client didn't configure any linker options then auto-configure the
797 // JIT linker.
798 if (!CreateObjectLinkingLayer) {
799 auto &TT = JTMB->getTargetTriple();
800 bool UseJITLink = false;
801 switch (TT.getArch()) {
802 case Triple::riscv64:
803 case Triple::loongarch64:
804 UseJITLink = true;
805 break;
806 case Triple::aarch64:
807 UseJITLink = !TT.isOSBinFormatCOFF();
808 break;
809 case Triple::arm:
810 case Triple::armeb:
811 case Triple::thumb:
812 case Triple::thumbeb:
813 UseJITLink = TT.isOSBinFormatELF();
814 break;
815 case Triple::x86_64:
816 UseJITLink = !TT.isOSBinFormatCOFF();
817 break;
818 case Triple::ppc64:
819 UseJITLink = TT.isPPC64ELFv2ABI();
820 break;
821 case Triple::ppc64le:
822 UseJITLink = TT.isOSBinFormatELF();
823 break;
824 default:
825 break;
826 }
827 if (UseJITLink) {
828 if (!JTMB->getCodeModel())
829 JTMB->setCodeModel(CodeModel::Small);
830 JTMB->setRelocationModel(Reloc::PIC_);
831 CreateObjectLinkingLayer = [](ExecutionSession &ES,
832 jitlink::JITLinkMemoryManager &MemMgr)
833 -> Expected<std::unique_ptr<ObjectLayer>> {
834 return std::make_unique<ObjectLinkingLayer>(args&: ES, args&: MemMgr);
835 };
836 }
837 }
838
839 // If we need a process JITDylib but no setup function has been given then
840 // create a default one.
841 if (!SetupProcessSymbolsJITDylib && LinkProcessSymbolsByDefault) {
842 LLVM_DEBUG(dbgs() << "Creating default Process JD setup function\n");
843 SetupProcessSymbolsJITDylib = [](LLJIT &J) -> Expected<JITDylibSP> {
844 auto &JD =
845 J.getExecutionSession().createBareJITDylib(Name: "<Process Symbols>");
846 auto G = EPCDynamicLibrarySearchGenerator::GetForTargetProcess(
847 ES&: J.getExecutionSession(), DylibMgr&: J.getDylibMgr());
848 if (!G)
849 return G.takeError();
850 JD.addGenerator(DefGenerator: std::move(*G));
851 return &JD;
852 };
853 }
854
855 return Error::success();
856}
857
858LLJIT::~LLJIT() {
859 if (auto Err = ES->endSession())
860 ES->reportError(Err: std::move(Err));
861}
862
863JITDylibSP LLJIT::getProcessSymbolsJITDylib() { return ProcessSymbols; }
864
865JITDylibSP LLJIT::getPlatformJITDylib() { return Platform; }
866
867Expected<JITDylib &> LLJIT::createJITDylib(std::string Name) {
868 auto JD = ES->createJITDylib(Name: std::move(Name));
869 if (!JD)
870 return JD.takeError();
871
872 JD->addToLinkOrder(NewLinks: DefaultLinks);
873 return JD;
874}
875
876Expected<JITDylib &> LLJIT::loadPlatformDynamicLibrary(const char *Path) {
877 auto G = EPCDynamicLibrarySearchGenerator::Load(ES&: *ES, DylibMgr&: *DylibMgr, LibraryPath: Path);
878 if (!G)
879 return G.takeError();
880
881 if (auto *ExistingJD = ES->getJITDylibByName(Name: Path))
882 return *ExistingJD;
883
884 auto &JD = ES->createBareJITDylib(Name: Path);
885 JD.addGenerator(DefGenerator: std::move(*G));
886 return JD;
887}
888
889Error LLJIT::linkStaticLibraryInto(JITDylib &JD,
890 std::unique_ptr<MemoryBuffer> LibBuffer) {
891 auto G = StaticLibraryDefinitionGenerator::Create(L&: *ObjLinkingLayer,
892 ArchiveBuffer: std::move(LibBuffer));
893 if (!G)
894 return G.takeError();
895
896 JD.addGenerator(DefGenerator: std::move(*G));
897
898 return Error::success();
899}
900
901Error LLJIT::linkStaticLibraryInto(JITDylib &JD, const char *Path) {
902 auto G = StaticLibraryDefinitionGenerator::Load(L&: *ObjLinkingLayer, FileName: Path);
903 if (!G)
904 return G.takeError();
905
906 JD.addGenerator(DefGenerator: std::move(*G));
907
908 return Error::success();
909}
910
911Error LLJIT::addIRModule(ResourceTrackerSP RT, ThreadSafeModule TSM) {
912 assert(TSM && "Can not add null module");
913
914 if (auto Err =
915 TSM.withModuleDo(F: [&](Module &M) { return applyDataLayout(M); }))
916 return Err;
917
918 return InitHelperTransformLayer->add(RT: std::move(RT), TSM: std::move(TSM));
919}
920
921Error LLJIT::addIRModule(JITDylib &JD, ThreadSafeModule TSM) {
922 return addIRModule(RT: JD.getDefaultResourceTracker(), TSM: std::move(TSM));
923}
924
925Error LLJIT::addObjectFile(ResourceTrackerSP RT,
926 std::unique_ptr<MemoryBuffer> Obj) {
927 assert(Obj && "Can not add null object");
928
929 return ObjTransformLayer->add(RT: std::move(RT), O: std::move(Obj));
930}
931
932Error LLJIT::addObjectFile(JITDylib &JD, std::unique_ptr<MemoryBuffer> Obj) {
933 return addObjectFile(RT: JD.getDefaultResourceTracker(), Obj: std::move(Obj));
934}
935
936Expected<ExecutorAddr> LLJIT::lookupLinkerMangled(JITDylib &JD,
937 SymbolStringPtr Name) {
938 if (auto Sym = ES->lookup(
939 SearchOrder: makeJITDylibSearchOrder(JDs: &JD, Flags: JITDylibLookupFlags::MatchAllSymbols),
940 Symbol: Name))
941 return Sym->getAddress();
942 else
943 return Sym.takeError();
944}
945
946Expected<std::unique_ptr<jitlink::JITLinkMemoryManager>>
947LLJIT::createMemoryManager(LLJITBuilderState &S, ExecutionSession &ES) {
948 if (S.CreateMemoryManager)
949 return S.CreateMemoryManager(ES);
950 return ES.getExecutorProcessControl().createDefaultMemoryManager();
951}
952
953Expected<std::unique_ptr<ObjectLayer>>
954LLJIT::createObjectLinkingLayer(LLJITBuilderState &S, ExecutionSession &ES,
955 jitlink::JITLinkMemoryManager &MemMgr) {
956
957 // If the config state provided an ObjectLinkingLayer factory then use it.
958 if (S.CreateObjectLinkingLayer)
959 return S.CreateObjectLinkingLayer(ES, MemMgr);
960
961 // Otherwise default to creating an RTDyldObjectLinkingLayer that constructs
962 // a new SectionMemoryManager for each object.
963 auto GetMemMgr = [](const MemoryBuffer &) {
964 return std::make_unique<SectionMemoryManager>();
965 };
966 auto Layer =
967 std::make_unique<RTDyldObjectLinkingLayer>(args&: ES, args: std::move(GetMemMgr));
968
969 if (S.JTMB->getTargetTriple().isOSBinFormatCOFF()) {
970 Layer->setOverrideObjectFlagsWithResponsibilityFlags(true);
971 Layer->setAutoClaimResponsibilityForObjectSymbols(true);
972 }
973
974 if (S.JTMB->getTargetTriple().isOSBinFormatELF() &&
975 (S.JTMB->getTargetTriple().getArch() == Triple::ArchType::ppc64 ||
976 S.JTMB->getTargetTriple().getArch() == Triple::ArchType::ppc64le))
977 Layer->setAutoClaimResponsibilityForObjectSymbols(true);
978
979 // FIXME: Explicit conversion to std::unique_ptr<ObjectLayer> added to silence
980 // errors from some GCC / libstdc++ bots. Remove this conversion (i.e.
981 // just return ObjLinkingLayer) once those bots are upgraded.
982 return std::unique_ptr<ObjectLayer>(std::move(Layer));
983}
984
985Expected<std::unique_ptr<IRCompileLayer::IRCompiler>>
986LLJIT::createCompileFunction(LLJITBuilderState &S,
987 JITTargetMachineBuilder JTMB) {
988
989 /// If there is a custom compile function creator set then use it.
990 if (S.CreateCompileFunction)
991 return S.CreateCompileFunction(std::move(JTMB));
992
993 // If using a custom EPC then use a ConcurrentIRCompiler by default.
994 if (*S.SupportConcurrentCompilation)
995 return std::make_unique<ConcurrentIRCompiler>(args: std::move(JTMB));
996
997 auto TM = JTMB.createTargetMachine();
998 if (!TM)
999 return TM.takeError();
1000
1001 return std::make_unique<TMOwningSimpleCompiler>(args: std::move(*TM));
1002}
1003
1004LLJIT::LLJIT(LLJITBuilderState &S, Error &Err)
1005 : DL(std::move(*S.DL)), TT(S.JTMB->getTargetTriple()) {
1006
1007 ErrorAsOutParameter _(Err);
1008
1009 assert(!(S.EPC && S.ES) && "EPC and ES should not both be set");
1010
1011 if (S.EPC) {
1012 ES = std::make_unique<ExecutionSession>(args: std::move(S.EPC));
1013 } else if (S.ES)
1014 ES = std::move(S.ES);
1015 else {
1016 if (auto EPC = SelfExecutorProcessControl::Create()) {
1017 ES = std::make_unique<ExecutionSession>(args: std::move(*EPC));
1018 } else {
1019 Err = EPC.takeError();
1020 return;
1021 }
1022 }
1023
1024 if (auto MM = createMemoryManager(S, ES&: *ES))
1025 MemMgr = std::move(*MM);
1026 else {
1027 Err = MM.takeError();
1028 return;
1029 }
1030
1031 if (auto DM = ES->getExecutorProcessControl().createDefaultDylibMgr())
1032 DylibMgr = std::move(*DM);
1033 else {
1034 Err = DM.takeError();
1035 return;
1036 }
1037
1038 auto ObjLayer = createObjectLinkingLayer(S, ES&: *ES, MemMgr&: *MemMgr);
1039 if (!ObjLayer) {
1040 Err = ObjLayer.takeError();
1041 return;
1042 }
1043 ObjLinkingLayer = std::move(*ObjLayer);
1044 ObjTransformLayer =
1045 std::make_unique<ObjectTransformLayer>(args&: *ES, args&: *ObjLinkingLayer);
1046
1047 {
1048 auto CompileFunction = createCompileFunction(S, JTMB: std::move(*S.JTMB));
1049 if (!CompileFunction) {
1050 Err = CompileFunction.takeError();
1051 return;
1052 }
1053 CompileLayer = std::make_unique<IRCompileLayer>(
1054 args&: *ES, args&: *ObjTransformLayer, args: std::move(*CompileFunction));
1055 TransformLayer = std::make_unique<IRTransformLayer>(args&: *ES, args&: *CompileLayer);
1056 InitHelperTransformLayer =
1057 std::make_unique<IRTransformLayer>(args&: *ES, args&: *TransformLayer);
1058 }
1059
1060 if (*S.SupportConcurrentCompilation)
1061 InitHelperTransformLayer->setCloneToNewContextOnEmit(true);
1062
1063 if (S.SetupProcessSymbolsJITDylib) {
1064 if (auto ProcSymsJD = S.SetupProcessSymbolsJITDylib(*this)) {
1065 ProcessSymbols = ProcSymsJD->get();
1066 } else {
1067 Err = ProcSymsJD.takeError();
1068 return;
1069 }
1070 }
1071
1072 if (S.PrePlatformSetup)
1073 if ((Err = S.PrePlatformSetup(*this)))
1074 return;
1075
1076 if (!S.SetUpPlatform)
1077 S.SetUpPlatform = setUpGenericLLVMIRPlatform;
1078
1079 if (auto PlatformJDOrErr = S.SetUpPlatform(*this)) {
1080 Platform = PlatformJDOrErr->get();
1081 if (Platform)
1082 DefaultLinks.push_back(
1083 x: {Platform, JITDylibLookupFlags::MatchExportedSymbolsOnly});
1084 } else {
1085 Err = PlatformJDOrErr.takeError();
1086 return;
1087 }
1088
1089 if (S.LinkProcessSymbolsByDefault)
1090 DefaultLinks.push_back(
1091 x: {ProcessSymbols, JITDylibLookupFlags::MatchExportedSymbolsOnly});
1092
1093 if (auto MainOrErr = createJITDylib(Name: "main"))
1094 Main = &*MainOrErr;
1095 else {
1096 Err = MainOrErr.takeError();
1097 return;
1098 }
1099}
1100
1101std::string LLJIT::mangle(StringRef UnmangledName) const {
1102 std::string MangledName;
1103 {
1104 raw_string_ostream MangledNameStream(MangledName);
1105 Mangler::getNameWithPrefix(OS&: MangledNameStream, GVName: UnmangledName, DL);
1106 }
1107 return MangledName;
1108}
1109
1110Error LLJIT::applyDataLayout(Module &M) {
1111 if (M.getDataLayout().isDefault())
1112 M.setDataLayout(DL);
1113
1114 if (M.getDataLayout() != DL)
1115 return make_error<StringError>(
1116 Args: "Added modules have incompatible data layouts: " +
1117 M.getDataLayout().getStringRepresentation() + " (module) vs " +
1118 DL.getStringRepresentation() + " (jit)",
1119 Args: inconvertibleErrorCode());
1120
1121 return Error::success();
1122}
1123
1124Error setUpOrcPlatformManually(LLJIT &J) {
1125 LLVM_DEBUG({ dbgs() << "Setting up orc platform support for LLJIT\n"; });
1126 J.setPlatformSupport(std::make_unique<ORCPlatformSupport>(args&: J));
1127 return Error::success();
1128}
1129
1130class LoadAndLinkDynLibrary {
1131public:
1132 LoadAndLinkDynLibrary(LLJIT &J) : J(J) {}
1133 Error operator()(JITDylib &JD, StringRef DLLName) {
1134 if (!DLLName.ends_with_insensitive(Suffix: ".dll"))
1135 return make_error<StringError>(Args: "DLLName not ending with .dll",
1136 Args: inconvertibleErrorCode());
1137 auto DLLNameStr = DLLName.str(); // Guarantees null-termination.
1138 auto DLLJD = J.loadPlatformDynamicLibrary(Path: DLLNameStr.c_str());
1139 if (!DLLJD)
1140 return DLLJD.takeError();
1141 JD.addToLinkOrder(JD&: *DLLJD);
1142 return Error::success();
1143 }
1144
1145private:
1146 LLJIT &J;
1147};
1148
1149Expected<JITDylibSP> ExecutorNativePlatform::operator()(LLJIT &J) {
1150 auto ProcessSymbolsJD = J.getProcessSymbolsJITDylib();
1151 if (!ProcessSymbolsJD)
1152 return make_error<StringError>(
1153 Args: "Native platforms require a process symbols JITDylib",
1154 Args: inconvertibleErrorCode());
1155
1156 const Triple &TT = J.getTargetTriple();
1157 ObjectLinkingLayer *ObjLinkingLayer =
1158 dyn_cast<ObjectLinkingLayer>(Val: &J.getObjLinkingLayer());
1159
1160 if (!ObjLinkingLayer)
1161 return make_error<StringError>(
1162 Args: "ExecutorNativePlatform requires ObjectLinkingLayer",
1163 Args: inconvertibleErrorCode());
1164
1165 std::unique_ptr<MemoryBuffer> RuntimeArchiveBuffer;
1166 if (OrcRuntime.index() == 0) {
1167 auto A = errorOrToExpected(EO: MemoryBuffer::getFile(Filename: std::get<0>(v&: OrcRuntime)));
1168 if (!A)
1169 return A.takeError();
1170 RuntimeArchiveBuffer = std::move(*A);
1171 } else
1172 RuntimeArchiveBuffer = std::move(std::get<1>(v&: OrcRuntime));
1173
1174 auto &ES = J.getExecutionSession();
1175 auto &PlatformJD = ES.createBareJITDylib(Name: "<Platform>");
1176 PlatformJD.addToLinkOrder(JD&: *ProcessSymbolsJD);
1177
1178 J.setPlatformSupport(std::make_unique<ORCPlatformSupport>(args&: J));
1179
1180 switch (TT.getObjectFormat()) {
1181 case Triple::COFF: {
1182 const char *VCRuntimePath = nullptr;
1183 bool StaticVCRuntime = false;
1184 if (VCRuntime) {
1185 VCRuntimePath = VCRuntime->first.c_str();
1186 StaticVCRuntime = VCRuntime->second;
1187 }
1188 if (auto P = COFFPlatform::Create(
1189 ObjLinkingLayer&: *ObjLinkingLayer, PlatformJD, OrcRuntimeArchiveBuffer: std::move(RuntimeArchiveBuffer),
1190 LoadDynLibrary: LoadAndLinkDynLibrary(J), StaticVCRuntime, VCRuntimePath))
1191 J.getExecutionSession().setPlatform(std::move(*P));
1192 else
1193 return P.takeError();
1194 break;
1195 }
1196 case Triple::ELF: {
1197 auto G = StaticLibraryDefinitionGenerator::Create(
1198 L&: *ObjLinkingLayer, ArchiveBuffer: std::move(RuntimeArchiveBuffer));
1199 if (!G)
1200 return G.takeError();
1201
1202 if (auto P =
1203 ELFNixPlatform::Create(ObjLinkingLayer&: *ObjLinkingLayer, PlatformJD, OrcRuntime: std::move(*G)))
1204 J.getExecutionSession().setPlatform(std::move(*P));
1205 else
1206 return P.takeError();
1207 break;
1208 }
1209 case Triple::MachO: {
1210 auto G = StaticLibraryDefinitionGenerator::Create(
1211 L&: *ObjLinkingLayer, ArchiveBuffer: std::move(RuntimeArchiveBuffer));
1212 if (!G)
1213 return G.takeError();
1214
1215 if (auto P =
1216 MachOPlatform::Create(ObjLinkingLayer&: *ObjLinkingLayer, PlatformJD, OrcRuntime: std::move(*G)))
1217 ES.setPlatform(std::move(*P));
1218 else
1219 return P.takeError();
1220 break;
1221 }
1222 default:
1223 return make_error<StringError>(Args: "Unsupported object format in triple " +
1224 TT.str(),
1225 Args: inconvertibleErrorCode());
1226 }
1227
1228 return &PlatformJD;
1229}
1230
1231Expected<JITDylibSP> setUpGenericLLVMIRPlatform(LLJIT &J) {
1232 LLVM_DEBUG(
1233 { dbgs() << "Setting up GenericLLVMIRPlatform support for LLJIT\n"; });
1234 auto ProcessSymbolsJD = J.getProcessSymbolsJITDylib();
1235 if (!ProcessSymbolsJD)
1236 return make_error<StringError>(
1237 Args: "Native platforms require a process symbols JITDylib",
1238 Args: inconvertibleErrorCode());
1239
1240 auto &PlatformJD = J.getExecutionSession().createBareJITDylib(Name: "<Platform>");
1241 PlatformJD.addToLinkOrder(JD&: *ProcessSymbolsJD);
1242
1243 if (auto *OLL = dyn_cast<ObjectLinkingLayer>(Val: &J.getObjLinkingLayer())) {
1244
1245 bool UseEHFrames = true;
1246
1247 // Enable compact-unwind support if possible.
1248 if (J.getTargetTriple().isOSDarwin() ||
1249 J.getTargetTriple().isOSBinFormatMachO()) {
1250
1251 // Check if the bootstrap map says that we should force eh-frames:
1252 // Older libunwinds require this as they don't have a dynamic
1253 // registration API for compact-unwind.
1254 std::optional<bool> ForceEHFrames;
1255 if (auto Err = J.getExecutionSession().getBootstrapMapValue<bool, bool>(
1256 Key: "darwin-use-ehframes-only", Val&: ForceEHFrames))
1257 return Err;
1258 if (ForceEHFrames.has_value())
1259 UseEHFrames = *ForceEHFrames;
1260 else
1261 UseEHFrames = false;
1262
1263 // If UseEHFrames hasn't been set then we're good to use compact-unwind.
1264 if (!UseEHFrames) {
1265 if (auto UIRP =
1266 UnwindInfoRegistrationPlugin::Create(ES&: J.getExecutionSession())) {
1267 OLL->addPlugin(P: std::move(*UIRP));
1268 LLVM_DEBUG(dbgs() << "Enabled compact-unwind support.\n");
1269 } else
1270 return UIRP.takeError();
1271 }
1272 }
1273
1274 // Otherwise fall back to standard unwind registration.
1275 if (UseEHFrames) {
1276 auto &ES = J.getExecutionSession();
1277 if (auto EHFP = EHFrameRegistrationPlugin::Create(ES)) {
1278 OLL->addPlugin(P: std::move(*EHFP));
1279 LLVM_DEBUG(dbgs() << "Enabled eh-frame support.\n");
1280 } else
1281 return EHFP.takeError();
1282 }
1283 }
1284
1285 J.setPlatformSupport(
1286 std::make_unique<GenericLLVMIRPlatformSupport>(args&: J, args&: PlatformJD));
1287
1288 return &PlatformJD;
1289}
1290
1291Expected<JITDylibSP> setUpInactivePlatform(LLJIT &J) {
1292 LLVM_DEBUG(
1293 { dbgs() << "Explicitly deactivated platform support for LLJIT\n"; });
1294 J.setPlatformSupport(std::make_unique<InactivePlatformSupport>());
1295 return nullptr;
1296}
1297
1298Error LLLazyJITBuilderState::prepareForConstruction() {
1299 if (auto Err = LLJITBuilderState::prepareForConstruction())
1300 return Err;
1301 TT = JTMB->getTargetTriple();
1302 return Error::success();
1303}
1304
1305Error LLLazyJIT::addLazyIRModule(JITDylib &JD, ThreadSafeModule TSM) {
1306 assert(TSM && "Can not add null module");
1307
1308 if (auto Err = TSM.withModuleDo(
1309 F: [&](Module &M) -> Error { return applyDataLayout(M); }))
1310 return Err;
1311
1312 return CODLayer->add(JD, TSM: std::move(TSM));
1313}
1314
1315LLLazyJIT::LLLazyJIT(LLLazyJITBuilderState &S, Error &Err) : LLJIT(S, Err) {
1316
1317 // If LLJIT construction failed then bail out.
1318 if (Err)
1319 return;
1320
1321 ErrorAsOutParameter _(&Err);
1322
1323 /// Take/Create the lazy-compile callthrough manager.
1324 if (S.LCTMgr)
1325 LCTMgr = std::move(S.LCTMgr);
1326 else {
1327 if (auto LCTMgrOrErr = createLocalLazyCallThroughManager(
1328 T: S.TT, ES&: *ES, ErrorHandlerAddr: S.LazyCompileFailureAddr))
1329 LCTMgr = std::move(*LCTMgrOrErr);
1330 else {
1331 Err = LCTMgrOrErr.takeError();
1332 return;
1333 }
1334 }
1335
1336 // Take/Create the indirect stubs manager builder.
1337 auto ISMBuilder = std::move(S.ISMBuilder);
1338
1339 // If none was provided, try to build one.
1340 if (!ISMBuilder)
1341 ISMBuilder = createLocalIndirectStubsManagerBuilder(T: S.TT);
1342
1343 // No luck. Bail out.
1344 if (!ISMBuilder) {
1345 Err = make_error<StringError>(Args: "Could not construct "
1346 "IndirectStubsManagerBuilder for target " +
1347 S.TT.str(),
1348 Args: inconvertibleErrorCode());
1349 return;
1350 }
1351
1352 // Create the IP Layer.
1353 IPLayer = std::make_unique<IRPartitionLayer>(args&: *ES, args&: *InitHelperTransformLayer);
1354
1355 // Create the COD layer.
1356 CODLayer = std::make_unique<CompileOnDemandLayer>(args&: *ES, args&: *IPLayer, args&: *LCTMgr,
1357 args: std::move(ISMBuilder));
1358
1359 if (*S.SupportConcurrentCompilation)
1360 CODLayer->setCloneToNewContextOnEmit(true);
1361}
1362
1363// In-process LLJIT uses eh-frame section wrappers via EPC, so we need to force
1364// them to be linked in.
1365LLVM_ATTRIBUTE_USED void linkComponents() {
1366 errs() << (void *)&llvm_orc_registerEHFrameSectionAllocAction
1367 << (void *)&llvm_orc_deregisterEHFrameSectionAllocAction;
1368}
1369
1370} // End namespace orc.
1371} // End namespace llvm.
1372