1//===--------- LLJIT.cpp - An ORC-based JIT for compiling LLVM IR ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/Orc/LLJIT.h"
10
11#include "llvm/Analysis/TargetLibraryInfo.h"
12#include "llvm/Config/llvm-config.h" // for LLVM_ENABLE_THREADS
13#include "llvm/ExecutionEngine/Orc/COFFPlatform.h"
14#include "llvm/ExecutionEngine/Orc/DlfcnSPS.h"
15#include "llvm/ExecutionEngine/Orc/EHFrameRegistrationPlugin.h"
16#include "llvm/ExecutionEngine/Orc/ELFNixPlatform.h"
17#include "llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h"
18#include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
19#include "llvm/ExecutionEngine/Orc/MachOPlatform.h"
20#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
21#include "llvm/ExecutionEngine/Orc/ObjectTransformLayer.h"
22#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
23#include "llvm/ExecutionEngine/Orc/RecordProxy.h"
24#include "llvm/ExecutionEngine/Orc/SelfExecutorProcessControl.h"
25#include "llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h"
26#include "llvm/ExecutionEngine/Orc/UnwindInfoRegistrationPlugin.h"
27#include "llvm/ExecutionEngine/SectionMemoryManager.h"
28#include "llvm/IR/GlobalVariable.h"
29#include "llvm/IR/IRBuilder.h"
30#include "llvm/IR/Mangler.h"
31#include "llvm/IR/Module.h"
32#include "llvm/Support/DynamicLibrary.h"
33
34#define DEBUG_TYPE "orc"
35
36using namespace llvm;
37using namespace llvm::orc;
38
39namespace {
40
41/// Adds helper function decls and wrapper functions that call the helper with
42/// some additional prefix arguments.
43///
44/// E.g. For wrapper "foo" with type i8(i8, i64), helper "bar", and prefix
45/// args i32 4 and i16 12345, this function will add:
46///
47/// declare i8 @bar(i32, i16, i8, i64)
48///
49/// define i8 @foo(i8, i64) {
50/// entry:
51/// %2 = call i8 @bar(i32 4, i16 12345, i8 %0, i64 %1)
52/// ret i8 %2
53/// }
54///
55Function *addHelperAndWrapper(Module &M, StringRef WrapperName,
56 FunctionType *WrapperFnType,
57 GlobalValue::VisibilityTypes WrapperVisibility,
58 StringRef HelperName,
59 ArrayRef<Value *> HelperPrefixArgs) {
60 std::vector<Type *> HelperArgTypes;
61 for (auto *Arg : HelperPrefixArgs)
62 HelperArgTypes.push_back(x: Arg->getType());
63 llvm::append_range(C&: HelperArgTypes, R: WrapperFnType->params());
64 auto *HelperFnType =
65 FunctionType::get(Result: WrapperFnType->getReturnType(), Params: HelperArgTypes, isVarArg: false);
66 auto *HelperFn = Function::Create(Ty: HelperFnType, Linkage: GlobalValue::ExternalLinkage,
67 N: HelperName, M);
68
69 auto *WrapperFn = Function::Create(
70 Ty: WrapperFnType, Linkage: GlobalValue::ExternalLinkage, N: WrapperName, M);
71 WrapperFn->setVisibility(WrapperVisibility);
72
73 auto *EntryBlock = BasicBlock::Create(Context&: M.getContext(), Name: "entry", Parent: WrapperFn);
74 IRBuilder<> IB(EntryBlock);
75
76 std::vector<Value *> HelperArgs;
77 llvm::append_range(C&: HelperArgs, R&: HelperPrefixArgs);
78 for (auto &Arg : WrapperFn->args())
79 HelperArgs.push_back(x: &Arg);
80 auto *HelperResult = IB.CreateCall(Callee: HelperFn, Args: HelperArgs);
81 if (HelperFn->getReturnType()->isVoidTy())
82 IB.CreateRetVoid();
83 else
84 IB.CreateRet(V: HelperResult);
85
86 return WrapperFn;
87}
88
89class GenericLLVMIRPlatformSupport;
90
91/// orc::Platform component of Generic LLVM IR Platform support.
92/// Just forwards calls to the GenericLLVMIRPlatformSupport class below.
93class GenericLLVMIRPlatform : public Platform {
94public:
95 GenericLLVMIRPlatform(GenericLLVMIRPlatformSupport &S) : S(S) {}
96 Error setupJITDylib(JITDylib &JD) override;
97 Error teardownJITDylib(JITDylib &JD) override;
98 Error notifyAdding(ResourceTracker &RT,
99 const MaterializationUnit &MU) override;
100 Error notifyRemoving(ResourceTracker &RT) override {
101 // Noop -- Nothing to do (yet).
102 return Error::success();
103 }
104
105private:
106 GenericLLVMIRPlatformSupport &S;
107};
108
109/// This transform parses llvm.global_ctors to produce a single initialization
110/// function for the module, records the function, then deletes
111/// llvm.global_ctors.
112class GlobalCtorDtorScraper {
113public:
114 GlobalCtorDtorScraper(GenericLLVMIRPlatformSupport &PS,
115 StringRef InitFunctionPrefix,
116 StringRef DeInitFunctionPrefix)
117 : PS(PS), InitFunctionPrefix(InitFunctionPrefix),
118 DeInitFunctionPrefix(DeInitFunctionPrefix) {}
119 Expected<ThreadSafeModule> operator()(ThreadSafeModule TSM,
120 MaterializationResponsibility &R);
121
122private:
123 GenericLLVMIRPlatformSupport &PS;
124 StringRef InitFunctionPrefix;
125 StringRef DeInitFunctionPrefix;
126};
127
128/// Generic IR Platform Support
129///
130/// Scrapes llvm.global_ctors and llvm.global_dtors and replaces them with
131/// specially named 'init' and 'deinit'. Injects definitions / interposes for
132/// some runtime API, including __cxa_atexit, dlopen, and dlclose.
133class GenericLLVMIRPlatformSupport : public LLJIT::PlatformSupport {
134public:
135 GenericLLVMIRPlatformSupport(LLJIT &J, JITDylib &PlatformJD)
136 : J(J), InitFunctionPrefix(J.mangle(UnmangledName: "__orc_init_func.")),
137 DeInitFunctionPrefix(J.mangle(UnmangledName: "__orc_deinit_func.")) {
138
139 getExecutionSession().setPlatform(
140 std::make_unique<GenericLLVMIRPlatform>(args&: *this));
141
142 setInitTransform(J, T: GlobalCtorDtorScraper(*this, InitFunctionPrefix,
143 DeInitFunctionPrefix));
144
145 SymbolMap StdInterposes;
146
147 StdInterposes[J.mangleAndIntern(UnmangledName: "__lljit.platform_support_instance")] = {
148 ExecutorAddr::fromPtr(Ptr: this), JITSymbolFlags::Exported};
149 StdInterposes[J.mangleAndIntern(UnmangledName: "__lljit.cxa_atexit_helper")] = {
150 ExecutorAddr::fromPtr(Ptr: registerCxaAtExitHelper), JITSymbolFlags()};
151
152 cantFail(Err: PlatformJD.define(MU: absoluteSymbols(Symbols: std::move(StdInterposes))));
153 cantFail(Err: setupJITDylib(PlatformJD));
154 cantFail(Err: J.addIRModule(JD&: PlatformJD, TSM: createPlatformRuntimeModule()));
155 }
156
157 ExecutionSession &getExecutionSession() { return J.getExecutionSession(); }
158
159 /// Adds a module that defines the __dso_handle global.
160 Error setupJITDylib(JITDylib &JD) {
161
162 // Add per-jitdylib standard interposes.
163 SymbolMap PerJDInterposes;
164 PerJDInterposes[J.mangleAndIntern(UnmangledName: "__lljit.run_atexits_helper")] = {
165 ExecutorAddr::fromPtr(Ptr: runAtExitsHelper), JITSymbolFlags()};
166 PerJDInterposes[J.mangleAndIntern(UnmangledName: "__lljit.atexit_helper")] = {
167 ExecutorAddr::fromPtr(Ptr: registerAtExitHelper), JITSymbolFlags()};
168 cantFail(Err: JD.define(MU: absoluteSymbols(Symbols: std::move(PerJDInterposes))));
169
170 auto Ctx = std::make_unique<LLVMContext>();
171 auto M = std::make_unique<Module>(args: "__standard_lib", args&: *Ctx);
172 M->setDataLayout(J.getDataLayout());
173
174 auto *Int64Ty = Type::getInt64Ty(C&: *Ctx);
175 auto *DSOHandle = new GlobalVariable(
176 *M, Int64Ty, true, GlobalValue::ExternalLinkage,
177 ConstantInt::get(Ty: Int64Ty, V: reinterpret_cast<uintptr_t>(&JD)),
178 "__dso_handle");
179 DSOHandle->setVisibility(GlobalValue::DefaultVisibility);
180 DSOHandle->setInitializer(
181 ConstantInt::get(Ty: Int64Ty, V: ExecutorAddr::fromPtr(Ptr: &JD).getValue()));
182
183 auto *GenericIRPlatformSupportTy =
184 StructType::create(Context&: *Ctx, Name: "lljit.GenericLLJITIRPlatformSupport");
185
186 auto *PlatformInstanceDecl = new GlobalVariable(
187 *M, GenericIRPlatformSupportTy, true, GlobalValue::ExternalLinkage,
188 nullptr, "__lljit.platform_support_instance");
189
190 auto *VoidTy = Type::getVoidTy(C&: *Ctx);
191 addHelperAndWrapper(
192 M&: *M, WrapperName: "__lljit_run_atexits", WrapperFnType: FunctionType::get(Result: VoidTy, Params: {}, isVarArg: false),
193 WrapperVisibility: GlobalValue::HiddenVisibility, HelperName: "__lljit.run_atexits_helper",
194 HelperPrefixArgs: {PlatformInstanceDecl, DSOHandle});
195
196 auto *IntTy = Type::getIntNTy(C&: *Ctx, N: sizeof(int) * CHAR_BIT);
197 auto *AtExitCallbackPtrTy = PointerType::getUnqual(C&: *Ctx);
198 auto *AtExit = addHelperAndWrapper(
199 M&: *M, WrapperName: "atexit", WrapperFnType: FunctionType::get(Result: IntTy, Params: {AtExitCallbackPtrTy}, isVarArg: false),
200 WrapperVisibility: GlobalValue::HiddenVisibility, HelperName: "__lljit.atexit_helper",
201 HelperPrefixArgs: {PlatformInstanceDecl, DSOHandle});
202 Attribute::AttrKind AtExitExtAttr =
203 TargetLibraryInfo::getExtAttrForI32Return(T: J.getTargetTriple());
204 if (AtExitExtAttr != Attribute::None)
205 AtExit->addRetAttr(Kind: AtExitExtAttr);
206
207 return J.addIRModule(JD, TSM: ThreadSafeModule(std::move(M), std::move(Ctx)));
208 }
209
210 Error notifyAdding(ResourceTracker &RT, const MaterializationUnit &MU) {
211 auto &JD = RT.getJITDylib();
212 if (auto &InitSym = MU.getInitializerSymbol())
213 InitSymbols[&JD].add(Name: InitSym, Flags: SymbolLookupFlags::WeaklyReferencedSymbol);
214 else {
215 // If there's no identified init symbol attached, but there is a symbol
216 // with the GenericIRPlatform::InitFunctionPrefix, then treat that as
217 // an init function. Add the symbol to both the InitSymbols map (which
218 // will trigger a lookup to materialize the module) and the InitFunctions
219 // map (which holds the names of the symbols to execute).
220 for (auto &KV : MU.getSymbols())
221 if ((*KV.first).starts_with(Prefix: InitFunctionPrefix)) {
222 InitSymbols[&JD].add(Name: KV.first,
223 Flags: SymbolLookupFlags::WeaklyReferencedSymbol);
224 InitFunctions[&JD].add(Name: KV.first);
225 } else if ((*KV.first).starts_with(Prefix: DeInitFunctionPrefix)) {
226 DeInitFunctions[&JD].add(Name: KV.first);
227 }
228 }
229 return Error::success();
230 }
231
232 Error initialize(JITDylib &JD) override {
233 LLVM_DEBUG({
234 dbgs() << "GenericLLVMIRPlatformSupport getting initializers to run\n";
235 });
236 if (auto Initializers = getInitializers(JD)) {
237 LLVM_DEBUG(
238 { dbgs() << "GenericLLVMIRPlatformSupport running initializers\n"; });
239 for (auto InitFnAddr : *Initializers) {
240 LLVM_DEBUG({
241 dbgs() << " Running init " << formatv("{0:x16}", InitFnAddr)
242 << "...\n";
243 });
244 auto *InitFn = InitFnAddr.toPtr<void (*)()>();
245 InitFn();
246 }
247 } else
248 return Initializers.takeError();
249 return Error::success();
250 }
251
252 Error deinitialize(JITDylib &JD) override {
253 LLVM_DEBUG({
254 dbgs() << "GenericLLVMIRPlatformSupport getting deinitializers to run\n";
255 });
256 if (auto Deinitializers = getDeinitializers(JD)) {
257 LLVM_DEBUG({
258 dbgs() << "GenericLLVMIRPlatformSupport running deinitializers\n";
259 });
260 for (auto DeinitFnAddr : *Deinitializers) {
261 LLVM_DEBUG({
262 dbgs() << " Running deinit " << formatv("{0:x16}", DeinitFnAddr)
263 << "...\n";
264 });
265 auto *DeinitFn = DeinitFnAddr.toPtr<void (*)()>();
266 DeinitFn();
267 }
268 } else
269 return Deinitializers.takeError();
270
271 return Error::success();
272 }
273
274 void registerInitFunc(JITDylib &JD, SymbolStringPtr InitName) {
275 getExecutionSession().runSessionLocked(
276 F: [&]() { InitFunctions[&JD].add(Name: InitName); });
277 }
278
279 void registerDeInitFunc(JITDylib &JD, SymbolStringPtr DeInitName) {
280 getExecutionSession().runSessionLocked(
281 F: [&]() { DeInitFunctions[&JD].add(Name: DeInitName); });
282 }
283
284private:
285 Expected<std::vector<ExecutorAddr>> getInitializers(JITDylib &JD) {
286 if (auto Err = issueInitLookups(JD))
287 return std::move(Err);
288
289 DenseMap<JITDylib *, SymbolLookupSet> LookupSymbols;
290 std::vector<JITDylibSP> DFSLinkOrder;
291
292 if (auto Err = getExecutionSession().runSessionLocked(F: [&]() -> Error {
293 if (auto DFSLinkOrderOrErr = JD.getDFSLinkOrder())
294 DFSLinkOrder = std::move(*DFSLinkOrderOrErr);
295 else
296 return DFSLinkOrderOrErr.takeError();
297
298 for (auto &NextJD : DFSLinkOrder) {
299 auto IFItr = InitFunctions.find(Val: NextJD.get());
300 if (IFItr != InitFunctions.end()) {
301 LookupSymbols[NextJD.get()] = std::move(IFItr->second);
302 InitFunctions.erase(I: IFItr);
303 }
304 }
305 return Error::success();
306 }))
307 return std::move(Err);
308
309 LLVM_DEBUG({
310 dbgs() << "JITDylib init order is [ ";
311 for (auto &JD : llvm::reverse(DFSLinkOrder))
312 dbgs() << "\"" << JD->getName() << "\" ";
313 dbgs() << "]\n";
314 dbgs() << "Looking up init functions:\n";
315 for (auto &KV : LookupSymbols)
316 dbgs() << " \"" << KV.first->getName() << "\": " << KV.second << "\n";
317 });
318
319 auto &ES = getExecutionSession();
320 auto LookupResult = Platform::lookupInitSymbols(ES, InitSyms: LookupSymbols);
321
322 if (!LookupResult)
323 return LookupResult.takeError();
324
325 std::vector<ExecutorAddr> Initializers;
326 while (!DFSLinkOrder.empty()) {
327 auto &NextJD = *DFSLinkOrder.back();
328 DFSLinkOrder.pop_back();
329 auto InitsItr = LookupResult->find(Val: &NextJD);
330 if (InitsItr == LookupResult->end())
331 continue;
332 for (auto &KV : InitsItr->second)
333 Initializers.push_back(x: KV.second.getAddress());
334 }
335
336 return Initializers;
337 }
338
339 Expected<std::vector<ExecutorAddr>> getDeinitializers(JITDylib &JD) {
340 auto &ES = getExecutionSession();
341
342 auto LLJITRunAtExits = J.mangleAndIntern(UnmangledName: "__lljit_run_atexits");
343
344 DenseMap<JITDylib *, SymbolLookupSet> LookupSymbols;
345 std::vector<JITDylibSP> DFSLinkOrder;
346
347 if (auto Err = ES.runSessionLocked(F: [&]() -> Error {
348 if (auto DFSLinkOrderOrErr = JD.getDFSLinkOrder())
349 DFSLinkOrder = std::move(*DFSLinkOrderOrErr);
350 else
351 return DFSLinkOrderOrErr.takeError();
352
353 for (auto &NextJD : DFSLinkOrder) {
354 auto &JDLookupSymbols = LookupSymbols[NextJD.get()];
355 auto DIFItr = DeInitFunctions.find(Val: NextJD.get());
356 if (DIFItr != DeInitFunctions.end()) {
357 LookupSymbols[NextJD.get()] = std::move(DIFItr->second);
358 DeInitFunctions.erase(I: DIFItr);
359 }
360 JDLookupSymbols.add(Name: LLJITRunAtExits,
361 Flags: SymbolLookupFlags::WeaklyReferencedSymbol);
362 }
363 return Error::success();
364 }))
365 return std::move(Err);
366
367 LLVM_DEBUG({
368 dbgs() << "JITDylib deinit order is [ ";
369 for (auto &JD : DFSLinkOrder)
370 dbgs() << "\"" << JD->getName() << "\" ";
371 dbgs() << "]\n";
372 dbgs() << "Looking up deinit functions:\n";
373 for (auto &KV : LookupSymbols)
374 dbgs() << " \"" << KV.first->getName() << "\": " << KV.second << "\n";
375 });
376
377 auto LookupResult = Platform::lookupInitSymbols(ES, InitSyms: LookupSymbols);
378
379 if (!LookupResult)
380 return LookupResult.takeError();
381
382 std::vector<ExecutorAddr> DeInitializers;
383 for (auto &NextJD : DFSLinkOrder) {
384 auto DeInitsItr = LookupResult->find(Val: NextJD.get());
385 assert(DeInitsItr != LookupResult->end() &&
386 "Every JD should have at least __lljit_run_atexits");
387
388 auto RunAtExitsItr = DeInitsItr->second.find(Val: LLJITRunAtExits);
389 if (RunAtExitsItr != DeInitsItr->second.end())
390 DeInitializers.push_back(x: RunAtExitsItr->second.getAddress());
391
392 for (auto &KV : DeInitsItr->second)
393 if (KV.first != LLJITRunAtExits)
394 DeInitializers.push_back(x: KV.second.getAddress());
395 }
396
397 return DeInitializers;
398 }
399
400 /// Issue lookups for all init symbols required to initialize JD (and any
401 /// JITDylibs that it depends on).
402 Error issueInitLookups(JITDylib &JD) {
403 DenseMap<JITDylib *, SymbolLookupSet> RequiredInitSymbols;
404 std::vector<JITDylibSP> DFSLinkOrder;
405
406 if (auto Err = getExecutionSession().runSessionLocked(F: [&]() -> Error {
407 if (auto DFSLinkOrderOrErr = JD.getDFSLinkOrder())
408 DFSLinkOrder = std::move(*DFSLinkOrderOrErr);
409 else
410 return DFSLinkOrderOrErr.takeError();
411
412 for (auto &NextJD : DFSLinkOrder) {
413 auto ISItr = InitSymbols.find(Val: NextJD.get());
414 if (ISItr != InitSymbols.end()) {
415 RequiredInitSymbols[NextJD.get()] = std::move(ISItr->second);
416 InitSymbols.erase(I: ISItr);
417 }
418 }
419 return Error::success();
420 }))
421 return Err;
422
423 return Platform::lookupInitSymbols(ES&: getExecutionSession(),
424 InitSyms: RequiredInitSymbols)
425 .takeError();
426 }
427
428 static void registerCxaAtExitHelper(void *Self, void (*F)(void *), void *Ctx,
429 void *DSOHandle) {
430 LLVM_DEBUG({
431 dbgs() << "Registering cxa atexit function " << (void *)F << " for JD "
432 << (*static_cast<JITDylib **>(DSOHandle))->getName() << "\n";
433 });
434 static_cast<GenericLLVMIRPlatformSupport *>(Self)->AtExitMgr.registerAtExit(
435 F, Ctx, DSOHandle);
436 }
437
438 static void registerAtExitHelper(void *Self, void *DSOHandle, void (*F)()) {
439 LLVM_DEBUG({
440 dbgs() << "Registering atexit function " << (void *)F << " for JD "
441 << (*static_cast<JITDylib **>(DSOHandle))->getName() << "\n";
442 });
443 static_cast<GenericLLVMIRPlatformSupport *>(Self)->AtExitMgr.registerAtExit(
444 F: reinterpret_cast<void (*)(void *)>(F), Ctx: nullptr, DSOHandle);
445 }
446
447 static void runAtExitsHelper(void *Self, void *DSOHandle) {
448 LLVM_DEBUG({
449 dbgs() << "Running atexit functions for JD "
450 << (*static_cast<JITDylib **>(DSOHandle))->getName() << "\n";
451 });
452 static_cast<GenericLLVMIRPlatformSupport *>(Self)->AtExitMgr.runAtExits(
453 DSOHandle);
454 }
455
456 // Constructs an LLVM IR module containing platform runtime globals,
457 // functions, and interposes.
458 ThreadSafeModule createPlatformRuntimeModule() {
459 auto Ctx = std::make_unique<LLVMContext>();
460 auto M = std::make_unique<Module>(args: "__standard_lib", args&: *Ctx);
461 M->setDataLayout(J.getDataLayout());
462
463 auto *GenericIRPlatformSupportTy =
464 StructType::create(Context&: *Ctx, Name: "lljit.GenericLLJITIRPlatformSupport");
465
466 auto *PlatformInstanceDecl = new GlobalVariable(
467 *M, GenericIRPlatformSupportTy, true, GlobalValue::ExternalLinkage,
468 nullptr, "__lljit.platform_support_instance");
469
470 auto *IntTy = Type::getIntNTy(C&: *Ctx, N: sizeof(int) * CHAR_BIT);
471 auto *BytePtrTy = PointerType::getUnqual(C&: *Ctx);
472 auto *CxaAtExitCallbackPtrTy = PointerType::getUnqual(C&: *Ctx);
473
474 auto *CxaAtExit = addHelperAndWrapper(
475 M&: *M, WrapperName: "__cxa_atexit",
476 WrapperFnType: FunctionType::get(Result: IntTy, Params: {CxaAtExitCallbackPtrTy, BytePtrTy, BytePtrTy},
477 isVarArg: false),
478 WrapperVisibility: GlobalValue::DefaultVisibility, HelperName: "__lljit.cxa_atexit_helper",
479 HelperPrefixArgs: {PlatformInstanceDecl});
480 Attribute::AttrKind CxaAtExitExtAttr =
481 TargetLibraryInfo::getExtAttrForI32Return(T: J.getTargetTriple());
482 if (CxaAtExitExtAttr != Attribute::None)
483 CxaAtExit->addRetAttr(Kind: CxaAtExitExtAttr);
484
485 return ThreadSafeModule(std::move(M), std::move(Ctx));
486 }
487
488 LLJIT &J;
489 std::string InitFunctionPrefix;
490 std::string DeInitFunctionPrefix;
491 DenseMap<JITDylib *, SymbolLookupSet> InitSymbols;
492 DenseMap<JITDylib *, SymbolLookupSet> InitFunctions;
493 DenseMap<JITDylib *, SymbolLookupSet> DeInitFunctions;
494 ItaniumCXAAtExitSupport AtExitMgr;
495};
496
497Error GenericLLVMIRPlatform::setupJITDylib(JITDylib &JD) {
498 return S.setupJITDylib(JD);
499}
500
501Error GenericLLVMIRPlatform::teardownJITDylib(JITDylib &JD) {
502 return Error::success();
503}
504
505Error GenericLLVMIRPlatform::notifyAdding(ResourceTracker &RT,
506 const MaterializationUnit &MU) {
507 return S.notifyAdding(RT, MU);
508}
509
510Expected<ThreadSafeModule>
511GlobalCtorDtorScraper::operator()(ThreadSafeModule TSM,
512 MaterializationResponsibility &R) {
513 auto Err = TSM.withModuleDo(F: [&](Module &M) -> Error {
514 auto &Ctx = M.getContext();
515 auto *GlobalCtors = M.getNamedGlobal(Name: "llvm.global_ctors");
516 auto *GlobalDtors = M.getNamedGlobal(Name: "llvm.global_dtors");
517
518 auto RegisterCOrDtors = [&](GlobalVariable *GlobalCOrDtors,
519 bool isCtor) -> Error {
520 // If there's no llvm.global_c/dtor or it's just a decl then skip.
521 if (!GlobalCOrDtors || GlobalCOrDtors->isDeclaration())
522 return Error::success();
523 std::string InitOrDeInitFunctionName;
524 if (isCtor)
525 raw_string_ostream(InitOrDeInitFunctionName)
526 << InitFunctionPrefix << M.getModuleIdentifier();
527 else
528 raw_string_ostream(InitOrDeInitFunctionName)
529 << DeInitFunctionPrefix << M.getModuleIdentifier();
530
531 MangleAndInterner Mangle(PS.getExecutionSession(), M.getDataLayout());
532 auto InternedInitOrDeInitName = Mangle(InitOrDeInitFunctionName);
533 if (auto Err = R.defineMaterializing(
534 SymbolFlags: {{InternedInitOrDeInitName, JITSymbolFlags::Callable}}))
535 return Err;
536
537 auto *InitOrDeInitFunc = Function::Create(
538 Ty: FunctionType::get(Result: Type::getVoidTy(C&: Ctx), Params: {}, isVarArg: false),
539 Linkage: GlobalValue::ExternalLinkage, N: InitOrDeInitFunctionName, M: &M);
540 InitOrDeInitFunc->setVisibility(GlobalValue::HiddenVisibility);
541 std::vector<std::pair<Function *, unsigned>> InitsOrDeInits;
542 auto COrDtors = isCtor ? getConstructors(M) : getDestructors(M);
543
544 for (auto E : COrDtors)
545 InitsOrDeInits.push_back(x: std::make_pair(x&: E.Func, y&: E.Priority));
546 llvm::stable_sort(Range&: InitsOrDeInits, C: llvm::less_second());
547
548 auto *InitOrDeInitFuncEntryBlock =
549 BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: InitOrDeInitFunc);
550 IRBuilder<> IB(InitOrDeInitFuncEntryBlock);
551 for (auto &KV : InitsOrDeInits)
552 IB.CreateCall(Callee: KV.first);
553 IB.CreateRetVoid();
554
555 if (isCtor)
556 PS.registerInitFunc(JD&: R.getTargetJITDylib(), InitName: InternedInitOrDeInitName);
557 else
558 PS.registerDeInitFunc(JD&: R.getTargetJITDylib(), DeInitName: InternedInitOrDeInitName);
559
560 GlobalCOrDtors->eraseFromParent();
561 return Error::success();
562 };
563
564 if (auto Err = RegisterCOrDtors(GlobalCtors, true))
565 return Err;
566 if (auto Err = RegisterCOrDtors(GlobalDtors, false))
567 return Err;
568
569 return Error::success();
570 });
571
572 if (Err)
573 return std::move(Err);
574
575 return std::move(TSM);
576}
577
578/// Inactive Platform Support
579///
580/// Explicitly disables platform support. JITDylibs are not scanned for special
581/// init/deinit symbols. No runtime API interposes are injected.
582class InactivePlatformSupport : public LLJIT::PlatformSupport {
583public:
584 InactivePlatformSupport() = default;
585
586 Error initialize(JITDylib &JD) override {
587 LLVM_DEBUG(dbgs() << "InactivePlatformSupport: no initializers running for "
588 << JD.getName() << "\n");
589 return Error::success();
590 }
591
592 Error deinitialize(JITDylib &JD) override {
593 LLVM_DEBUG(
594 dbgs() << "InactivePlatformSupport: no deinitializers running for "
595 << JD.getName() << "\n");
596 return Error::success();
597 }
598};
599
600} // end anonymous namespace
601
602namespace llvm {
603namespace orc {
604
605Error ORCPlatformSupport::initialize(orc::JITDylib &JD) {
606 enum dlopen_mode : int32_t {
607 ORC_RT_RTLD_LAZY = 0x1,
608 ORC_RT_RTLD_NOW = 0x2,
609 ORC_RT_RTLD_LOCAL = 0x4,
610 ORC_RT_RTLD_GLOBAL = 0x8
611 };
612
613 auto &ES = J.getExecutionSession();
614 auto MainSearchOrder = J.getMainJITDylib().withLinkOrderDo(
615 F: [](const JITDylibSearchOrder &SO) { return SO; });
616
617 if (InitializedDylib.contains(Ptr: &JD)) {
618 // Already initialized: re-run initializers via dlupdate.
619 DlfcnUpdateProxy Update;
620 if (auto Err =
621 lookupAndApply(K: LookupKind::Static, SearchOrder: MainSearchOrder,
622 PrepareFns: {recordProxy<sps::DlfcnUpdateProxySpec>(P: &Update)}))
623 return Err;
624 auto Result = Update(ES, DSOHandles[&JD]);
625 if (!Result)
626 return Result.takeError();
627 if (*Result)
628 return make_error<StringError>(Args: "dlupdate failed",
629 Args: inconvertibleErrorCode());
630 return Error::success();
631 }
632
633 InitializedDylib.insert(Ptr: &JD);
634 DlfcnOpenProxy Open;
635 if (auto Err = lookupAndApply(K: LookupKind::Static, SearchOrder: MainSearchOrder,
636 PrepareFns: {recordProxy<sps::DlfcnOpenProxySpec>(P: &Open)}))
637 return Err;
638 auto H = Open(ES, JD.getName(), int32_t(ORC_RT_RTLD_LAZY));
639 if (!H)
640 return H.takeError();
641 DSOHandles[&JD] = *H;
642 return Error::success();
643}
644
645Error ORCPlatformSupport::deinitialize(orc::JITDylib &JD) {
646 auto &ES = J.getExecutionSession();
647 auto MainSearchOrder = J.getMainJITDylib().withLinkOrderDo(
648 F: [](const JITDylibSearchOrder &SO) { return SO; });
649
650 DlfcnCloseProxy Close;
651 if (auto Err =
652 lookupAndApply(K: LookupKind::Static, SearchOrder: MainSearchOrder,
653 PrepareFns: {recordProxy<sps::DlfcnCloseProxySpec>(P: &Close)}))
654 return Err;
655 auto Result = Close(ES, DSOHandles[&JD]);
656 if (!Result)
657 return Result.takeError();
658 if (*Result)
659 return make_error<StringError>(Args: "dlclose failed", Args: inconvertibleErrorCode());
660 DSOHandles.erase(Val: &JD);
661 InitializedDylib.erase(Ptr: &JD);
662 return Error::success();
663}
664
665void LLJIT::PlatformSupport::setInitTransform(
666 LLJIT &J, IRTransformLayer::TransformFunction T) {
667 J.InitHelperTransformLayer->setTransform(std::move(T));
668}
669
670LLJIT::PlatformSupport::~PlatformSupport() = default;
671
672Error LLJITBuilderState::prepareForConstruction() {
673
674 LLVM_DEBUG(dbgs() << "Preparing to create LLJIT instance...\n");
675
676 if (!JTMB) {
677 LLVM_DEBUG({
678 dbgs() << " No explicitly set JITTargetMachineBuilder. "
679 "Detecting host...\n";
680 });
681 if (auto JTMBOrErr = JITTargetMachineBuilder::detectHost())
682 JTMB = std::move(*JTMBOrErr);
683 else
684 return JTMBOrErr.takeError();
685 }
686
687 if ((ES || EPC) && NumCompileThreads)
688 return make_error<StringError>(
689 Args: "NumCompileThreads cannot be used with a custom ExecutionSession or "
690 "ExecutorProcessControl",
691 Args: inconvertibleErrorCode());
692
693#if !LLVM_ENABLE_THREADS
694 if (NumCompileThreads)
695 return make_error<StringError>(
696 "LLJIT num-compile-threads is " + Twine(NumCompileThreads) +
697 " but LLVM was compiled with LLVM_ENABLE_THREADS=Off",
698 inconvertibleErrorCode());
699#endif // !LLVM_ENABLE_THREADS
700
701 // Only used in debug builds.
702 [[maybe_unused]] bool ConcurrentCompilationSettingDefaulted =
703 !SupportConcurrentCompilation;
704
705 if (!SupportConcurrentCompilation) {
706#if LLVM_ENABLE_THREADS
707 SupportConcurrentCompilation = NumCompileThreads || ES || EPC;
708#else
709 SupportConcurrentCompilation = false;
710#endif // LLVM_ENABLE_THREADS
711 } else {
712#if !LLVM_ENABLE_THREADS
713 if (*SupportConcurrentCompilation)
714 return make_error<StringError>(
715 "LLJIT concurrent compilation support requested, but LLVM was built "
716 "with LLVM_ENABLE_THREADS=Off",
717 inconvertibleErrorCode());
718#endif // !LLVM_ENABLE_THREADS
719 }
720
721 LLVM_DEBUG({
722 dbgs() << " JITTargetMachineBuilder is "
723 << JITTargetMachineBuilderPrinter(*JTMB, " ")
724 << " Pre-constructed ExecutionSession: " << (ES ? "Yes" : "No")
725 << "\n"
726 << " DataLayout: ";
727 if (DL)
728 dbgs() << DL->getStringRepresentation() << "\n";
729 else
730 dbgs() << "None (will be created by JITTargetMachineBuilder)\n";
731
732 dbgs() << " Custom object-linking-layer creator: "
733 << (CreateObjectLinkingLayer ? "Yes" : "No") << "\n"
734 << " Custom compile-function creator: "
735 << (CreateCompileFunction ? "Yes" : "No") << "\n"
736 << " Custom platform-setup function: "
737 << (SetUpPlatform ? "Yes" : "No") << "\n"
738 << " Support concurrent compilation: "
739 << (*SupportConcurrentCompilation ? "Yes" : "No");
740 if (ConcurrentCompilationSettingDefaulted)
741 dbgs() << " (defaulted based on ES / EPC / NumCompileThreads)\n";
742 else
743 dbgs() << "\n";
744 dbgs() << " Number of compile threads: " << NumCompileThreads << "\n";
745 });
746
747 // Create DL if not specified.
748 if (!DL) {
749 if (auto DLOrErr = JTMB->getDefaultDataLayoutForTarget())
750 DL = std::move(*DLOrErr);
751 else
752 return DLOrErr.takeError();
753 }
754
755 // If neither ES nor EPC has been set then create an EPC instance.
756 if (!ES && !EPC) {
757 LLVM_DEBUG({
758 dbgs() << "ExecutorProcessControl not specified, "
759 "Creating SelfExecutorProcessControl instance\n";
760 });
761
762 std::unique_ptr<TaskDispatcher> D = nullptr;
763#if LLVM_ENABLE_THREADS
764 if (*SupportConcurrentCompilation) {
765 std::optional<size_t> NumThreads = std ::nullopt;
766 if (NumCompileThreads)
767 NumThreads = NumCompileThreads;
768 D = std::make_unique<DynamicThreadPoolTaskDispatcher>(args&: NumThreads);
769 } else
770 D = std::make_unique<InPlaceTaskDispatcher>();
771#endif // LLVM_ENABLE_THREADS
772 if (auto EPCOrErr =
773 SelfExecutorProcessControl::Create(SSP: nullptr, D: std::move(D)))
774 EPC = std::move(*EPCOrErr);
775 else
776 return EPCOrErr.takeError();
777 } else if (EPC) {
778 LLVM_DEBUG({
779 dbgs() << "Using explicitly specified ExecutorProcessControl instance "
780 << EPC.get() << "\n";
781 });
782 } else {
783 LLVM_DEBUG({
784 dbgs() << "Using explicitly specified ExecutionSession instance "
785 << ES.get() << "\n";
786 });
787 }
788
789 // If the client didn't configure any linker options then auto-configure the
790 // JIT linker.
791 if (!CreateObjectLinkingLayer) {
792 auto &TT = JTMB->getTargetTriple();
793 bool UseJITLink = false;
794 switch (TT.getArch()) {
795 case Triple::riscv64:
796 case Triple::loongarch64:
797 UseJITLink = true;
798 break;
799 case Triple::aarch64:
800 UseJITLink = !TT.isOSBinFormatCOFF();
801 break;
802 case Triple::arm:
803 case Triple::armeb:
804 case Triple::thumb:
805 case Triple::thumbeb:
806 UseJITLink = TT.isOSBinFormatELF();
807 break;
808 case Triple::x86_64:
809 UseJITLink = !TT.isOSBinFormatCOFF();
810 break;
811 case Triple::ppc64:
812 UseJITLink = TT.isPPC64ELFv2ABI();
813 break;
814 case Triple::ppc64le:
815 UseJITLink = TT.isOSBinFormatELF();
816 break;
817 case Triple::systemz:
818 UseJITLink = TT.isOSBinFormatELF();
819 break;
820 default:
821 break;
822 }
823 if (UseJITLink) {
824 if (!JTMB->getCodeModel())
825 JTMB->setCodeModel(CodeModel::Small);
826 JTMB->setRelocationModel(Reloc::PIC_);
827 CreateObjectLinkingLayer = [](ExecutionSession &ES,
828 jitlink::JITLinkMemoryManager &MemMgr)
829 -> Expected<std::unique_ptr<ObjectLayer>> {
830 return std::make_unique<ObjectLinkingLayer>(args&: ES, args&: MemMgr);
831 };
832 }
833 }
834
835 // If we need a process JITDylib but no setup function has been given then
836 // create a default one.
837 if (!SetupProcessSymbolsJITDylib && LinkProcessSymbolsByDefault) {
838 LLVM_DEBUG(dbgs() << "Creating default Process JD setup function\n");
839 SetupProcessSymbolsJITDylib = [](LLJIT &J) -> Expected<JITDylibSP> {
840 auto &JD =
841 J.getExecutionSession().createBareJITDylib(Name: "<Process Symbols>");
842 auto G = EPCDynamicLibrarySearchGenerator::GetForTargetProcess(
843 ES&: J.getExecutionSession(), DylibMgr&: J.getDylibMgr());
844 if (!G)
845 return G.takeError();
846 JD.addGenerator(DefGenerator: std::move(*G));
847 return &JD;
848 };
849 }
850
851 return Error::success();
852}
853
854LLJIT::~LLJIT() {
855 if (auto Err = ES->endSession())
856 ES->reportError(Err: std::move(Err));
857}
858
859JITDylibSP LLJIT::getProcessSymbolsJITDylib() { return ProcessSymbols; }
860
861JITDylibSP LLJIT::getPlatformJITDylib() { return Platform; }
862
863Expected<JITDylib &> LLJIT::createJITDylib(std::string Name) {
864 auto JD = ES->createJITDylib(Name: std::move(Name));
865 if (!JD)
866 return JD.takeError();
867
868 JD->addToLinkOrder(NewLinks: DefaultLinks);
869 return JD;
870}
871
872Expected<JITDylib &> LLJIT::loadPlatformDynamicLibrary(const char *Path) {
873 auto G = EPCDynamicLibrarySearchGenerator::Load(ES&: *ES, DylibMgr&: *DylibMgr, LibraryPath: Path);
874 if (!G)
875 return G.takeError();
876
877 if (auto *ExistingJD = ES->getJITDylibByName(Name: Path))
878 return *ExistingJD;
879
880 auto &JD = ES->createBareJITDylib(Name: Path);
881 JD.addGenerator(DefGenerator: std::move(*G));
882 return JD;
883}
884
885Error LLJIT::linkStaticLibraryInto(JITDylib &JD,
886 std::unique_ptr<MemoryBuffer> LibBuffer) {
887 auto G = StaticLibraryDefinitionGenerator::Create(L&: *ObjLinkingLayer,
888 ArchiveBuffer: std::move(LibBuffer));
889 if (!G)
890 return G.takeError();
891
892 JD.addGenerator(DefGenerator: std::move(*G));
893
894 return Error::success();
895}
896
897Error LLJIT::linkStaticLibraryInto(JITDylib &JD, const char *Path) {
898 auto G = StaticLibraryDefinitionGenerator::Load(L&: *ObjLinkingLayer, FileName: Path);
899 if (!G)
900 return G.takeError();
901
902 JD.addGenerator(DefGenerator: std::move(*G));
903
904 return Error::success();
905}
906
907Error LLJIT::addIRModule(ResourceTrackerSP RT, ThreadSafeModule TSM) {
908 assert(TSM && "Can not add null module");
909
910 if (auto Err =
911 TSM.withModuleDo(F: [&](Module &M) { return applyTargetConfig(M); }))
912 return Err;
913
914 return InitHelperTransformLayer->add(RT: std::move(RT), TSM: std::move(TSM));
915}
916
917Error LLJIT::addIRModule(JITDylib &JD, ThreadSafeModule TSM) {
918 return addIRModule(RT: JD.getDefaultResourceTracker(), TSM: std::move(TSM));
919}
920
921Error LLJIT::addObjectFile(ResourceTrackerSP RT,
922 std::unique_ptr<MemoryBuffer> Obj) {
923 assert(Obj && "Can not add null object");
924
925 return ObjTransformLayer->add(RT: std::move(RT), O: std::move(Obj));
926}
927
928Error LLJIT::addObjectFile(JITDylib &JD, std::unique_ptr<MemoryBuffer> Obj) {
929 return addObjectFile(RT: JD.getDefaultResourceTracker(), Obj: std::move(Obj));
930}
931
932Expected<ExecutorAddr> LLJIT::lookupLinkerMangled(JITDylib &JD,
933 SymbolStringPtr Name) {
934 if (auto Sym = ES->lookup(
935 SearchOrder: makeJITDylibSearchOrder(JDs: &JD, Flags: JITDylibLookupFlags::MatchAllSymbols),
936 Symbol: Name))
937 return Sym->getAddress();
938 else
939 return Sym.takeError();
940}
941
942Expected<std::unique_ptr<jitlink::JITLinkMemoryManager>>
943LLJIT::createMemoryManager(LLJITBuilderState &S, ExecutionSession &ES) {
944 if (S.CreateMemoryManager)
945 return S.CreateMemoryManager(ES);
946 return ES.getExecutorProcessControl().createDefaultMemoryManager();
947}
948
949Expected<std::unique_ptr<ObjectLayer>>
950LLJIT::createObjectLinkingLayer(LLJITBuilderState &S, ExecutionSession &ES,
951 jitlink::JITLinkMemoryManager &MemMgr) {
952
953 // If the config state provided an ObjectLinkingLayer factory then use it.
954 if (S.CreateObjectLinkingLayer)
955 return S.CreateObjectLinkingLayer(ES, MemMgr);
956
957 // Otherwise default to creating an RTDyldObjectLinkingLayer that constructs
958 // a new SectionMemoryManager for each object.
959 auto GetMemMgr = [](const MemoryBuffer &) {
960 return std::make_unique<SectionMemoryManager>();
961 };
962 auto Layer =
963 std::make_unique<RTDyldObjectLinkingLayer>(args&: ES, args: std::move(GetMemMgr));
964
965 if (S.JTMB->getTargetTriple().isOSBinFormatCOFF()) {
966 Layer->setOverrideObjectFlagsWithResponsibilityFlags(true);
967 Layer->setAutoClaimResponsibilityForObjectSymbols(true);
968 }
969
970 if (S.JTMB->getTargetTriple().isOSBinFormatELF() &&
971 (S.JTMB->getTargetTriple().getArch() == Triple::ArchType::ppc64 ||
972 S.JTMB->getTargetTriple().getArch() == Triple::ArchType::ppc64le))
973 Layer->setAutoClaimResponsibilityForObjectSymbols(true);
974
975 // FIXME: Explicit conversion to std::unique_ptr<ObjectLayer> added to silence
976 // errors from some GCC / libstdc++ bots. Remove this conversion (i.e.
977 // just return ObjLinkingLayer) once those bots are upgraded.
978 return std::unique_ptr<ObjectLayer>(std::move(Layer));
979}
980
981Expected<std::unique_ptr<IRCompileLayer::IRCompiler>>
982LLJIT::createCompileFunction(LLJITBuilderState &S,
983 JITTargetMachineBuilder JTMB) {
984
985 /// If there is a custom compile function creator set then use it.
986 if (S.CreateCompileFunction)
987 return S.CreateCompileFunction(std::move(JTMB));
988
989 // If using a custom EPC then use a ConcurrentIRCompiler by default.
990 if (*S.SupportConcurrentCompilation)
991 return std::make_unique<ConcurrentIRCompiler>(args: std::move(JTMB));
992
993 auto TM = JTMB.createTargetMachine();
994 if (!TM)
995 return TM.takeError();
996
997 return std::make_unique<TMOwningSimpleCompiler>(args: std::move(*TM));
998}
999
1000LLJIT::LLJIT(LLJITBuilderState &S, Error &Err)
1001 : DL(std::move(*S.DL)), TT(S.JTMB->getTargetTriple()) {
1002
1003 ErrorAsOutParameter _(Err);
1004
1005 assert(!(S.EPC && S.ES) && "EPC and ES should not both be set");
1006
1007 if (S.EPC) {
1008 ES = std::make_unique<ExecutionSession>(args: std::move(S.EPC));
1009 } else if (S.ES)
1010 ES = std::move(S.ES);
1011 else {
1012 if (auto EPC = SelfExecutorProcessControl::Create()) {
1013 ES = std::make_unique<ExecutionSession>(args: std::move(*EPC));
1014 } else {
1015 Err = EPC.takeError();
1016 return;
1017 }
1018 }
1019
1020 if (auto MM = createMemoryManager(S, ES&: *ES))
1021 MemMgr = std::move(*MM);
1022 else {
1023 Err = MM.takeError();
1024 return;
1025 }
1026
1027 if (auto DM = ES->getExecutorProcessControl().createDefaultDylibMgr())
1028 DylibMgr = std::move(*DM);
1029 else {
1030 Err = DM.takeError();
1031 return;
1032 }
1033
1034 auto ObjLayer = createObjectLinkingLayer(S, ES&: *ES, MemMgr&: *MemMgr);
1035 if (!ObjLayer) {
1036 Err = ObjLayer.takeError();
1037 return;
1038 }
1039 ObjLinkingLayer = std::move(*ObjLayer);
1040 ObjTransformLayer =
1041 std::make_unique<ObjectTransformLayer>(args&: *ES, args&: *ObjLinkingLayer);
1042
1043 {
1044 auto CompileFunction = createCompileFunction(S, JTMB: std::move(*S.JTMB));
1045 if (!CompileFunction) {
1046 Err = CompileFunction.takeError();
1047 return;
1048 }
1049 CompileLayer = std::make_unique<IRCompileLayer>(
1050 args&: *ES, args&: *ObjTransformLayer, args: std::move(*CompileFunction));
1051 TransformLayer = std::make_unique<IRTransformLayer>(args&: *ES, args&: *CompileLayer);
1052 InitHelperTransformLayer =
1053 std::make_unique<IRTransformLayer>(args&: *ES, args&: *TransformLayer);
1054 }
1055
1056 if (*S.SupportConcurrentCompilation)
1057 InitHelperTransformLayer->setCloneToNewContextOnEmit(true);
1058
1059 if (S.SetupProcessSymbolsJITDylib) {
1060 if (auto ProcSymsJD = S.SetupProcessSymbolsJITDylib(*this)) {
1061 ProcessSymbols = ProcSymsJD->get();
1062 } else {
1063 Err = ProcSymsJD.takeError();
1064 return;
1065 }
1066 }
1067
1068 if (S.PrePlatformSetup)
1069 if ((Err = S.PrePlatformSetup(*this)))
1070 return;
1071
1072 if (!S.SetUpPlatform)
1073 S.SetUpPlatform = setUpGenericLLVMIRPlatform;
1074
1075 if (auto PlatformJDOrErr = S.SetUpPlatform(*this)) {
1076 Platform = PlatformJDOrErr->get();
1077 if (Platform)
1078 DefaultLinks.push_back(
1079 x: {Platform, JITDylibLookupFlags::MatchExportedSymbolsOnly});
1080 } else {
1081 Err = PlatformJDOrErr.takeError();
1082 return;
1083 }
1084
1085 if (S.LinkProcessSymbolsByDefault)
1086 DefaultLinks.push_back(
1087 x: {ProcessSymbols, JITDylibLookupFlags::MatchExportedSymbolsOnly});
1088
1089 if (auto MainOrErr = createJITDylib(Name: "main"))
1090 Main = &*MainOrErr;
1091 else {
1092 Err = MainOrErr.takeError();
1093 return;
1094 }
1095}
1096
1097std::string LLJIT::mangle(StringRef UnmangledName) const {
1098 std::string MangledName;
1099 {
1100 raw_string_ostream MangledNameStream(MangledName);
1101 llvm::Mangler::getNameWithPrefix(OS&: MangledNameStream, GVName: UnmangledName, DL);
1102 }
1103 return MangledName;
1104}
1105
1106Error LLJIT::applyTargetConfig(Module &M) {
1107 if (M.getTargetTriple().empty())
1108 M.setTargetTriple(TT);
1109
1110 if (M.getDataLayout().isDefault())
1111 M.setDataLayout(DL);
1112
1113 if (M.getDataLayout() != DL)
1114 return make_error<StringError>(
1115 Args: "Added modules have incompatible data layouts: " +
1116 M.getDataLayout().getStringRepresentation() + " (module) vs " +
1117 DL.getStringRepresentation() + " (jit)",
1118 Args: inconvertibleErrorCode());
1119
1120 return Error::success();
1121}
1122
1123Error setUpOrcPlatformManually(LLJIT &J) {
1124 LLVM_DEBUG({ dbgs() << "Setting up orc platform support for LLJIT\n"; });
1125 J.setPlatformSupport(std::make_unique<ORCPlatformSupport>(args&: J));
1126 return Error::success();
1127}
1128
1129class LoadAndLinkDynLibrary {
1130public:
1131 LoadAndLinkDynLibrary(LLJIT &J) : J(J) {}
1132 Error operator()(JITDylib &JD, StringRef DLLName) {
1133 if (!DLLName.ends_with_insensitive(Suffix: ".dll"))
1134 return make_error<StringError>(Args: "DLLName not ending with .dll",
1135 Args: inconvertibleErrorCode());
1136 auto DLLNameStr = DLLName.str(); // Guarantees null-termination.
1137 auto DLLJD = J.loadPlatformDynamicLibrary(Path: DLLNameStr.c_str());
1138 if (!DLLJD)
1139 return DLLJD.takeError();
1140 JD.addToLinkOrder(JD&: *DLLJD);
1141 return Error::success();
1142 }
1143
1144private:
1145 LLJIT &J;
1146};
1147
1148Expected<JITDylibSP> ExecutorNativePlatform::operator()(LLJIT &J) {
1149 auto ProcessSymbolsJD = J.getProcessSymbolsJITDylib();
1150 if (!ProcessSymbolsJD)
1151 return make_error<StringError>(
1152 Args: "Native platforms require a process symbols JITDylib",
1153 Args: inconvertibleErrorCode());
1154
1155 const Triple &TT = J.getTargetTriple();
1156 ObjectLinkingLayer *ObjLinkingLayer =
1157 dyn_cast<ObjectLinkingLayer>(Val: &J.getObjLinkingLayer());
1158
1159 if (!ObjLinkingLayer)
1160 return make_error<StringError>(
1161 Args: "ExecutorNativePlatform requires ObjectLinkingLayer",
1162 Args: inconvertibleErrorCode());
1163
1164 std::unique_ptr<MemoryBuffer> RuntimeArchiveBuffer;
1165 if (OrcRuntime.index() == 0) {
1166 auto A = errorOrToExpected(EO: MemoryBuffer::getFile(Filename: std::get<0>(v&: OrcRuntime)));
1167 if (!A)
1168 return A.takeError();
1169 RuntimeArchiveBuffer = std::move(*A);
1170 } else
1171 RuntimeArchiveBuffer = std::move(std::get<1>(v&: OrcRuntime));
1172
1173 auto &ES = J.getExecutionSession();
1174 auto &PlatformJD = ES.createBareJITDylib(Name: "<Platform>");
1175 PlatformJD.addToLinkOrder(JD&: *ProcessSymbolsJD);
1176
1177 J.setPlatformSupport(std::make_unique<ORCPlatformSupport>(args&: J));
1178
1179 switch (TT.getObjectFormat()) {
1180 case Triple::COFF: {
1181 const char *VCRuntimePath = nullptr;
1182 bool StaticVCRuntime = false;
1183 if (VCRuntime) {
1184 VCRuntimePath = VCRuntime->first.c_str();
1185 StaticVCRuntime = VCRuntime->second;
1186 }
1187 if (auto P = COFFPlatform::Create(
1188 ObjLinkingLayer&: *ObjLinkingLayer, PlatformJD, OrcRuntimeArchiveBuffer: std::move(RuntimeArchiveBuffer),
1189 LoadDynLibrary: LoadAndLinkDynLibrary(J), StaticVCRuntime, VCRuntimePath))
1190 J.getExecutionSession().setPlatform(std::move(*P));
1191 else
1192 return P.takeError();
1193 break;
1194 }
1195 case Triple::ELF: {
1196 auto G = StaticLibraryDefinitionGenerator::Create(
1197 L&: *ObjLinkingLayer, ArchiveBuffer: std::move(RuntimeArchiveBuffer));
1198 if (!G)
1199 return G.takeError();
1200
1201 if (auto P =
1202 ELFNixPlatform::Create(ObjLinkingLayer&: *ObjLinkingLayer, PlatformJD, OrcRuntime: std::move(*G)))
1203 J.getExecutionSession().setPlatform(std::move(*P));
1204 else
1205 return P.takeError();
1206 break;
1207 }
1208 case Triple::MachO: {
1209 auto G = StaticLibraryDefinitionGenerator::Create(
1210 L&: *ObjLinkingLayer, ArchiveBuffer: std::move(RuntimeArchiveBuffer));
1211 if (!G)
1212 return G.takeError();
1213
1214 if (auto P =
1215 MachOPlatform::Create(ObjLinkingLayer&: *ObjLinkingLayer, PlatformJD, OrcRuntime: std::move(*G)))
1216 ES.setPlatform(std::move(*P));
1217 else
1218 return P.takeError();
1219 break;
1220 }
1221 default:
1222 return make_error<StringError>(Args: "Unsupported object format in triple " +
1223 TT.str(),
1224 Args: inconvertibleErrorCode());
1225 }
1226
1227 return &PlatformJD;
1228}
1229
1230Expected<JITDylibSP> setUpGenericLLVMIRPlatform(LLJIT &J) {
1231 LLVM_DEBUG(
1232 { dbgs() << "Setting up GenericLLVMIRPlatform support for LLJIT\n"; });
1233 auto ProcessSymbolsJD = J.getProcessSymbolsJITDylib();
1234 if (!ProcessSymbolsJD)
1235 return make_error<StringError>(
1236 Args: "Native platforms require a process symbols JITDylib",
1237 Args: inconvertibleErrorCode());
1238
1239 auto &PlatformJD = J.getExecutionSession().createBareJITDylib(Name: "<Platform>");
1240 PlatformJD.addToLinkOrder(JD&: *ProcessSymbolsJD);
1241
1242 if (auto *OLL = dyn_cast<ObjectLinkingLayer>(Val: &J.getObjLinkingLayer())) {
1243
1244 bool UseEHFrames = true;
1245
1246 // Enable compact-unwind support if possible.
1247 if (J.getTargetTriple().isOSDarwin() ||
1248 J.getTargetTriple().isOSBinFormatMachO()) {
1249
1250 // Check if the bootstrap map says that we should force eh-frames:
1251 // Older libunwinds require this as they don't have a dynamic
1252 // registration API for compact-unwind.
1253 std::optional<bool> ForceEHFrames;
1254 if (auto Err = J.getExecutionSession().getBootstrapMapValue<bool, bool>(
1255 Key: "darwin-use-ehframes-only", Val&: ForceEHFrames))
1256 return Err;
1257 if (ForceEHFrames.has_value())
1258 UseEHFrames = *ForceEHFrames;
1259 else
1260 UseEHFrames = false;
1261
1262 // If UseEHFrames hasn't been set then we're good to use compact-unwind.
1263 if (!UseEHFrames) {
1264 if (auto UIRP =
1265 UnwindInfoRegistrationPlugin::Create(ES&: J.getExecutionSession())) {
1266 OLL->addPlugin(P: std::move(*UIRP));
1267 LLVM_DEBUG(dbgs() << "Enabled compact-unwind support.\n");
1268 } else
1269 return UIRP.takeError();
1270 }
1271 }
1272
1273 // Otherwise fall back to standard unwind registration.
1274 if (UseEHFrames) {
1275 auto &ES = J.getExecutionSession();
1276 if (auto EHFP = EHFrameRegistrationPlugin::Create(ES)) {
1277 OLL->addPlugin(P: std::move(*EHFP));
1278 LLVM_DEBUG(dbgs() << "Enabled eh-frame support.\n");
1279 } else
1280 return EHFP.takeError();
1281 }
1282 }
1283
1284 J.setPlatformSupport(
1285 std::make_unique<GenericLLVMIRPlatformSupport>(args&: J, args&: PlatformJD));
1286
1287 return &PlatformJD;
1288}
1289
1290Expected<JITDylibSP> setUpInactivePlatform(LLJIT &J) {
1291 LLVM_DEBUG(
1292 { dbgs() << "Explicitly deactivated platform support for LLJIT\n"; });
1293 J.setPlatformSupport(std::make_unique<InactivePlatformSupport>());
1294 return nullptr;
1295}
1296
1297Error LLLazyJITBuilderState::prepareForConstruction() {
1298 if (auto Err = LLJITBuilderState::prepareForConstruction())
1299 return Err;
1300 TT = JTMB->getTargetTriple();
1301 return Error::success();
1302}
1303
1304Error LLLazyJIT::addLazyIRModule(JITDylib &JD, ThreadSafeModule TSM) {
1305 assert(TSM && "Can not add null module");
1306
1307 if (auto Err = TSM.withModuleDo(
1308 F: [&](Module &M) -> Error { return applyTargetConfig(M); }))
1309 return Err;
1310
1311 return CODLayer->add(JD, TSM: std::move(TSM));
1312}
1313
1314// End the session before this class's members (CODLayer, IPLayer, LCTMgr) are
1315// destroyed: endSession joins the compile threads, and those threads may still
1316// be operating on the CompileOnDemandLayer's per-dylib IndirectStubsManagers.
1317LLLazyJIT::~LLLazyJIT() {
1318 if (auto Err = ES->endSession())
1319 ES->reportError(Err: std::move(Err));
1320}
1321
1322LLLazyJIT::LLLazyJIT(LLLazyJITBuilderState &S, Error &Err) : LLJIT(S, Err) {
1323
1324 // If LLJIT construction failed then bail out.
1325 if (Err)
1326 return;
1327
1328 ErrorAsOutParameter _(&Err);
1329
1330 /// Take/Create the lazy-compile callthrough manager.
1331 if (S.LCTMgr)
1332 LCTMgr = std::move(S.LCTMgr);
1333 else {
1334 if (auto LCTMgrOrErr = createLocalLazyCallThroughManager(
1335 T: S.TT, ES&: *ES, ErrorHandlerAddr: S.LazyCompileFailureAddr))
1336 LCTMgr = std::move(*LCTMgrOrErr);
1337 else {
1338 Err = LCTMgrOrErr.takeError();
1339 return;
1340 }
1341 }
1342
1343 // Take/Create the indirect stubs manager builder.
1344 auto ISMBuilder = std::move(S.ISMBuilder);
1345
1346 // If none was provided, try to build one.
1347 if (!ISMBuilder)
1348 ISMBuilder = createLocalIndirectStubsManagerBuilder(T: S.TT);
1349
1350 // No luck. Bail out.
1351 if (!ISMBuilder) {
1352 Err = make_error<StringError>(Args: "Could not construct "
1353 "IndirectStubsManagerBuilder for target " +
1354 S.TT.str(),
1355 Args: inconvertibleErrorCode());
1356 return;
1357 }
1358
1359 // Create the IP Layer.
1360 IPLayer = std::make_unique<IRPartitionLayer>(args&: *ES, args&: *InitHelperTransformLayer);
1361
1362 // Create the COD layer.
1363 CODLayer = std::make_unique<CompileOnDemandLayer>(args&: *ES, args&: *IPLayer, args&: *LCTMgr,
1364 args: std::move(ISMBuilder));
1365
1366 if (*S.SupportConcurrentCompilation)
1367 CODLayer->setCloneToNewContextOnEmit(true);
1368}
1369
1370// In-process LLJIT uses eh-frame section wrappers via EPC, so we need to force
1371// them to be linked in.
1372LLVM_ATTRIBUTE_USED void linkComponents() {
1373 errs() << (void *)&llvm_orc_registerEHFrameSectionAllocAction
1374 << (void *)&llvm_orc_deregisterEHFrameSectionAllocAction;
1375}
1376
1377} // End namespace orc.
1378} // End namespace llvm.
1379