1//===--------- LLJIT.cpp - An ORC-based JIT for compiling LLVM IR ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/Orc/LLJIT.h"
10
11#include "llvm/Analysis/TargetLibraryInfo.h"
12#include "llvm/Config/llvm-config.h" // for LLVM_ENABLE_THREADS
13#include "llvm/ExecutionEngine/Orc/COFFPlatform.h"
14#include "llvm/ExecutionEngine/Orc/EHFrameRegistrationPlugin.h"
15#include "llvm/ExecutionEngine/Orc/ELFNixPlatform.h"
16#include "llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h"
17#include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
18#include "llvm/ExecutionEngine/Orc/MachOPlatform.h"
19#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
20#include "llvm/ExecutionEngine/Orc/ObjectTransformLayer.h"
21#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
22#include "llvm/ExecutionEngine/Orc/SelfExecutorProcessControl.h"
23#include "llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h"
24#include "llvm/ExecutionEngine/Orc/UnwindInfoRegistrationPlugin.h"
25#include "llvm/ExecutionEngine/SectionMemoryManager.h"
26#include "llvm/IR/GlobalVariable.h"
27#include "llvm/IR/IRBuilder.h"
28#include "llvm/IR/Mangler.h"
29#include "llvm/IR/Module.h"
30#include "llvm/Support/DynamicLibrary.h"
31
32#define DEBUG_TYPE "orc"
33
34using namespace llvm;
35using namespace llvm::orc;
36
37namespace {
38
39/// Adds helper function decls and wrapper functions that call the helper with
40/// some additional prefix arguments.
41///
42/// E.g. For wrapper "foo" with type i8(i8, i64), helper "bar", and prefix
43/// args i32 4 and i16 12345, this function will add:
44///
45/// declare i8 @bar(i32, i16, i8, i64)
46///
47/// define i8 @foo(i8, i64) {
48/// entry:
49/// %2 = call i8 @bar(i32 4, i16 12345, i8 %0, i64 %1)
50/// ret i8 %2
51/// }
52///
53Function *addHelperAndWrapper(Module &M, StringRef WrapperName,
54 FunctionType *WrapperFnType,
55 GlobalValue::VisibilityTypes WrapperVisibility,
56 StringRef HelperName,
57 ArrayRef<Value *> HelperPrefixArgs) {
58 std::vector<Type *> HelperArgTypes;
59 for (auto *Arg : HelperPrefixArgs)
60 HelperArgTypes.push_back(x: Arg->getType());
61 llvm::append_range(C&: HelperArgTypes, R: WrapperFnType->params());
62 auto *HelperFnType =
63 FunctionType::get(Result: WrapperFnType->getReturnType(), Params: HelperArgTypes, isVarArg: false);
64 auto *HelperFn = Function::Create(Ty: HelperFnType, Linkage: GlobalValue::ExternalLinkage,
65 N: HelperName, M);
66
67 auto *WrapperFn = Function::Create(
68 Ty: WrapperFnType, Linkage: GlobalValue::ExternalLinkage, N: WrapperName, M);
69 WrapperFn->setVisibility(WrapperVisibility);
70
71 auto *EntryBlock = BasicBlock::Create(Context&: M.getContext(), Name: "entry", Parent: WrapperFn);
72 IRBuilder<> IB(EntryBlock);
73
74 std::vector<Value *> HelperArgs;
75 llvm::append_range(C&: HelperArgs, R&: HelperPrefixArgs);
76 for (auto &Arg : WrapperFn->args())
77 HelperArgs.push_back(x: &Arg);
78 auto *HelperResult = IB.CreateCall(Callee: HelperFn, Args: HelperArgs);
79 if (HelperFn->getReturnType()->isVoidTy())
80 IB.CreateRetVoid();
81 else
82 IB.CreateRet(V: HelperResult);
83
84 return WrapperFn;
85}
86
87class GenericLLVMIRPlatformSupport;
88
89/// orc::Platform component of Generic LLVM IR Platform support.
90/// Just forwards calls to the GenericLLVMIRPlatformSupport class below.
91class GenericLLVMIRPlatform : public Platform {
92public:
93 GenericLLVMIRPlatform(GenericLLVMIRPlatformSupport &S) : S(S) {}
94 Error setupJITDylib(JITDylib &JD) override;
95 Error teardownJITDylib(JITDylib &JD) override;
96 Error notifyAdding(ResourceTracker &RT,
97 const MaterializationUnit &MU) override;
98 Error notifyRemoving(ResourceTracker &RT) override {
99 // Noop -- Nothing to do (yet).
100 return Error::success();
101 }
102
103private:
104 GenericLLVMIRPlatformSupport &S;
105};
106
107/// This transform parses llvm.global_ctors to produce a single initialization
108/// function for the module, records the function, then deletes
109/// llvm.global_ctors.
110class GlobalCtorDtorScraper {
111public:
112 GlobalCtorDtorScraper(GenericLLVMIRPlatformSupport &PS,
113 StringRef InitFunctionPrefix,
114 StringRef DeInitFunctionPrefix)
115 : PS(PS), InitFunctionPrefix(InitFunctionPrefix),
116 DeInitFunctionPrefix(DeInitFunctionPrefix) {}
117 Expected<ThreadSafeModule> operator()(ThreadSafeModule TSM,
118 MaterializationResponsibility &R);
119
120private:
121 GenericLLVMIRPlatformSupport &PS;
122 StringRef InitFunctionPrefix;
123 StringRef DeInitFunctionPrefix;
124};
125
126/// Generic IR Platform Support
127///
128/// Scrapes llvm.global_ctors and llvm.global_dtors and replaces them with
129/// specially named 'init' and 'deinit'. Injects definitions / interposes for
130/// some runtime API, including __cxa_atexit, dlopen, and dlclose.
131class GenericLLVMIRPlatformSupport : public LLJIT::PlatformSupport {
132public:
133 GenericLLVMIRPlatformSupport(LLJIT &J, JITDylib &PlatformJD)
134 : J(J), InitFunctionPrefix(J.mangle(UnmangledName: "__orc_init_func.")),
135 DeInitFunctionPrefix(J.mangle(UnmangledName: "__orc_deinit_func.")) {
136
137 getExecutionSession().setPlatform(
138 std::make_unique<GenericLLVMIRPlatform>(args&: *this));
139
140 setInitTransform(J, T: GlobalCtorDtorScraper(*this, InitFunctionPrefix,
141 DeInitFunctionPrefix));
142
143 SymbolMap StdInterposes;
144
145 StdInterposes[J.mangleAndIntern(UnmangledName: "__lljit.platform_support_instance")] = {
146 ExecutorAddr::fromPtr(Ptr: this), JITSymbolFlags::Exported};
147 StdInterposes[J.mangleAndIntern(UnmangledName: "__lljit.cxa_atexit_helper")] = {
148 ExecutorAddr::fromPtr(Ptr: registerCxaAtExitHelper), JITSymbolFlags()};
149
150 cantFail(Err: PlatformJD.define(MU: absoluteSymbols(Symbols: std::move(StdInterposes))));
151 cantFail(Err: setupJITDylib(PlatformJD));
152 cantFail(Err: J.addIRModule(JD&: PlatformJD, TSM: createPlatformRuntimeModule()));
153 }
154
155 ExecutionSession &getExecutionSession() { return J.getExecutionSession(); }
156
157 /// Adds a module that defines the __dso_handle global.
158 Error setupJITDylib(JITDylib &JD) {
159
160 // Add per-jitdylib standard interposes.
161 SymbolMap PerJDInterposes;
162 PerJDInterposes[J.mangleAndIntern(UnmangledName: "__lljit.run_atexits_helper")] = {
163 ExecutorAddr::fromPtr(Ptr: runAtExitsHelper), JITSymbolFlags()};
164 PerJDInterposes[J.mangleAndIntern(UnmangledName: "__lljit.atexit_helper")] = {
165 ExecutorAddr::fromPtr(Ptr: registerAtExitHelper), JITSymbolFlags()};
166 cantFail(Err: JD.define(MU: absoluteSymbols(Symbols: std::move(PerJDInterposes))));
167
168 auto Ctx = std::make_unique<LLVMContext>();
169 auto M = std::make_unique<Module>(args: "__standard_lib", args&: *Ctx);
170 M->setDataLayout(J.getDataLayout());
171
172 auto *Int64Ty = Type::getInt64Ty(C&: *Ctx);
173 auto *DSOHandle = new GlobalVariable(
174 *M, Int64Ty, true, GlobalValue::ExternalLinkage,
175 ConstantInt::get(Ty: Int64Ty, V: reinterpret_cast<uintptr_t>(&JD)),
176 "__dso_handle");
177 DSOHandle->setVisibility(GlobalValue::DefaultVisibility);
178 DSOHandle->setInitializer(
179 ConstantInt::get(Ty: Int64Ty, V: ExecutorAddr::fromPtr(Ptr: &JD).getValue()));
180
181 auto *GenericIRPlatformSupportTy =
182 StructType::create(Context&: *Ctx, Name: "lljit.GenericLLJITIRPlatformSupport");
183
184 auto *PlatformInstanceDecl = new GlobalVariable(
185 *M, GenericIRPlatformSupportTy, true, GlobalValue::ExternalLinkage,
186 nullptr, "__lljit.platform_support_instance");
187
188 auto *VoidTy = Type::getVoidTy(C&: *Ctx);
189 addHelperAndWrapper(
190 M&: *M, WrapperName: "__lljit_run_atexits", WrapperFnType: FunctionType::get(Result: VoidTy, Params: {}, isVarArg: false),
191 WrapperVisibility: GlobalValue::HiddenVisibility, HelperName: "__lljit.run_atexits_helper",
192 HelperPrefixArgs: {PlatformInstanceDecl, DSOHandle});
193
194 auto *IntTy = Type::getIntNTy(C&: *Ctx, N: sizeof(int) * CHAR_BIT);
195 auto *AtExitCallbackPtrTy = PointerType::getUnqual(C&: *Ctx);
196 auto *AtExit = addHelperAndWrapper(
197 M&: *M, WrapperName: "atexit", WrapperFnType: FunctionType::get(Result: IntTy, Params: {AtExitCallbackPtrTy}, isVarArg: false),
198 WrapperVisibility: GlobalValue::HiddenVisibility, HelperName: "__lljit.atexit_helper",
199 HelperPrefixArgs: {PlatformInstanceDecl, DSOHandle});
200 Attribute::AttrKind AtExitExtAttr =
201 TargetLibraryInfo::getExtAttrForI32Return(T: J.getTargetTriple());
202 if (AtExitExtAttr != Attribute::None)
203 AtExit->addRetAttr(Kind: AtExitExtAttr);
204
205 return J.addIRModule(JD, TSM: ThreadSafeModule(std::move(M), std::move(Ctx)));
206 }
207
208 Error notifyAdding(ResourceTracker &RT, const MaterializationUnit &MU) {
209 auto &JD = RT.getJITDylib();
210 if (auto &InitSym = MU.getInitializerSymbol())
211 InitSymbols[&JD].add(Name: InitSym, Flags: SymbolLookupFlags::WeaklyReferencedSymbol);
212 else {
213 // If there's no identified init symbol attached, but there is a symbol
214 // with the GenericIRPlatform::InitFunctionPrefix, then treat that as
215 // an init function. Add the symbol to both the InitSymbols map (which
216 // will trigger a lookup to materialize the module) and the InitFunctions
217 // map (which holds the names of the symbols to execute).
218 for (auto &KV : MU.getSymbols())
219 if ((*KV.first).starts_with(Prefix: InitFunctionPrefix)) {
220 InitSymbols[&JD].add(Name: KV.first,
221 Flags: SymbolLookupFlags::WeaklyReferencedSymbol);
222 InitFunctions[&JD].add(Name: KV.first);
223 } else if ((*KV.first).starts_with(Prefix: DeInitFunctionPrefix)) {
224 DeInitFunctions[&JD].add(Name: KV.first);
225 }
226 }
227 return Error::success();
228 }
229
230 Error initialize(JITDylib &JD) override {
231 LLVM_DEBUG({
232 dbgs() << "GenericLLVMIRPlatformSupport getting initializers to run\n";
233 });
234 if (auto Initializers = getInitializers(JD)) {
235 LLVM_DEBUG(
236 { dbgs() << "GenericLLVMIRPlatformSupport running initializers\n"; });
237 for (auto InitFnAddr : *Initializers) {
238 LLVM_DEBUG({
239 dbgs() << " Running init " << formatv("{0:x16}", InitFnAddr)
240 << "...\n";
241 });
242 auto *InitFn = InitFnAddr.toPtr<void (*)()>();
243 InitFn();
244 }
245 } else
246 return Initializers.takeError();
247 return Error::success();
248 }
249
250 Error deinitialize(JITDylib &JD) override {
251 LLVM_DEBUG({
252 dbgs() << "GenericLLVMIRPlatformSupport getting deinitializers to run\n";
253 });
254 if (auto Deinitializers = getDeinitializers(JD)) {
255 LLVM_DEBUG({
256 dbgs() << "GenericLLVMIRPlatformSupport running deinitializers\n";
257 });
258 for (auto DeinitFnAddr : *Deinitializers) {
259 LLVM_DEBUG({
260 dbgs() << " Running deinit " << formatv("{0:x16}", DeinitFnAddr)
261 << "...\n";
262 });
263 auto *DeinitFn = DeinitFnAddr.toPtr<void (*)()>();
264 DeinitFn();
265 }
266 } else
267 return Deinitializers.takeError();
268
269 return Error::success();
270 }
271
272 void registerInitFunc(JITDylib &JD, SymbolStringPtr InitName) {
273 getExecutionSession().runSessionLocked(
274 F: [&]() { InitFunctions[&JD].add(Name: InitName); });
275 }
276
277 void registerDeInitFunc(JITDylib &JD, SymbolStringPtr DeInitName) {
278 getExecutionSession().runSessionLocked(
279 F: [&]() { DeInitFunctions[&JD].add(Name: DeInitName); });
280 }
281
282private:
283 Expected<std::vector<ExecutorAddr>> getInitializers(JITDylib &JD) {
284 if (auto Err = issueInitLookups(JD))
285 return std::move(Err);
286
287 DenseMap<JITDylib *, SymbolLookupSet> LookupSymbols;
288 std::vector<JITDylibSP> DFSLinkOrder;
289
290 if (auto Err = getExecutionSession().runSessionLocked(F: [&]() -> Error {
291 if (auto DFSLinkOrderOrErr = JD.getDFSLinkOrder())
292 DFSLinkOrder = std::move(*DFSLinkOrderOrErr);
293 else
294 return DFSLinkOrderOrErr.takeError();
295
296 for (auto &NextJD : DFSLinkOrder) {
297 auto IFItr = InitFunctions.find(Val: NextJD.get());
298 if (IFItr != InitFunctions.end()) {
299 LookupSymbols[NextJD.get()] = std::move(IFItr->second);
300 InitFunctions.erase(I: IFItr);
301 }
302 }
303 return Error::success();
304 }))
305 return std::move(Err);
306
307 LLVM_DEBUG({
308 dbgs() << "JITDylib init order is [ ";
309 for (auto &JD : llvm::reverse(DFSLinkOrder))
310 dbgs() << "\"" << JD->getName() << "\" ";
311 dbgs() << "]\n";
312 dbgs() << "Looking up init functions:\n";
313 for (auto &KV : LookupSymbols)
314 dbgs() << " \"" << KV.first->getName() << "\": " << KV.second << "\n";
315 });
316
317 auto &ES = getExecutionSession();
318 auto LookupResult = Platform::lookupInitSymbols(ES, InitSyms: LookupSymbols);
319
320 if (!LookupResult)
321 return LookupResult.takeError();
322
323 std::vector<ExecutorAddr> Initializers;
324 while (!DFSLinkOrder.empty()) {
325 auto &NextJD = *DFSLinkOrder.back();
326 DFSLinkOrder.pop_back();
327 auto InitsItr = LookupResult->find(Val: &NextJD);
328 if (InitsItr == LookupResult->end())
329 continue;
330 for (auto &KV : InitsItr->second)
331 Initializers.push_back(x: KV.second.getAddress());
332 }
333
334 return Initializers;
335 }
336
337 Expected<std::vector<ExecutorAddr>> getDeinitializers(JITDylib &JD) {
338 auto &ES = getExecutionSession();
339
340 auto LLJITRunAtExits = J.mangleAndIntern(UnmangledName: "__lljit_run_atexits");
341
342 DenseMap<JITDylib *, SymbolLookupSet> LookupSymbols;
343 std::vector<JITDylibSP> DFSLinkOrder;
344
345 if (auto Err = ES.runSessionLocked(F: [&]() -> Error {
346 if (auto DFSLinkOrderOrErr = JD.getDFSLinkOrder())
347 DFSLinkOrder = std::move(*DFSLinkOrderOrErr);
348 else
349 return DFSLinkOrderOrErr.takeError();
350
351 for (auto &NextJD : DFSLinkOrder) {
352 auto &JDLookupSymbols = LookupSymbols[NextJD.get()];
353 auto DIFItr = DeInitFunctions.find(Val: NextJD.get());
354 if (DIFItr != DeInitFunctions.end()) {
355 LookupSymbols[NextJD.get()] = std::move(DIFItr->second);
356 DeInitFunctions.erase(I: DIFItr);
357 }
358 JDLookupSymbols.add(Name: LLJITRunAtExits,
359 Flags: SymbolLookupFlags::WeaklyReferencedSymbol);
360 }
361 return Error::success();
362 }))
363 return std::move(Err);
364
365 LLVM_DEBUG({
366 dbgs() << "JITDylib deinit order is [ ";
367 for (auto &JD : DFSLinkOrder)
368 dbgs() << "\"" << JD->getName() << "\" ";
369 dbgs() << "]\n";
370 dbgs() << "Looking up deinit functions:\n";
371 for (auto &KV : LookupSymbols)
372 dbgs() << " \"" << KV.first->getName() << "\": " << KV.second << "\n";
373 });
374
375 auto LookupResult = Platform::lookupInitSymbols(ES, InitSyms: LookupSymbols);
376
377 if (!LookupResult)
378 return LookupResult.takeError();
379
380 std::vector<ExecutorAddr> DeInitializers;
381 for (auto &NextJD : DFSLinkOrder) {
382 auto DeInitsItr = LookupResult->find(Val: NextJD.get());
383 assert(DeInitsItr != LookupResult->end() &&
384 "Every JD should have at least __lljit_run_atexits");
385
386 auto RunAtExitsItr = DeInitsItr->second.find(Val: LLJITRunAtExits);
387 if (RunAtExitsItr != DeInitsItr->second.end())
388 DeInitializers.push_back(x: RunAtExitsItr->second.getAddress());
389
390 for (auto &KV : DeInitsItr->second)
391 if (KV.first != LLJITRunAtExits)
392 DeInitializers.push_back(x: KV.second.getAddress());
393 }
394
395 return DeInitializers;
396 }
397
398 /// Issue lookups for all init symbols required to initialize JD (and any
399 /// JITDylibs that it depends on).
400 Error issueInitLookups(JITDylib &JD) {
401 DenseMap<JITDylib *, SymbolLookupSet> RequiredInitSymbols;
402 std::vector<JITDylibSP> DFSLinkOrder;
403
404 if (auto Err = getExecutionSession().runSessionLocked(F: [&]() -> Error {
405 if (auto DFSLinkOrderOrErr = JD.getDFSLinkOrder())
406 DFSLinkOrder = std::move(*DFSLinkOrderOrErr);
407 else
408 return DFSLinkOrderOrErr.takeError();
409
410 for (auto &NextJD : DFSLinkOrder) {
411 auto ISItr = InitSymbols.find(Val: NextJD.get());
412 if (ISItr != InitSymbols.end()) {
413 RequiredInitSymbols[NextJD.get()] = std::move(ISItr->second);
414 InitSymbols.erase(I: ISItr);
415 }
416 }
417 return Error::success();
418 }))
419 return Err;
420
421 return Platform::lookupInitSymbols(ES&: getExecutionSession(),
422 InitSyms: RequiredInitSymbols)
423 .takeError();
424 }
425
426 static void registerCxaAtExitHelper(void *Self, void (*F)(void *), void *Ctx,
427 void *DSOHandle) {
428 LLVM_DEBUG({
429 dbgs() << "Registering cxa atexit function " << (void *)F << " for JD "
430 << (*static_cast<JITDylib **>(DSOHandle))->getName() << "\n";
431 });
432 static_cast<GenericLLVMIRPlatformSupport *>(Self)->AtExitMgr.registerAtExit(
433 F, Ctx, DSOHandle);
434 }
435
436 static void registerAtExitHelper(void *Self, void *DSOHandle, void (*F)()) {
437 LLVM_DEBUG({
438 dbgs() << "Registering atexit function " << (void *)F << " for JD "
439 << (*static_cast<JITDylib **>(DSOHandle))->getName() << "\n";
440 });
441 static_cast<GenericLLVMIRPlatformSupport *>(Self)->AtExitMgr.registerAtExit(
442 F: reinterpret_cast<void (*)(void *)>(F), Ctx: nullptr, DSOHandle);
443 }
444
445 static void runAtExitsHelper(void *Self, void *DSOHandle) {
446 LLVM_DEBUG({
447 dbgs() << "Running atexit functions for JD "
448 << (*static_cast<JITDylib **>(DSOHandle))->getName() << "\n";
449 });
450 static_cast<GenericLLVMIRPlatformSupport *>(Self)->AtExitMgr.runAtExits(
451 DSOHandle);
452 }
453
454 // Constructs an LLVM IR module containing platform runtime globals,
455 // functions, and interposes.
456 ThreadSafeModule createPlatformRuntimeModule() {
457 auto Ctx = std::make_unique<LLVMContext>();
458 auto M = std::make_unique<Module>(args: "__standard_lib", args&: *Ctx);
459 M->setDataLayout(J.getDataLayout());
460
461 auto *GenericIRPlatformSupportTy =
462 StructType::create(Context&: *Ctx, Name: "lljit.GenericLLJITIRPlatformSupport");
463
464 auto *PlatformInstanceDecl = new GlobalVariable(
465 *M, GenericIRPlatformSupportTy, true, GlobalValue::ExternalLinkage,
466 nullptr, "__lljit.platform_support_instance");
467
468 auto *IntTy = Type::getIntNTy(C&: *Ctx, N: sizeof(int) * CHAR_BIT);
469 auto *BytePtrTy = PointerType::getUnqual(C&: *Ctx);
470 auto *CxaAtExitCallbackPtrTy = PointerType::getUnqual(C&: *Ctx);
471
472 auto *CxaAtExit = addHelperAndWrapper(
473 M&: *M, WrapperName: "__cxa_atexit",
474 WrapperFnType: FunctionType::get(Result: IntTy, Params: {CxaAtExitCallbackPtrTy, BytePtrTy, BytePtrTy},
475 isVarArg: false),
476 WrapperVisibility: GlobalValue::DefaultVisibility, HelperName: "__lljit.cxa_atexit_helper",
477 HelperPrefixArgs: {PlatformInstanceDecl});
478 Attribute::AttrKind CxaAtExitExtAttr =
479 TargetLibraryInfo::getExtAttrForI32Return(T: J.getTargetTriple());
480 if (CxaAtExitExtAttr != Attribute::None)
481 CxaAtExit->addRetAttr(Kind: CxaAtExitExtAttr);
482
483 return ThreadSafeModule(std::move(M), std::move(Ctx));
484 }
485
486 LLJIT &J;
487 std::string InitFunctionPrefix;
488 std::string DeInitFunctionPrefix;
489 DenseMap<JITDylib *, SymbolLookupSet> InitSymbols;
490 DenseMap<JITDylib *, SymbolLookupSet> InitFunctions;
491 DenseMap<JITDylib *, SymbolLookupSet> DeInitFunctions;
492 ItaniumCXAAtExitSupport AtExitMgr;
493};
494
495Error GenericLLVMIRPlatform::setupJITDylib(JITDylib &JD) {
496 return S.setupJITDylib(JD);
497}
498
499Error GenericLLVMIRPlatform::teardownJITDylib(JITDylib &JD) {
500 return Error::success();
501}
502
503Error GenericLLVMIRPlatform::notifyAdding(ResourceTracker &RT,
504 const MaterializationUnit &MU) {
505 return S.notifyAdding(RT, MU);
506}
507
508Expected<ThreadSafeModule>
509GlobalCtorDtorScraper::operator()(ThreadSafeModule TSM,
510 MaterializationResponsibility &R) {
511 auto Err = TSM.withModuleDo(F: [&](Module &M) -> Error {
512 auto &Ctx = M.getContext();
513 auto *GlobalCtors = M.getNamedGlobal(Name: "llvm.global_ctors");
514 auto *GlobalDtors = M.getNamedGlobal(Name: "llvm.global_dtors");
515
516 auto RegisterCOrDtors = [&](GlobalVariable *GlobalCOrDtors,
517 bool isCtor) -> Error {
518 // If there's no llvm.global_c/dtor or it's just a decl then skip.
519 if (!GlobalCOrDtors || GlobalCOrDtors->isDeclaration())
520 return Error::success();
521 std::string InitOrDeInitFunctionName;
522 if (isCtor)
523 raw_string_ostream(InitOrDeInitFunctionName)
524 << InitFunctionPrefix << M.getModuleIdentifier();
525 else
526 raw_string_ostream(InitOrDeInitFunctionName)
527 << DeInitFunctionPrefix << M.getModuleIdentifier();
528
529 MangleAndInterner Mangle(PS.getExecutionSession(), M.getDataLayout());
530 auto InternedInitOrDeInitName = Mangle(InitOrDeInitFunctionName);
531 if (auto Err = R.defineMaterializing(
532 SymbolFlags: {{InternedInitOrDeInitName, JITSymbolFlags::Callable}}))
533 return Err;
534
535 auto *InitOrDeInitFunc = Function::Create(
536 Ty: FunctionType::get(Result: Type::getVoidTy(C&: Ctx), Params: {}, isVarArg: false),
537 Linkage: GlobalValue::ExternalLinkage, N: InitOrDeInitFunctionName, M: &M);
538 InitOrDeInitFunc->setVisibility(GlobalValue::HiddenVisibility);
539 std::vector<std::pair<Function *, unsigned>> InitsOrDeInits;
540 auto COrDtors = isCtor ? getConstructors(M) : getDestructors(M);
541
542 for (auto E : COrDtors)
543 InitsOrDeInits.push_back(x: std::make_pair(x&: E.Func, y&: E.Priority));
544 llvm::stable_sort(Range&: InitsOrDeInits, C: llvm::less_second());
545
546 auto *InitOrDeInitFuncEntryBlock =
547 BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: InitOrDeInitFunc);
548 IRBuilder<> IB(InitOrDeInitFuncEntryBlock);
549 for (auto &KV : InitsOrDeInits)
550 IB.CreateCall(Callee: KV.first);
551 IB.CreateRetVoid();
552
553 if (isCtor)
554 PS.registerInitFunc(JD&: R.getTargetJITDylib(), InitName: InternedInitOrDeInitName);
555 else
556 PS.registerDeInitFunc(JD&: R.getTargetJITDylib(), DeInitName: InternedInitOrDeInitName);
557
558 GlobalCOrDtors->eraseFromParent();
559 return Error::success();
560 };
561
562 if (auto Err = RegisterCOrDtors(GlobalCtors, true))
563 return Err;
564 if (auto Err = RegisterCOrDtors(GlobalDtors, false))
565 return Err;
566
567 return Error::success();
568 });
569
570 if (Err)
571 return std::move(Err);
572
573 return std::move(TSM);
574}
575
576/// Inactive Platform Support
577///
578/// Explicitly disables platform support. JITDylibs are not scanned for special
579/// init/deinit symbols. No runtime API interposes are injected.
580class InactivePlatformSupport : public LLJIT::PlatformSupport {
581public:
582 InactivePlatformSupport() = default;
583
584 Error initialize(JITDylib &JD) override {
585 LLVM_DEBUG(dbgs() << "InactivePlatformSupport: no initializers running for "
586 << JD.getName() << "\n");
587 return Error::success();
588 }
589
590 Error deinitialize(JITDylib &JD) override {
591 LLVM_DEBUG(
592 dbgs() << "InactivePlatformSupport: no deinitializers running for "
593 << JD.getName() << "\n");
594 return Error::success();
595 }
596};
597
598} // end anonymous namespace
599
600namespace llvm {
601namespace orc {
602
603Error ORCPlatformSupport::initialize(orc::JITDylib &JD) {
604 using llvm::orc::shared::SPSExecutorAddr;
605 using llvm::orc::shared::SPSString;
606 using SPSDLOpenSig = SPSExecutorAddr(SPSString, int32_t);
607 using SPSDLUpdateSig = int32_t(SPSExecutorAddr);
608 enum dlopen_mode : int32_t {
609 ORC_RT_RTLD_LAZY = 0x1,
610 ORC_RT_RTLD_NOW = 0x2,
611 ORC_RT_RTLD_LOCAL = 0x4,
612 ORC_RT_RTLD_GLOBAL = 0x8
613 };
614
615 auto &ES = J.getExecutionSession();
616 auto MainSearchOrder = J.getMainJITDylib().withLinkOrderDo(
617 F: [](const JITDylibSearchOrder &SO) { return SO; });
618 StringRef WrapperToCall = "__orc_rt_jit_dlopen_wrapper";
619 bool dlupdate = false;
620 const Triple &TT = ES.getTargetTriple();
621 if (TT.isOSBinFormatMachO() || TT.isOSBinFormatELF()) {
622 if (InitializedDylib.contains(Ptr: &JD)) {
623 WrapperToCall = "__orc_rt_jit_dlupdate_wrapper";
624 dlupdate = true;
625 } else
626 InitializedDylib.insert(Ptr: &JD);
627 }
628
629 if (auto WrapperAddr =
630 ES.lookup(SearchOrder: MainSearchOrder, Symbol: J.mangleAndIntern(UnmangledName: WrapperToCall))) {
631 if (dlupdate) {
632 int32_t result;
633 auto E = ES.callSPSWrapper<SPSDLUpdateSig>(WrapperFnAddr: WrapperAddr->getAddress(),
634 WrapperCallArgs&: result, WrapperCallArgs&: DSOHandles[&JD]);
635 if (result)
636 return make_error<StringError>(Args: "dlupdate failed",
637 Args: inconvertibleErrorCode());
638 return E;
639 }
640 return ES.callSPSWrapper<SPSDLOpenSig>(WrapperFnAddr: WrapperAddr->getAddress(),
641 WrapperCallArgs&: DSOHandles[&JD], WrapperCallArgs: JD.getName(),
642 WrapperCallArgs: int32_t(ORC_RT_RTLD_LAZY));
643 } else
644 return WrapperAddr.takeError();
645}
646
647Error ORCPlatformSupport::deinitialize(orc::JITDylib &JD) {
648 using llvm::orc::shared::SPSExecutorAddr;
649 using SPSDLCloseSig = int32_t(SPSExecutorAddr);
650
651 auto &ES = J.getExecutionSession();
652 auto MainSearchOrder = J.getMainJITDylib().withLinkOrderDo(
653 F: [](const JITDylibSearchOrder &SO) { return SO; });
654
655 if (auto WrapperAddr = ES.lookup(
656 SearchOrder: MainSearchOrder, Symbol: J.mangleAndIntern(UnmangledName: "__orc_rt_jit_dlclose_wrapper"))) {
657 int32_t result;
658 auto E = J.getExecutionSession().callSPSWrapper<SPSDLCloseSig>(
659 WrapperFnAddr: WrapperAddr->getAddress(), WrapperCallArgs&: result, WrapperCallArgs&: DSOHandles[&JD]);
660 if (E)
661 return E;
662 else if (result)
663 return make_error<StringError>(Args: "dlclose failed",
664 Args: inconvertibleErrorCode());
665 DSOHandles.erase(Val: &JD);
666 InitializedDylib.erase(Ptr: &JD);
667 } else
668 return WrapperAddr.takeError();
669 return Error::success();
670}
671
672void LLJIT::PlatformSupport::setInitTransform(
673 LLJIT &J, IRTransformLayer::TransformFunction T) {
674 J.InitHelperTransformLayer->setTransform(std::move(T));
675}
676
677LLJIT::PlatformSupport::~PlatformSupport() = default;
678
679Error LLJITBuilderState::prepareForConstruction() {
680
681 LLVM_DEBUG(dbgs() << "Preparing to create LLJIT instance...\n");
682
683 if (!JTMB) {
684 LLVM_DEBUG({
685 dbgs() << " No explicitly set JITTargetMachineBuilder. "
686 "Detecting host...\n";
687 });
688 if (auto JTMBOrErr = JITTargetMachineBuilder::detectHost())
689 JTMB = std::move(*JTMBOrErr);
690 else
691 return JTMBOrErr.takeError();
692 }
693
694 if ((ES || EPC) && NumCompileThreads)
695 return make_error<StringError>(
696 Args: "NumCompileThreads cannot be used with a custom ExecutionSession or "
697 "ExecutorProcessControl",
698 Args: inconvertibleErrorCode());
699
700#if !LLVM_ENABLE_THREADS
701 if (NumCompileThreads)
702 return make_error<StringError>(
703 "LLJIT num-compile-threads is " + Twine(NumCompileThreads) +
704 " but LLVM was compiled with LLVM_ENABLE_THREADS=Off",
705 inconvertibleErrorCode());
706#endif // !LLVM_ENABLE_THREADS
707
708 // Only used in debug builds.
709 [[maybe_unused]] bool ConcurrentCompilationSettingDefaulted =
710 !SupportConcurrentCompilation;
711
712 if (!SupportConcurrentCompilation) {
713#if LLVM_ENABLE_THREADS
714 SupportConcurrentCompilation = NumCompileThreads || ES || EPC;
715#else
716 SupportConcurrentCompilation = false;
717#endif // LLVM_ENABLE_THREADS
718 } else {
719#if !LLVM_ENABLE_THREADS
720 if (*SupportConcurrentCompilation)
721 return make_error<StringError>(
722 "LLJIT concurrent compilation support requested, but LLVM was built "
723 "with LLVM_ENABLE_THREADS=Off",
724 inconvertibleErrorCode());
725#endif // !LLVM_ENABLE_THREADS
726 }
727
728 LLVM_DEBUG({
729 dbgs() << " JITTargetMachineBuilder is "
730 << JITTargetMachineBuilderPrinter(*JTMB, " ")
731 << " Pre-constructed ExecutionSession: " << (ES ? "Yes" : "No")
732 << "\n"
733 << " DataLayout: ";
734 if (DL)
735 dbgs() << DL->getStringRepresentation() << "\n";
736 else
737 dbgs() << "None (will be created by JITTargetMachineBuilder)\n";
738
739 dbgs() << " Custom object-linking-layer creator: "
740 << (CreateObjectLinkingLayer ? "Yes" : "No") << "\n"
741 << " Custom compile-function creator: "
742 << (CreateCompileFunction ? "Yes" : "No") << "\n"
743 << " Custom platform-setup function: "
744 << (SetUpPlatform ? "Yes" : "No") << "\n"
745 << " Support concurrent compilation: "
746 << (*SupportConcurrentCompilation ? "Yes" : "No");
747 if (ConcurrentCompilationSettingDefaulted)
748 dbgs() << " (defaulted based on ES / EPC / NumCompileThreads)\n";
749 else
750 dbgs() << "\n";
751 dbgs() << " Number of compile threads: " << NumCompileThreads << "\n";
752 });
753
754 // Create DL if not specified.
755 if (!DL) {
756 if (auto DLOrErr = JTMB->getDefaultDataLayoutForTarget())
757 DL = std::move(*DLOrErr);
758 else
759 return DLOrErr.takeError();
760 }
761
762 // If neither ES nor EPC has been set then create an EPC instance.
763 if (!ES && !EPC) {
764 LLVM_DEBUG({
765 dbgs() << "ExecutorProcessControl not specified, "
766 "Creating SelfExecutorProcessControl instance\n";
767 });
768
769 std::unique_ptr<TaskDispatcher> D = nullptr;
770#if LLVM_ENABLE_THREADS
771 if (*SupportConcurrentCompilation) {
772 std::optional<size_t> NumThreads = std ::nullopt;
773 if (NumCompileThreads)
774 NumThreads = NumCompileThreads;
775 D = std::make_unique<DynamicThreadPoolTaskDispatcher>(args&: NumThreads);
776 } else
777 D = std::make_unique<InPlaceTaskDispatcher>();
778#endif // LLVM_ENABLE_THREADS
779 if (auto EPCOrErr =
780 SelfExecutorProcessControl::Create(SSP: nullptr, D: std::move(D), MemMgr: nullptr))
781 EPC = std::move(*EPCOrErr);
782 else
783 return EPCOrErr.takeError();
784 } else if (EPC) {
785 LLVM_DEBUG({
786 dbgs() << "Using explicitly specified ExecutorProcessControl instance "
787 << EPC.get() << "\n";
788 });
789 } else {
790 LLVM_DEBUG({
791 dbgs() << "Using explicitly specified ExecutionSession instance "
792 << ES.get() << "\n";
793 });
794 }
795
796 // If the client didn't configure any linker options then auto-configure the
797 // JIT linker.
798 if (!CreateObjectLinkingLayer) {
799 auto &TT = JTMB->getTargetTriple();
800 bool UseJITLink = false;
801 switch (TT.getArch()) {
802 case Triple::riscv64:
803 case Triple::loongarch64:
804 UseJITLink = true;
805 break;
806 case Triple::aarch64:
807 UseJITLink = !TT.isOSBinFormatCOFF();
808 break;
809 case Triple::arm:
810 case Triple::armeb:
811 case Triple::thumb:
812 case Triple::thumbeb:
813 UseJITLink = TT.isOSBinFormatELF();
814 break;
815 case Triple::x86_64:
816 UseJITLink = !TT.isOSBinFormatCOFF();
817 break;
818 case Triple::ppc64:
819 UseJITLink = TT.isPPC64ELFv2ABI();
820 break;
821 case Triple::ppc64le:
822 UseJITLink = TT.isOSBinFormatELF();
823 break;
824 default:
825 break;
826 }
827 if (UseJITLink) {
828 if (!JTMB->getCodeModel())
829 JTMB->setCodeModel(CodeModel::Small);
830 JTMB->setRelocationModel(Reloc::PIC_);
831 CreateObjectLinkingLayer =
832 [](ExecutionSession &ES) -> Expected<std::unique_ptr<ObjectLayer>> {
833 return std::make_unique<ObjectLinkingLayer>(args&: ES);
834 };
835 }
836 }
837
838 // If we need a process JITDylib but no setup function has been given then
839 // create a default one.
840 if (!SetupProcessSymbolsJITDylib && LinkProcessSymbolsByDefault) {
841 LLVM_DEBUG(dbgs() << "Creating default Process JD setup function\n");
842 SetupProcessSymbolsJITDylib = [](LLJIT &J) -> Expected<JITDylibSP> {
843 auto &JD =
844 J.getExecutionSession().createBareJITDylib(Name: "<Process Symbols>");
845 auto G = EPCDynamicLibrarySearchGenerator::GetForTargetProcess(
846 ES&: J.getExecutionSession());
847 if (!G)
848 return G.takeError();
849 JD.addGenerator(DefGenerator: std::move(*G));
850 return &JD;
851 };
852 }
853
854 return Error::success();
855}
856
857LLJIT::~LLJIT() {
858 if (auto Err = ES->endSession())
859 ES->reportError(Err: std::move(Err));
860}
861
862JITDylibSP LLJIT::getProcessSymbolsJITDylib() { return ProcessSymbols; }
863
864JITDylibSP LLJIT::getPlatformJITDylib() { return Platform; }
865
866Expected<JITDylib &> LLJIT::createJITDylib(std::string Name) {
867 auto JD = ES->createJITDylib(Name: std::move(Name));
868 if (!JD)
869 return JD.takeError();
870
871 JD->addToLinkOrder(NewLinks: DefaultLinks);
872 return JD;
873}
874
875Expected<JITDylib &> LLJIT::loadPlatformDynamicLibrary(const char *Path) {
876 auto G = EPCDynamicLibrarySearchGenerator::Load(ES&: *ES, LibraryPath: Path);
877 if (!G)
878 return G.takeError();
879
880 if (auto *ExistingJD = ES->getJITDylibByName(Name: Path))
881 return *ExistingJD;
882
883 auto &JD = ES->createBareJITDylib(Name: Path);
884 JD.addGenerator(DefGenerator: std::move(*G));
885 return JD;
886}
887
888Error LLJIT::linkStaticLibraryInto(JITDylib &JD,
889 std::unique_ptr<MemoryBuffer> LibBuffer) {
890 auto G = StaticLibraryDefinitionGenerator::Create(L&: *ObjLinkingLayer,
891 ArchiveBuffer: std::move(LibBuffer));
892 if (!G)
893 return G.takeError();
894
895 JD.addGenerator(DefGenerator: std::move(*G));
896
897 return Error::success();
898}
899
900Error LLJIT::linkStaticLibraryInto(JITDylib &JD, const char *Path) {
901 auto G = StaticLibraryDefinitionGenerator::Load(L&: *ObjLinkingLayer, FileName: Path);
902 if (!G)
903 return G.takeError();
904
905 JD.addGenerator(DefGenerator: std::move(*G));
906
907 return Error::success();
908}
909
910Error LLJIT::addIRModule(ResourceTrackerSP RT, ThreadSafeModule TSM) {
911 assert(TSM && "Can not add null module");
912
913 if (auto Err =
914 TSM.withModuleDo(F: [&](Module &M) { return applyDataLayout(M); }))
915 return Err;
916
917 return InitHelperTransformLayer->add(RT: std::move(RT), TSM: std::move(TSM));
918}
919
920Error LLJIT::addIRModule(JITDylib &JD, ThreadSafeModule TSM) {
921 return addIRModule(RT: JD.getDefaultResourceTracker(), TSM: std::move(TSM));
922}
923
924Error LLJIT::addObjectFile(ResourceTrackerSP RT,
925 std::unique_ptr<MemoryBuffer> Obj) {
926 assert(Obj && "Can not add null object");
927
928 return ObjTransformLayer->add(RT: std::move(RT), O: std::move(Obj));
929}
930
931Error LLJIT::addObjectFile(JITDylib &JD, std::unique_ptr<MemoryBuffer> Obj) {
932 return addObjectFile(RT: JD.getDefaultResourceTracker(), Obj: std::move(Obj));
933}
934
935Expected<ExecutorAddr> LLJIT::lookupLinkerMangled(JITDylib &JD,
936 SymbolStringPtr Name) {
937 if (auto Sym = ES->lookup(
938 SearchOrder: makeJITDylibSearchOrder(JDs: &JD, Flags: JITDylibLookupFlags::MatchAllSymbols),
939 Symbol: Name))
940 return Sym->getAddress();
941 else
942 return Sym.takeError();
943}
944
945Expected<std::unique_ptr<ObjectLayer>>
946LLJIT::createObjectLinkingLayer(LLJITBuilderState &S, ExecutionSession &ES) {
947
948 // If the config state provided an ObjectLinkingLayer factory then use it.
949 if (S.CreateObjectLinkingLayer)
950 return S.CreateObjectLinkingLayer(ES);
951
952 // Otherwise default to creating an RTDyldObjectLinkingLayer that constructs
953 // a new SectionMemoryManager for each object.
954 auto GetMemMgr = [](const MemoryBuffer &) {
955 return std::make_unique<SectionMemoryManager>();
956 };
957 auto Layer =
958 std::make_unique<RTDyldObjectLinkingLayer>(args&: ES, args: std::move(GetMemMgr));
959
960 if (S.JTMB->getTargetTriple().isOSBinFormatCOFF()) {
961 Layer->setOverrideObjectFlagsWithResponsibilityFlags(true);
962 Layer->setAutoClaimResponsibilityForObjectSymbols(true);
963 }
964
965 if (S.JTMB->getTargetTriple().isOSBinFormatELF() &&
966 (S.JTMB->getTargetTriple().getArch() == Triple::ArchType::ppc64 ||
967 S.JTMB->getTargetTriple().getArch() == Triple::ArchType::ppc64le))
968 Layer->setAutoClaimResponsibilityForObjectSymbols(true);
969
970 // FIXME: Explicit conversion to std::unique_ptr<ObjectLayer> added to silence
971 // errors from some GCC / libstdc++ bots. Remove this conversion (i.e.
972 // just return ObjLinkingLayer) once those bots are upgraded.
973 return std::unique_ptr<ObjectLayer>(std::move(Layer));
974}
975
976Expected<std::unique_ptr<IRCompileLayer::IRCompiler>>
977LLJIT::createCompileFunction(LLJITBuilderState &S,
978 JITTargetMachineBuilder JTMB) {
979
980 /// If there is a custom compile function creator set then use it.
981 if (S.CreateCompileFunction)
982 return S.CreateCompileFunction(std::move(JTMB));
983
984 // If using a custom EPC then use a ConcurrentIRCompiler by default.
985 if (*S.SupportConcurrentCompilation)
986 return std::make_unique<ConcurrentIRCompiler>(args: std::move(JTMB));
987
988 auto TM = JTMB.createTargetMachine();
989 if (!TM)
990 return TM.takeError();
991
992 return std::make_unique<TMOwningSimpleCompiler>(args: std::move(*TM));
993}
994
995LLJIT::LLJIT(LLJITBuilderState &S, Error &Err)
996 : DL(std::move(*S.DL)), TT(S.JTMB->getTargetTriple()) {
997
998 ErrorAsOutParameter _(Err);
999
1000 assert(!(S.EPC && S.ES) && "EPC and ES should not both be set");
1001
1002 if (S.EPC) {
1003 ES = std::make_unique<ExecutionSession>(args: std::move(S.EPC));
1004 } else if (S.ES)
1005 ES = std::move(S.ES);
1006 else {
1007 if (auto EPC = SelfExecutorProcessControl::Create()) {
1008 ES = std::make_unique<ExecutionSession>(args: std::move(*EPC));
1009 } else {
1010 Err = EPC.takeError();
1011 return;
1012 }
1013 }
1014
1015 auto ObjLayer = createObjectLinkingLayer(S, ES&: *ES);
1016 if (!ObjLayer) {
1017 Err = ObjLayer.takeError();
1018 return;
1019 }
1020 ObjLinkingLayer = std::move(*ObjLayer);
1021 ObjTransformLayer =
1022 std::make_unique<ObjectTransformLayer>(args&: *ES, args&: *ObjLinkingLayer);
1023
1024 {
1025 auto CompileFunction = createCompileFunction(S, JTMB: std::move(*S.JTMB));
1026 if (!CompileFunction) {
1027 Err = CompileFunction.takeError();
1028 return;
1029 }
1030 CompileLayer = std::make_unique<IRCompileLayer>(
1031 args&: *ES, args&: *ObjTransformLayer, args: std::move(*CompileFunction));
1032 TransformLayer = std::make_unique<IRTransformLayer>(args&: *ES, args&: *CompileLayer);
1033 InitHelperTransformLayer =
1034 std::make_unique<IRTransformLayer>(args&: *ES, args&: *TransformLayer);
1035 }
1036
1037 if (*S.SupportConcurrentCompilation)
1038 InitHelperTransformLayer->setCloneToNewContextOnEmit(true);
1039
1040 if (S.SetupProcessSymbolsJITDylib) {
1041 if (auto ProcSymsJD = S.SetupProcessSymbolsJITDylib(*this)) {
1042 ProcessSymbols = ProcSymsJD->get();
1043 } else {
1044 Err = ProcSymsJD.takeError();
1045 return;
1046 }
1047 }
1048
1049 if (S.PrePlatformSetup)
1050 if ((Err = S.PrePlatformSetup(*this)))
1051 return;
1052
1053 if (!S.SetUpPlatform)
1054 S.SetUpPlatform = setUpGenericLLVMIRPlatform;
1055
1056 if (auto PlatformJDOrErr = S.SetUpPlatform(*this)) {
1057 Platform = PlatformJDOrErr->get();
1058 if (Platform)
1059 DefaultLinks.push_back(
1060 x: {Platform, JITDylibLookupFlags::MatchExportedSymbolsOnly});
1061 } else {
1062 Err = PlatformJDOrErr.takeError();
1063 return;
1064 }
1065
1066 if (S.LinkProcessSymbolsByDefault)
1067 DefaultLinks.push_back(
1068 x: {ProcessSymbols, JITDylibLookupFlags::MatchExportedSymbolsOnly});
1069
1070 if (auto MainOrErr = createJITDylib(Name: "main"))
1071 Main = &*MainOrErr;
1072 else {
1073 Err = MainOrErr.takeError();
1074 return;
1075 }
1076}
1077
1078std::string LLJIT::mangle(StringRef UnmangledName) const {
1079 std::string MangledName;
1080 {
1081 raw_string_ostream MangledNameStream(MangledName);
1082 Mangler::getNameWithPrefix(OS&: MangledNameStream, GVName: UnmangledName, DL);
1083 }
1084 return MangledName;
1085}
1086
1087Error LLJIT::applyDataLayout(Module &M) {
1088 if (M.getDataLayout().isDefault())
1089 M.setDataLayout(DL);
1090
1091 if (M.getDataLayout() != DL)
1092 return make_error<StringError>(
1093 Args: "Added modules have incompatible data layouts: " +
1094 M.getDataLayout().getStringRepresentation() + " (module) vs " +
1095 DL.getStringRepresentation() + " (jit)",
1096 Args: inconvertibleErrorCode());
1097
1098 return Error::success();
1099}
1100
1101Error setUpOrcPlatformManually(LLJIT &J) {
1102 LLVM_DEBUG({ dbgs() << "Setting up orc platform support for LLJIT\n"; });
1103 J.setPlatformSupport(std::make_unique<ORCPlatformSupport>(args&: J));
1104 return Error::success();
1105}
1106
1107class LoadAndLinkDynLibrary {
1108public:
1109 LoadAndLinkDynLibrary(LLJIT &J) : J(J) {}
1110 Error operator()(JITDylib &JD, StringRef DLLName) {
1111 if (!DLLName.ends_with_insensitive(Suffix: ".dll"))
1112 return make_error<StringError>(Args: "DLLName not ending with .dll",
1113 Args: inconvertibleErrorCode());
1114 auto DLLNameStr = DLLName.str(); // Guarantees null-termination.
1115 auto DLLJD = J.loadPlatformDynamicLibrary(Path: DLLNameStr.c_str());
1116 if (!DLLJD)
1117 return DLLJD.takeError();
1118 JD.addToLinkOrder(JD&: *DLLJD);
1119 return Error::success();
1120 }
1121
1122private:
1123 LLJIT &J;
1124};
1125
1126Expected<JITDylibSP> ExecutorNativePlatform::operator()(LLJIT &J) {
1127 auto ProcessSymbolsJD = J.getProcessSymbolsJITDylib();
1128 if (!ProcessSymbolsJD)
1129 return make_error<StringError>(
1130 Args: "Native platforms require a process symbols JITDylib",
1131 Args: inconvertibleErrorCode());
1132
1133 const Triple &TT = J.getTargetTriple();
1134 ObjectLinkingLayer *ObjLinkingLayer =
1135 dyn_cast<ObjectLinkingLayer>(Val: &J.getObjLinkingLayer());
1136
1137 if (!ObjLinkingLayer)
1138 return make_error<StringError>(
1139 Args: "ExecutorNativePlatform requires ObjectLinkingLayer",
1140 Args: inconvertibleErrorCode());
1141
1142 std::unique_ptr<MemoryBuffer> RuntimeArchiveBuffer;
1143 if (OrcRuntime.index() == 0) {
1144 auto A = errorOrToExpected(EO: MemoryBuffer::getFile(Filename: std::get<0>(v&: OrcRuntime)));
1145 if (!A)
1146 return A.takeError();
1147 RuntimeArchiveBuffer = std::move(*A);
1148 } else
1149 RuntimeArchiveBuffer = std::move(std::get<1>(v&: OrcRuntime));
1150
1151 auto &ES = J.getExecutionSession();
1152 auto &PlatformJD = ES.createBareJITDylib(Name: "<Platform>");
1153 PlatformJD.addToLinkOrder(JD&: *ProcessSymbolsJD);
1154
1155 J.setPlatformSupport(std::make_unique<ORCPlatformSupport>(args&: J));
1156
1157 switch (TT.getObjectFormat()) {
1158 case Triple::COFF: {
1159 const char *VCRuntimePath = nullptr;
1160 bool StaticVCRuntime = false;
1161 if (VCRuntime) {
1162 VCRuntimePath = VCRuntime->first.c_str();
1163 StaticVCRuntime = VCRuntime->second;
1164 }
1165 if (auto P = COFFPlatform::Create(
1166 ObjLinkingLayer&: *ObjLinkingLayer, PlatformJD, OrcRuntimeArchiveBuffer: std::move(RuntimeArchiveBuffer),
1167 LoadDynLibrary: LoadAndLinkDynLibrary(J), StaticVCRuntime, VCRuntimePath))
1168 J.getExecutionSession().setPlatform(std::move(*P));
1169 else
1170 return P.takeError();
1171 break;
1172 }
1173 case Triple::ELF: {
1174 auto G = StaticLibraryDefinitionGenerator::Create(
1175 L&: *ObjLinkingLayer, ArchiveBuffer: std::move(RuntimeArchiveBuffer));
1176 if (!G)
1177 return G.takeError();
1178
1179 if (auto P =
1180 ELFNixPlatform::Create(ObjLinkingLayer&: *ObjLinkingLayer, PlatformJD, OrcRuntime: std::move(*G)))
1181 J.getExecutionSession().setPlatform(std::move(*P));
1182 else
1183 return P.takeError();
1184 break;
1185 }
1186 case Triple::MachO: {
1187 auto G = StaticLibraryDefinitionGenerator::Create(
1188 L&: *ObjLinkingLayer, ArchiveBuffer: std::move(RuntimeArchiveBuffer));
1189 if (!G)
1190 return G.takeError();
1191
1192 if (auto P =
1193 MachOPlatform::Create(ObjLinkingLayer&: *ObjLinkingLayer, PlatformJD, OrcRuntime: std::move(*G)))
1194 ES.setPlatform(std::move(*P));
1195 else
1196 return P.takeError();
1197 break;
1198 }
1199 default:
1200 return make_error<StringError>(Args: "Unsupported object format in triple " +
1201 TT.str(),
1202 Args: inconvertibleErrorCode());
1203 }
1204
1205 return &PlatformJD;
1206}
1207
1208Expected<JITDylibSP> setUpGenericLLVMIRPlatform(LLJIT &J) {
1209 LLVM_DEBUG(
1210 { dbgs() << "Setting up GenericLLVMIRPlatform support for LLJIT\n"; });
1211 auto ProcessSymbolsJD = J.getProcessSymbolsJITDylib();
1212 if (!ProcessSymbolsJD)
1213 return make_error<StringError>(
1214 Args: "Native platforms require a process symbols JITDylib",
1215 Args: inconvertibleErrorCode());
1216
1217 auto &PlatformJD = J.getExecutionSession().createBareJITDylib(Name: "<Platform>");
1218 PlatformJD.addToLinkOrder(JD&: *ProcessSymbolsJD);
1219
1220 if (auto *OLL = dyn_cast<ObjectLinkingLayer>(Val: &J.getObjLinkingLayer())) {
1221
1222 bool UseEHFrames = true;
1223
1224 // Enable compact-unwind support if possible.
1225 if (J.getTargetTriple().isOSDarwin() ||
1226 J.getTargetTriple().isOSBinFormatMachO()) {
1227
1228 // Check if the bootstrap map says that we should force eh-frames:
1229 // Older libunwinds require this as they don't have a dynamic
1230 // registration API for compact-unwind.
1231 std::optional<bool> ForceEHFrames;
1232 if (auto Err = J.getExecutionSession().getBootstrapMapValue<bool, bool>(
1233 Key: "darwin-use-ehframes-only", Val&: ForceEHFrames))
1234 return Err;
1235 if (ForceEHFrames.has_value())
1236 UseEHFrames = *ForceEHFrames;
1237 else
1238 UseEHFrames = false;
1239
1240 // If UseEHFrames hasn't been set then we're good to use compact-unwind.
1241 if (!UseEHFrames) {
1242 if (auto UIRP =
1243 UnwindInfoRegistrationPlugin::Create(ES&: J.getExecutionSession())) {
1244 OLL->addPlugin(P: std::move(*UIRP));
1245 LLVM_DEBUG(dbgs() << "Enabled compact-unwind support.\n");
1246 } else
1247 return UIRP.takeError();
1248 }
1249 }
1250
1251 // Otherwise fall back to standard unwind registration.
1252 if (UseEHFrames) {
1253 auto &ES = J.getExecutionSession();
1254 if (auto EHFP = EHFrameRegistrationPlugin::Create(ES)) {
1255 OLL->addPlugin(P: std::move(*EHFP));
1256 LLVM_DEBUG(dbgs() << "Enabled eh-frame support.\n");
1257 } else
1258 return EHFP.takeError();
1259 }
1260 }
1261
1262 J.setPlatformSupport(
1263 std::make_unique<GenericLLVMIRPlatformSupport>(args&: J, args&: PlatformJD));
1264
1265 return &PlatformJD;
1266}
1267
1268Expected<JITDylibSP> setUpInactivePlatform(LLJIT &J) {
1269 LLVM_DEBUG(
1270 { dbgs() << "Explicitly deactivated platform support for LLJIT\n"; });
1271 J.setPlatformSupport(std::make_unique<InactivePlatformSupport>());
1272 return nullptr;
1273}
1274
1275Error LLLazyJITBuilderState::prepareForConstruction() {
1276 if (auto Err = LLJITBuilderState::prepareForConstruction())
1277 return Err;
1278 TT = JTMB->getTargetTriple();
1279 return Error::success();
1280}
1281
1282Error LLLazyJIT::addLazyIRModule(JITDylib &JD, ThreadSafeModule TSM) {
1283 assert(TSM && "Can not add null module");
1284
1285 if (auto Err = TSM.withModuleDo(
1286 F: [&](Module &M) -> Error { return applyDataLayout(M); }))
1287 return Err;
1288
1289 return CODLayer->add(JD, TSM: std::move(TSM));
1290}
1291
1292LLLazyJIT::LLLazyJIT(LLLazyJITBuilderState &S, Error &Err) : LLJIT(S, Err) {
1293
1294 // If LLJIT construction failed then bail out.
1295 if (Err)
1296 return;
1297
1298 ErrorAsOutParameter _(&Err);
1299
1300 /// Take/Create the lazy-compile callthrough manager.
1301 if (S.LCTMgr)
1302 LCTMgr = std::move(S.LCTMgr);
1303 else {
1304 if (auto LCTMgrOrErr = createLocalLazyCallThroughManager(
1305 T: S.TT, ES&: *ES, ErrorHandlerAddr: S.LazyCompileFailureAddr))
1306 LCTMgr = std::move(*LCTMgrOrErr);
1307 else {
1308 Err = LCTMgrOrErr.takeError();
1309 return;
1310 }
1311 }
1312
1313 // Take/Create the indirect stubs manager builder.
1314 auto ISMBuilder = std::move(S.ISMBuilder);
1315
1316 // If none was provided, try to build one.
1317 if (!ISMBuilder)
1318 ISMBuilder = createLocalIndirectStubsManagerBuilder(T: S.TT);
1319
1320 // No luck. Bail out.
1321 if (!ISMBuilder) {
1322 Err = make_error<StringError>(Args: "Could not construct "
1323 "IndirectStubsManagerBuilder for target " +
1324 S.TT.str(),
1325 Args: inconvertibleErrorCode());
1326 return;
1327 }
1328
1329 // Create the IP Layer.
1330 IPLayer = std::make_unique<IRPartitionLayer>(args&: *ES, args&: *InitHelperTransformLayer);
1331
1332 // Create the COD layer.
1333 CODLayer = std::make_unique<CompileOnDemandLayer>(args&: *ES, args&: *IPLayer, args&: *LCTMgr,
1334 args: std::move(ISMBuilder));
1335
1336 if (*S.SupportConcurrentCompilation)
1337 CODLayer->setCloneToNewContextOnEmit(true);
1338}
1339
1340// In-process LLJIT uses eh-frame section wrappers via EPC, so we need to force
1341// them to be linked in.
1342LLVM_ATTRIBUTE_USED void linkComponents() {
1343 errs() << (void *)&llvm_orc_registerEHFrameSectionAllocAction
1344 << (void *)&llvm_orc_deregisterEHFrameSectionAllocAction;
1345}
1346
1347} // End namespace orc.
1348} // End namespace llvm.
1349