1//===- ExecutionEngine.cpp - MLIR Execution engine and utils --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the execution engine for MLIR modules based on LLVM Orc
10// JIT engine.
11//
12//===----------------------------------------------------------------------===//
13#include "mlir/ExecutionEngine/ExecutionEngine.h"
14#include "mlir/Dialect/LLVMIR/LLVMModuleOps.h"
15#include "mlir/IR/BuiltinOps.h"
16#include "mlir/Support/FileUtilities.h"
17#include "mlir/Target/LLVMIR/Export.h"
18
19#include "llvm/ExecutionEngine/JITEventListener.h"
20#include "llvm/ExecutionEngine/ObjectCache.h"
21#include "llvm/ExecutionEngine/Orc/CompileUtils.h"
22#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
23#include "llvm/ExecutionEngine/Orc/IRCompileLayer.h"
24#include "llvm/ExecutionEngine/Orc/IRTransformLayer.h"
25#include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h"
26#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
27#include "llvm/IR/IRBuilder.h"
28#include "llvm/MC/TargetRegistry.h"
29#include "llvm/Support/Debug.h"
30#include "llvm/Support/Error.h"
31#include "llvm/Support/LibraryOptionsContext.h"
32#include "llvm/Support/ToolOutputFile.h"
33#include "llvm/TargetParser/Host.h"
34#include "llvm/TargetParser/SubtargetFeature.h"
35
36#define DEBUG_TYPE "execution-engine"
37
38using namespace mlir;
39using llvm::dbgs;
40using llvm::Error;
41using llvm::errs;
42using llvm::Expected;
43using llvm::LLVMContext;
44using llvm::MemoryBuffer;
45using llvm::MemoryBufferRef;
46using llvm::Module;
47using llvm::SectionMemoryManager;
48using llvm::StringError;
49using llvm::Triple;
50using llvm::orc::DynamicLibrarySearchGenerator;
51using llvm::orc::ExecutionSession;
52using llvm::orc::IRCompileLayer;
53using llvm::orc::JITTargetMachineBuilder;
54using llvm::orc::MangleAndInterner;
55using llvm::orc::RTDyldObjectLinkingLayer;
56using llvm::orc::SymbolMap;
57using llvm::orc::ThreadSafeModule;
58using llvm::orc::TMOwningSimpleCompiler;
59
60/// Wrap a string into an llvm::StringError.
61static Error makeStringError(const Twine &message) {
62 return llvm::make_error<StringError>(Args: message.str(),
63 Args: llvm::inconvertibleErrorCode());
64}
65
66void SimpleObjectCache::notifyObjectCompiled(const Module *m,
67 MemoryBufferRef objBuffer) {
68 cachedObjects[m->getModuleIdentifier()] = MemoryBuffer::getMemBufferCopy(
69 InputData: objBuffer.getBuffer(), BufferName: objBuffer.getBufferIdentifier());
70}
71
72std::unique_ptr<MemoryBuffer> SimpleObjectCache::getObject(const Module *m) {
73 auto i = cachedObjects.find(Key: m->getModuleIdentifier());
74 if (i == cachedObjects.end()) {
75 LLVM_DEBUG(dbgs() << "No object for " << m->getModuleIdentifier()
76 << " in cache. Compiling.\n");
77 return nullptr;
78 }
79 LLVM_DEBUG(dbgs() << "Object for " << m->getModuleIdentifier()
80 << " loaded from cache.\n");
81 return MemoryBuffer::getMemBuffer(Ref: i->second->getMemBufferRef());
82}
83
84void SimpleObjectCache::dumpToObjectFile(StringRef outputFilename) {
85 // Set up the output file.
86 std::string errorMessage;
87 auto file = openOutputFile(outputFilename, errorMessage: &errorMessage);
88 if (!file) {
89 llvm::errs() << errorMessage << "\n";
90 return;
91 }
92
93 // Dump the object generated for a single module to the output file.
94 assert(cachedObjects.size() == 1 && "Expected only one object entry.");
95 auto &cachedObject = cachedObjects.begin()->second;
96 file->os() << cachedObject->getBuffer();
97 file->keep();
98}
99
100bool SimpleObjectCache::isEmpty() { return cachedObjects.empty(); }
101
102void ExecutionEngine::dumpToObjectFile(StringRef filename) {
103 if (cache == nullptr) {
104 llvm::errs() << "cannot dump ExecutionEngine object code to file: "
105 "object cache is disabled\n";
106 return;
107 }
108 // Compilation is lazy and it doesn't populate object cache unless requested.
109 // In case object dump is requested before cache is populated, we need to
110 // force compilation manually.
111 if (cache->isEmpty()) {
112 for (std::string &functionName : functionNames) {
113 auto result = lookupPacked(name: functionName);
114 if (!result) {
115 llvm::errs() << "Could not compile " << functionName << ":\n "
116 << result.takeError() << "\n";
117 return;
118 }
119 }
120 }
121 cache->dumpToObjectFile(outputFilename: filename);
122}
123
124void ExecutionEngine::registerSymbols(
125 llvm::function_ref<SymbolMap(MangleAndInterner)> symbolMap) {
126 auto &mainJitDylib = jit->getMainJITDylib();
127 cantFail(Err: mainJitDylib.define(
128 MU: absoluteSymbols(Symbols: symbolMap(llvm::orc::MangleAndInterner(
129 mainJitDylib.getExecutionSession(), jit->getDataLayout())))));
130}
131
132void ExecutionEngine::setupTargetTripleAndDataLayout(Module *llvmModule,
133 llvm::TargetMachine *tm) {
134 llvmModule->setDataLayout(tm->getTargetTriple().computeDataLayout());
135 llvmModule->setTargetTriple(tm->getTargetTriple());
136}
137
138static std::string makePackedFunctionName(StringRef name) {
139 return "_mlir_" + name.str();
140}
141
142// For each function in the LLVM module, define an interface function that wraps
143// all the arguments of the original function and all its results into an i8**
144// pointer to provide a unified invocation interface.
145static void packFunctionArguments(Module *module) {
146 auto &ctx = module->getContext();
147 llvm::IRBuilder<> builder(*module);
148 DenseSet<llvm::Function *> interfaceFunctions;
149 for (auto &func : module->getFunctionList()) {
150 if (func.isDeclaration() || func.hasLocalLinkage())
151 continue;
152 if (interfaceFunctions.count(V: &func))
153 continue;
154
155 // Given a function `foo(<...>)`, define the interface function
156 // `mlir_foo(i8**)`.
157 auto *newType =
158 llvm::FunctionType::get(Result: builder.getVoidTy(), Params: builder.getPtrTy(),
159 /*isVarArg=*/false);
160 auto newName = makePackedFunctionName(name: func.getName());
161 auto funcCst = module->getOrInsertFunction(Name: newName, T: newType);
162 llvm::Function *interfaceFunc = cast<llvm::Function>(Val: funcCst.getCallee());
163 interfaceFunctions.insert(V: interfaceFunc);
164
165 // Extract the arguments from the type-erased argument list and cast them to
166 // the proper types.
167 auto *bb = llvm::BasicBlock::Create(Context&: ctx);
168 bb->insertInto(Parent: interfaceFunc);
169 builder.SetInsertPoint(bb);
170 llvm::Value *argList = interfaceFunc->arg_begin();
171 SmallVector<llvm::Value *, 8> args;
172 args.reserve(N: llvm::size(Range: func.args()));
173 for (auto [index, arg] : llvm::enumerate(First: func.args())) {
174 llvm::Value *argIndex = llvm::Constant::getIntegerValue(
175 Ty: builder.getInt64Ty(), V: APInt(64, index));
176 llvm::Value *argPtrPtr =
177 builder.CreateGEP(Ty: builder.getPtrTy(), Ptr: argList, IdxList: argIndex);
178 llvm::Value *argPtr = builder.CreateLoad(Ty: builder.getPtrTy(), Ptr: argPtrPtr);
179 llvm::Type *argTy = arg.getType();
180 llvm::Value *load = builder.CreateLoad(Ty: argTy, Ptr: argPtr);
181 args.push_back(Elt: load);
182 }
183
184 // Call the implementation function with the extracted arguments.
185 llvm::Value *result = builder.CreateCall(Callee: &func, Args: args);
186
187 // Assuming the result is one value, potentially of type `void`.
188 if (!result->getType()->isVoidTy()) {
189 llvm::Value *retIndex = llvm::Constant::getIntegerValue(
190 Ty: builder.getInt64Ty(), V: APInt(64, llvm::size(Range: func.args())));
191 llvm::Value *retPtrPtr =
192 builder.CreateGEP(Ty: builder.getPtrTy(), Ptr: argList, IdxList: retIndex);
193 llvm::Value *retPtr = builder.CreateLoad(Ty: builder.getPtrTy(), Ptr: retPtrPtr);
194 builder.CreateStore(Val: result, Ptr: retPtr);
195 }
196
197 // The interface function returns void.
198 builder.CreateRetVoid();
199 }
200}
201
202ExecutionEngine::ExecutionEngine(bool enableObjectDump,
203 bool enableGDBNotificationListener,
204 bool enablePerfNotificationListener)
205 : cache(enableObjectDump ? new SimpleObjectCache() : nullptr),
206 functionNames(),
207 gdbListener(enableGDBNotificationListener
208 ? llvm::JITEventListener::createGDBRegistrationListener()
209 : nullptr),
210 perfListener(nullptr) {
211 if (enablePerfNotificationListener) {
212 if (auto *listener = llvm::JITEventListener::createPerfJITEventListener())
213 perfListener = listener;
214 else if (auto *listener =
215 llvm::JITEventListener::createIntelJITEventListener())
216 perfListener = listener;
217 }
218}
219
220ExecutionEngine::~ExecutionEngine() {
221 // Execute the global destructors from the module being processed.
222 if (jit)
223 llvm::consumeError(Err: jit->deinitialize(JD&: jit->getMainJITDylib()));
224 // Run all dynamic library destroy callbacks to prepare for the shutdown.
225 for (LibraryDestroyFn destroy : destroyFns)
226 destroy();
227}
228
229Expected<std::unique_ptr<ExecutionEngine>>
230ExecutionEngine::create(Operation *m, const ExecutionEngineOptions &options,
231 std::unique_ptr<llvm::TargetMachine> tm) {
232 auto engine = std::make_unique<ExecutionEngine>(
233 args: options.enableObjectDump, args: options.enableGDBNotificationListener,
234 args: options.enablePerfNotificationListener);
235
236 // Remember all entry-points if object dumping is enabled.
237 if (options.enableObjectDump) {
238 for (auto funcOp : m->getRegion(index: 0).getOps<LLVM::LLVMFuncOp>()) {
239 if (funcOp.getBlocks().empty())
240 continue;
241 StringRef funcName = funcOp.getSymName();
242 engine->functionNames.push_back(x: funcName.str());
243 }
244 }
245
246 std::unique_ptr<llvm::LLVMContext> ctx(new llvm::LLVMContext);
247 if (options.libraryOptions)
248 ctx->getLibraryOptions().assign(Other: *options.libraryOptions);
249 auto llvmModule = options.llvmModuleBuilder
250 ? options.llvmModuleBuilder(m, *ctx)
251 : translateModuleToLLVMIR(module: m, llvmContext&: *ctx);
252 if (!llvmModule)
253 return makeStringError(message: "could not convert to LLVM IR");
254
255 // If no valid TargetMachine was passed, create a default TM ignoring any
256 // input arguments from the user.
257 if (!tm) {
258 auto tmBuilderOrError = llvm::orc::JITTargetMachineBuilder::detectHost();
259 if (!tmBuilderOrError)
260 return tmBuilderOrError.takeError();
261
262 tmBuilderOrError->getOptions().MCOptions.LibraryOptions =
263 options.libraryOptions;
264 auto tmOrError = tmBuilderOrError->createTargetMachine();
265 if (!tmOrError)
266 return tmOrError.takeError();
267 tm = std::move(tmOrError.get());
268 }
269
270 // TODO: Currently, the LLVM module created above has no triple associated
271 // with it. Instead, the triple is extracted from the TargetMachine, which is
272 // either based on the host defaults or command line arguments when specified
273 // (set-up by callers of this method). It could also be passed to the
274 // translation or dialect conversion instead of this.
275 setupTargetTripleAndDataLayout(llvmModule: llvmModule.get(), tm: tm.get());
276 packFunctionArguments(module: llvmModule.get());
277
278 auto dataLayout = llvmModule->getDataLayout();
279
280 // Use absolute library path so that gdb can find the symbol table.
281 SmallVector<SmallString<256>, 4> sharedLibPaths;
282 transform(
283 Range: options.sharedLibPaths, d_first: std::back_inserter(x&: sharedLibPaths),
284 F: [](StringRef libPath) {
285 SmallString<256> absPath(libPath.begin(), libPath.end());
286 cantFail(Err: llvm::errorCodeToError(EC: llvm::sys::fs::make_absolute(path&: absPath)));
287 return absPath;
288 });
289
290 // If shared library implements custom execution layer library init and
291 // destroy functions, we'll use them to register the library. Otherwise, load
292 // the library as JITDyLib below.
293 llvm::StringMap<void *> exportSymbols;
294 SmallVector<LibraryDestroyFn> destroyFns;
295 SmallVector<StringRef> jitDyLibPaths;
296
297 for (auto &libPath : sharedLibPaths) {
298 auto lib = llvm::sys::DynamicLibrary::getPermanentLibrary(
299 filename: libPath.str().str().c_str());
300 void *initSym = lib.getAddressOfSymbol(symbolName: kLibraryInitFnName);
301 void *destroySim = lib.getAddressOfSymbol(symbolName: kLibraryDestroyFnName);
302
303 // Library does not provide call backs, rely on symbol visiblity.
304 if (!initSym || !destroySim) {
305 jitDyLibPaths.push_back(Elt: libPath);
306 continue;
307 }
308
309 auto initFn = reinterpret_cast<LibraryInitFn>(initSym);
310 initFn(exportSymbols);
311
312 auto destroyFn = reinterpret_cast<LibraryDestroyFn>(destroySim);
313 destroyFns.push_back(Elt: destroyFn);
314 }
315 engine->destroyFns = std::move(destroyFns);
316
317 // Callback to create the object layer with symbol resolution to current
318 // process and dynamically linked libraries.
319 auto objectLinkingLayerCreator = [&](ExecutionSession &session,
320 llvm::jitlink::JITLinkMemoryManager
321 &IgnoredMemMgr) {
322 // Needed to respect AArch64 ABI requirements on the distance between
323 // TEXT and GOT sections.
324 bool reserveAlloc = llvmModule->getTargetTriple().isAArch64();
325 auto objectLayer = std::make_unique<RTDyldObjectLinkingLayer>(
326 args&: session, args: [sectionMemoryMapper = options.sectionMemoryMapper,
327 reserveAlloc](const MemoryBuffer &) {
328 return std::make_unique<SectionMemoryManager>(args: sectionMemoryMapper,
329 args: reserveAlloc);
330 });
331
332 // Register JIT event listeners if they are enabled.
333 if (engine->gdbListener)
334 objectLayer->registerJITEventListener(L&: *engine->gdbListener);
335 if (engine->perfListener)
336 objectLayer->registerJITEventListener(L&: *engine->perfListener);
337
338 // COFF format binaries (Windows) need special handling to deal with
339 // exported symbol visibility.
340 // cf llvm/lib/ExecutionEngine/Orc/LLJIT.cpp LLJIT::createObjectLinkingLayer
341 const llvm::Triple &targetTriple = llvmModule->getTargetTriple();
342 if (targetTriple.isOSBinFormatCOFF()) {
343 objectLayer->setOverrideObjectFlagsWithResponsibilityFlags(true);
344 objectLayer->setAutoClaimResponsibilityForObjectSymbols(true);
345 }
346
347 // Resolve symbols from shared libraries.
348 for (auto &libPath : jitDyLibPaths) {
349 auto mb = llvm::MemoryBuffer::getFile(Filename: libPath);
350 if (!mb) {
351 errs() << "Failed to create MemoryBuffer for: " << libPath
352 << "\nError: " << mb.getError().message() << "\n";
353 continue;
354 }
355 auto &jd = session.createBareJITDylib(Name: std::string(libPath));
356 auto loaded = DynamicLibrarySearchGenerator::Load(
357 FileName: libPath.str().c_str(), GlobalPrefix: dataLayout.getGlobalPrefix());
358 if (!loaded) {
359 errs() << "Could not load " << libPath << ":\n " << loaded.takeError()
360 << "\n";
361 continue;
362 }
363 jd.addGenerator(DefGenerator: std::move(*loaded));
364 cantFail(Err: objectLayer->add(JD&: jd, O: std::move(mb.get())));
365 }
366
367 return objectLayer;
368 };
369
370 // Callback to inspect the cache and recompile on demand. This follows Lang's
371 // LLJITWithObjectCache example.
372 auto compileFunctionCreator = [&](JITTargetMachineBuilder jtmb)
373 -> Expected<std::unique_ptr<IRCompileLayer::IRCompiler>> {
374 if (options.jitCodeGenOptLevel)
375 jtmb.setCodeGenOptLevel(*options.jitCodeGenOptLevel);
376 return std::make_unique<TMOwningSimpleCompiler>(args: std::move(tm),
377 args: engine->cache.get());
378 };
379
380 // Create the LLJIT by calling the LLJITBuilder with 2 callbacks.
381 auto jit =
382 cantFail(ValOrErr: llvm::orc::LLJITBuilder()
383 .setCompileFunctionCreator(compileFunctionCreator)
384 .setObjectLinkingLayerCreator(objectLinkingLayerCreator)
385 .setDataLayout(dataLayout)
386 .create());
387
388 // Add a ThreadSafemodule to the engine and return.
389 ThreadSafeModule tsm(std::move(llvmModule), std::move(ctx));
390 if (options.transformer)
391 cantFail(Err: tsm.withModuleDo(
392 F: [&](llvm::Module &module) { return options.transformer(&module); }));
393 cantFail(Err: jit->addIRModule(TSM: std::move(tsm)));
394 engine->jit = std::move(jit);
395
396 // Resolve symbols that are statically linked in the current process.
397 llvm::orc::JITDylib &mainJD = engine->jit->getMainJITDylib();
398 mainJD.addGenerator(
399 DefGenerator: cantFail(ValOrErr: DynamicLibrarySearchGenerator::GetForCurrentProcess(
400 GlobalPrefix: dataLayout.getGlobalPrefix())));
401
402 // Build a runtime symbol map from the exported symbols and register them.
403 auto runtimeSymbolMap = [&](llvm::orc::MangleAndInterner interner) {
404 auto symbolMap = llvm::orc::SymbolMap();
405 for (auto &exportSymbol : exportSymbols)
406 symbolMap[interner(exportSymbol.getKey())] = {
407 llvm::orc::ExecutorAddr::fromPtr(Ptr: exportSymbol.getValue()),
408 llvm::JITSymbolFlags::Exported};
409 return symbolMap;
410 };
411 engine->registerSymbols(symbolMap: runtimeSymbolMap);
412 return std::move(engine);
413}
414
415Expected<void (*)(void **)>
416ExecutionEngine::lookupPacked(StringRef name) const {
417 auto result = lookup(name: makePackedFunctionName(name));
418 if (!result)
419 return result.takeError();
420 return reinterpret_cast<void (*)(void **)>(result.get());
421}
422
423Expected<void *> ExecutionEngine::lookup(StringRef name) const {
424 auto expectedSymbol = jit->lookup(UnmangledName: name);
425
426 // JIT lookup may return an Error referring to strings stored internally by
427 // the JIT. If the Error outlives the ExecutionEngine, it would want have a
428 // dangling reference, which is currently caught by an assertion inside JIT
429 // thanks to hand-rolled reference counting. Rewrap the error message into a
430 // string before returning. Alternatively, ORC JIT should consider copying
431 // the string into the error message.
432 if (!expectedSymbol) {
433 std::string errorMessage;
434 llvm::raw_string_ostream os(errorMessage);
435 llvm::handleAllErrors(E: expectedSymbol.takeError(),
436 Handlers: [&os](llvm::ErrorInfoBase &ei) { ei.log(OS&: os); });
437 return makeStringError(message: errorMessage);
438 }
439
440 if (void *fptr = expectedSymbol->toPtr<void *>())
441 return fptr;
442 return makeStringError(message: "looked up function is null");
443}
444
445Error ExecutionEngine::invokePacked(StringRef name,
446 MutableArrayRef<void *> args) {
447 initialize();
448 auto expectedFPtr = lookupPacked(name);
449 if (!expectedFPtr)
450 return expectedFPtr.takeError();
451 auto fptr = *expectedFPtr;
452
453 (*fptr)(args.data());
454
455 return Error::success();
456}
457
458void ExecutionEngine::initialize() {
459 if (isInitialized)
460 return;
461 cantFail(Err: jit->initialize(JD&: jit->getMainJITDylib()));
462 isInitialized = true;
463}
464