1//===--- tools/clang-repl/ClangRepl.cpp - clang-repl - the Clang REPL -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements a REPL tool on top of clang.
10//
11//===----------------------------------------------------------------------===//
12
13#include "clang/Basic/Diagnostic.h"
14#include "clang/Basic/DiagnosticFrontend.h"
15#include "clang/Basic/Version.h"
16#include "clang/Config/config.h"
17#include "clang/Frontend/CompilerInstance.h"
18#include "clang/Interpreter/CodeCompletion.h"
19#include "clang/Interpreter/Interpreter.h"
20#include "clang/Lex/Preprocessor.h"
21#include "clang/Sema/Sema.h"
22
23#include "llvm/ADT/SmallString.h"
24#include "llvm/ADT/StringRef.h"
25#include "llvm/LineEditor/LineEditor.h"
26#include "llvm/Support/CommandLine.h"
27#include "llvm/Support/FileSystem.h"
28#include "llvm/Support/LibraryOptionsContext.h"
29#include "llvm/Support/ManagedStatic.h" // llvm_shutdown
30#include "llvm/Support/Path.h"
31#include "llvm/Support/Signals.h"
32#include "llvm/Support/TargetSelect.h"
33#include "llvm/Support/VirtualFileSystem.h"
34#include "llvm/Support/raw_ostream.h"
35#include "llvm/TargetParser/Host.h"
36#include "llvm/TargetParser/Triple.h"
37#include <optional>
38
39#include <string>
40#include <vector>
41
42// Disable LSan for this test.
43// FIXME: Re-enable once we can assume GCC 13.2 or higher.
44// https://llvm.org/github.com/llvm/llvm-project/issues/67586.
45#if LLVM_ADDRESS_SANITIZER_BUILD || LLVM_HWADDRESS_SANITIZER_BUILD
46#include <sanitizer/lsan_interface.h>
47LLVM_ATTRIBUTE_USED int __lsan_is_turned_off() { return 1; }
48#endif
49
50#define DEBUG_TYPE "clang-repl"
51
52static llvm::cl::opt<bool> CudaEnabled("cuda", llvm::cl::Hidden);
53static llvm::cl::opt<bool> HipEnabled("hip", llvm::cl::Hidden);
54static llvm::cl::opt<std::string> RocmPath("rocm-path", llvm::cl::Hidden);
55static llvm::cl::opt<std::string> CudaPath("cuda-path", llvm::cl::Hidden);
56static llvm::cl::opt<std::string> OffloadArch("offload-arch", llvm::cl::Hidden);
57static llvm::cl::OptionCategory OOPCategory("Out-of-process Execution Options");
58static llvm::cl::opt<std::string> SlabAllocateSizeString(
59 "slab-allocate",
60 llvm::cl::desc("Allocate from a slab of the given size "
61 "(allowable suffixes: Kb, Mb, Gb. default = "
62 "Kb)"),
63 llvm::cl::init(Val: ""), llvm::cl::cat(OOPCategory));
64static llvm::cl::opt<std::string>
65 OOPExecutor("oop-executor",
66 llvm::cl::desc("Launch an out-of-process executor to run code"),
67 llvm::cl::init(Val: ""), llvm::cl::ValueOptional,
68 llvm::cl::cat(OOPCategory));
69static llvm::cl::opt<std::string> OOPExecutorConnect(
70 "oop-executor-connect",
71 llvm::cl::desc(
72 "Connect to an out-of-process executor through a TCP socket"),
73 llvm::cl::value_desc("<hostname>:<port>"));
74static llvm::cl::opt<std::string>
75 OrcRuntimePath("orc-runtime", llvm::cl::desc("Path to the ORC runtime"),
76 llvm::cl::init(Val: ""), llvm::cl::ValueOptional,
77 llvm::cl::cat(OOPCategory));
78static llvm::cl::opt<bool> UseSharedMemory(
79 "use-shared-memory",
80 llvm::cl::desc("Use shared memory to transfer generated code and data"),
81 llvm::cl::init(Val: false), llvm::cl::cat(OOPCategory));
82static llvm::cl::list<std::string>
83 ClangArgs("Xcc",
84 llvm::cl::desc("Argument to pass to the CompilerInvocation"),
85 llvm::cl::CommaSeparated);
86static llvm::cl::opt<bool> OptHostSupportsJit("host-supports-jit",
87 llvm::cl::Hidden);
88static llvm::cl::opt<bool> OptHostJitTriple("host-jit-triple",
89 llvm::cl::Hidden);
90static llvm::cl::list<std::string> OptInputs(llvm::cl::Positional,
91 llvm::cl::desc("[code to run]"));
92
93static llvm::Error sanitizeOopArguments(const char *ArgV0) {
94 // Only one of -oop-executor and -oop-executor-connect can be used.
95 if (!!OOPExecutor.getNumOccurrences() &&
96 !!OOPExecutorConnect.getNumOccurrences())
97 return llvm::make_error<llvm::StringError>(
98 Args: "Only one of -" + OOPExecutor.ArgStr + " and -" +
99 OOPExecutorConnect.ArgStr + " can be specified",
100 Args: llvm::inconvertibleErrorCode());
101
102 llvm::Triple SystemTriple(llvm::sys::getProcessTriple());
103 // TODO: Remove once out-of-process execution support is implemented for
104 // non-Unix platforms.
105 if ((!SystemTriple.isOSBinFormatELF() &&
106 !SystemTriple.isOSBinFormatMachO()) &&
107 (OOPExecutor.getNumOccurrences() ||
108 OOPExecutorConnect.getNumOccurrences()))
109 return llvm::make_error<llvm::StringError>(
110 Args: SystemTriple.isOSEmscripten()
111 ? "Out-of-process execution is not supported by the WebAssembly "
112 "executor"
113 : "Out-of-process execution is only supported on Unix platforms",
114 Args: llvm::inconvertibleErrorCode());
115
116 // If -slab-allocate is passed, check that we're not trying to use it in
117 // -oop-executor or -oop-executor-connect mode.
118 //
119 // FIXME: Remove once we enable remote slab allocation.
120 if (SlabAllocateSizeString != "") {
121 if (OOPExecutor.getNumOccurrences() ||
122 OOPExecutorConnect.getNumOccurrences())
123 return llvm::make_error<llvm::StringError>(
124 Args: "-slab-allocate cannot be used with -oop-executor or "
125 "-oop-executor-connect",
126 Args: llvm::inconvertibleErrorCode());
127 }
128
129 // Out-of-process executors require the ORC runtime. ORC Runtime Path
130 // resolution is done in Interpreter.cpp.
131
132 // If -oop-executor was used but no value was specified then use a sensible
133 // default.
134 if (!!OOPExecutor.getNumOccurrences() && OOPExecutor.empty()) {
135 llvm::SmallString<256> OOPExecutorPath(llvm::sys::fs::getMainExecutable(
136 argv0: ArgV0, MainExecAddr: reinterpret_cast<void *>(&sanitizeOopArguments)));
137 llvm::sys::path::remove_filename(path&: OOPExecutorPath);
138 llvm::sys::path::append(path&: OOPExecutorPath, a: "llvm-jitlink-executor");
139 OOPExecutor = OOPExecutorPath.str().str();
140 }
141
142 return llvm::Error::success();
143}
144
145static llvm::Expected<unsigned> getSlabAllocSize(llvm::StringRef SizeString) {
146 SizeString = SizeString.trim();
147
148 uint64_t Units = 1024;
149
150 if (SizeString.ends_with_insensitive(Suffix: "kb"))
151 SizeString = SizeString.drop_back(N: 2).rtrim();
152 else if (SizeString.ends_with_insensitive(Suffix: "mb")) {
153 Units = 1024 * 1024;
154 SizeString = SizeString.drop_back(N: 2).rtrim();
155 } else if (SizeString.ends_with_insensitive(Suffix: "gb")) {
156 Units = 1024 * 1024 * 1024;
157 SizeString = SizeString.drop_back(N: 2).rtrim();
158 } else if (SizeString.empty())
159 return 0;
160
161 uint64_t SlabSize = 0;
162 if (SizeString.getAsInteger(Radix: 10, Result&: SlabSize))
163 return llvm::make_error<llvm::StringError>(
164 Args: "Invalid numeric format for slab size", Args: llvm::inconvertibleErrorCode());
165
166 return SlabSize * Units;
167}
168
169static void LLVMErrorHandler(void *UserData, const char *Message,
170 bool GenCrashDiag) {
171 auto &Diags = *static_cast<clang::DiagnosticsEngine *>(UserData);
172
173 Diags.Report(DiagID: clang::diag::err_fe_error_backend) << Message;
174
175 // Run the interrupt handlers to make sure any special cleanups get done, in
176 // particular that we remove files registered with RemoveFileOnSignal.
177 llvm::sys::RunInterruptHandlers();
178
179 // We cannot recover from llvm errors. When reporting a fatal error, exit
180 // with status 70 to generate crash diagnostics. For BSD systems this is
181 // defined as an internal software error. Otherwise, exit with status 1.
182
183 exit(status: GenCrashDiag ? 70 : 1);
184}
185
186// If we are running with -verify a reported has to be returned as unsuccess.
187// This is relevant especially for the test suite.
188static int checkDiagErrors(const clang::CompilerInstance *CI, bool HasError) {
189 unsigned Errs = CI->getDiagnostics().getClient()->getNumErrors();
190 if (CI->getDiagnosticOpts().VerifyDiagnostics) {
191 // If there was an error that came from the verifier we must return 1 as
192 // an exit code for the process. This will make the test fail as expected.
193 clang::DiagnosticConsumer *Client = CI->getDiagnostics().getClient();
194 Client->EndSourceFile();
195 Errs = Client->getNumErrors();
196
197 // The interpreter expects BeginSourceFile/EndSourceFiles to be balanced.
198 Client->BeginSourceFile(LangOpts: CI->getLangOpts(), PP: &CI->getPreprocessor());
199 }
200 return (Errs || HasError) ? EXIT_FAILURE : EXIT_SUCCESS;
201}
202
203struct ReplListCompleter {
204 clang::IncrementalCompilerBuilder &CB;
205 clang::Interpreter &MainInterp;
206 ReplListCompleter(clang::IncrementalCompilerBuilder &CB,
207 clang::Interpreter &Interp)
208 : CB(CB), MainInterp(Interp) {};
209
210 std::vector<llvm::LineEditor::Completion> operator()(llvm::StringRef Buffer,
211 size_t Pos) const;
212 std::vector<llvm::LineEditor::Completion>
213 operator()(llvm::StringRef Buffer, size_t Pos, llvm::Error &ErrRes) const;
214};
215
216std::vector<llvm::LineEditor::Completion>
217ReplListCompleter::operator()(llvm::StringRef Buffer, size_t Pos) const {
218 auto Err = llvm::Error::success();
219 auto res = (*this)(Buffer, Pos, Err);
220 if (Err)
221 llvm::logAllUnhandledErrors(E: std::move(Err), OS&: llvm::errs(), ErrorBanner: "error: ");
222 return res;
223}
224
225std::vector<llvm::LineEditor::Completion>
226ReplListCompleter::operator()(llvm::StringRef Buffer, size_t Pos,
227 llvm::Error &ErrRes) const {
228 std::vector<llvm::LineEditor::Completion> Comps;
229 std::vector<std::string> Results;
230
231 auto CI = CB.CreateCpp();
232 if (auto Err = CI.takeError()) {
233 ErrRes = std::move(Err);
234 return {};
235 }
236
237 size_t Lines =
238 std::count(first: Buffer.begin(), last: std::next(x: Buffer.begin(), n: Pos), value: '\n') + 1;
239 auto Interp = clang::Interpreter::create(CI: std::move(*CI));
240
241 if (auto Err = Interp.takeError()) {
242 // log the error and returns an empty vector;
243 ErrRes = std::move(Err);
244
245 return {};
246 }
247 auto *MainCI = (*Interp)->getCompilerInstance();
248 auto CC = clang::ReplCodeCompleter();
249 CC.codeComplete(InterpCI: MainCI, Content: Buffer, Line: Lines, Col: Pos + 1,
250 ParentCI: MainInterp.getCompilerInstance(), CCResults&: Results);
251 for (auto c : Results) {
252 if (c.find(str: CC.Prefix) == 0)
253 Comps.push_back(
254 x: llvm::LineEditor::Completion(c.substr(pos: CC.Prefix.size()), c));
255 }
256 return Comps;
257}
258
259llvm::ExitOnError ExitOnErr;
260int main(int argc, const char **argv) {
261 llvm::sys::PrintStackTraceOnErrorSignal(Argv0: argv[0]);
262
263 ExitOnErr.setBanner("clang-repl: ");
264 llvm::llvm_shutdown_obj Y; // Call llvm_shutdown() on exit.
265
266 // Initialize all targets (required for device offloading)
267 llvm::InitializeAllTargetInfos();
268 llvm::InitializeAllTargets();
269 llvm::InitializeAllTargetMCs();
270 llvm::InitializeAllAsmPrinters();
271 llvm::InitializeAllAsmParsers();
272
273 // Parse after the target initialization, which registers target library
274 // options. Pass library options to the compiler as -mllvm.
275 llvm::cl::LibraryOptionsContext LibOpts;
276 llvm::cl::parseCommandLineOptions(Context&: LibOpts, Args: llvm::ArrayRef(argv, argc));
277 std::vector<const char *> ClangArgv;
278 for (const std::string &Arg : ClangArgs)
279 ClangArgv.push_back(x: Arg.c_str());
280 for (const char *Arg : LibOpts.args()) {
281 ClangArgv.push_back(x: "-mllvm");
282 ClangArgv.push_back(x: Arg);
283 }
284
285 auto IEB = clang::IncrementalExecutorBuilder::createDefault();
286 if (OptHostSupportsJit) {
287 llvm::outs() << (IEB->supportsJIT() ? "true\n" : "false\n");
288 return 0;
289 } else if (OptHostJitTriple) {
290 auto T = ExitOnErr(IEB->getHostJITTriple());
291 llvm::outs() << T.normalize() << '\n';
292 return 0;
293 }
294
295 ExitOnErr(sanitizeOopArguments(ArgV0: argv[0]));
296
297 clang::IncrementalCompilerBuilder CB;
298 CB.SetCompilerArgs(ClangArgv);
299
300 auto SizeOrErr = getSlabAllocSize(SizeString: SlabAllocateSizeString);
301 if (!SizeOrErr) {
302 llvm::logAllUnhandledErrors(E: SizeOrErr.takeError(), OS&: llvm::errs(), ErrorBanner: "error: ");
303 return EXIT_FAILURE;
304 }
305
306 clang::IncrementalExecutorBuilder::Options ExecutorOpts;
307 ExecutorOpts.IsOutOfProcess =
308 !OOPExecutor.empty() || !OOPExecutorConnect.empty();
309 ExecutorOpts.ExecutorPath = OOPExecutor;
310 ExecutorOpts.RuntimePath = OrcRuntimePath;
311 ExecutorOpts.SlabAllocateSize = *SizeOrErr;
312 ExecutorOpts.UseSharedMemory = UseSharedMemory;
313 ExitOnErr(IEB->configure(CB, Opts: ExecutorOpts));
314
315 // If both -cuda and -hip are passed, the one that appears later wins.
316 // FIXME: Support both CUDA and HIP setups at the same time, for example with
317 // one interpreter per device language in a multi-interpreter setup.
318 bool UseHip = HipEnabled && (!CudaEnabled || HipEnabled.getPosition() >
319 CudaEnabled.getPosition());
320 bool DeviceEnabled = HipEnabled || CudaEnabled;
321
322 clang::OffloadType OffloadKind =
323 UseHip ? clang::OffloadType::HIP : clang::OffloadType::CUDA;
324 std::string DevicePath = UseHip ? RocmPath : CudaPath;
325 llvm::StringRef DeviceOffloadArch = !OffloadArch.empty()
326 ? llvm::StringRef(OffloadArch)
327 : (UseHip ? "native" : "sm_35");
328
329 std::unique_ptr<clang::CompilerInstance> DeviceCI;
330
331 if (DeviceEnabled) {
332 CB.SetDeviceSDK(DevicePath);
333 CB.SetOffloadArch(DeviceOffloadArch);
334 DeviceCI = ExitOnErr(CB.CreateDevice(Type: OffloadKind));
335 }
336
337 // FIXME: Investigate if we could use runToolOnCodeWithArgs from tooling. It
338 // can replace the boilerplate code for creation of the compiler instance.
339 std::unique_ptr<clang::CompilerInstance> CI;
340 if (DeviceEnabled) {
341 CI = ExitOnErr(CB.CreateHost(Type: OffloadKind));
342 } else {
343 CI = ExitOnErr(CB.CreateCpp());
344 }
345
346 // Set an error handler, so that any LLVM backend diagnostics go through our
347 // error handler.
348 llvm::install_fatal_error_handler(handler: LLVMErrorHandler,
349 user_data: static_cast<void *>(&CI->getDiagnostics()));
350
351 // Load any requested plugins.
352 CI->LoadRequestedPlugins();
353 if (DeviceEnabled)
354 DeviceCI->LoadRequestedPlugins();
355
356 std::unique_ptr<clang::Interpreter> Interp;
357
358 if (DeviceEnabled) {
359 Interp = ExitOnErr(clang::Interpreter::createWithDevice(
360 CI: std::move(CI), DCI: std::move(DeviceCI), Type: OffloadKind));
361
362 const char *RuntimeLib = UseHip ? "libamdhip64.so" : "libcudart.so";
363 if (DevicePath.empty()) {
364 ExitOnErr(Interp->LoadDynamicLibrary(name: RuntimeLib));
365 } else {
366 std::string RuntimeLibPath = DevicePath + "/lib/" + RuntimeLib;
367 ExitOnErr(Interp->LoadDynamicLibrary(name: RuntimeLibPath.c_str()));
368 }
369 } else {
370 Interp =
371 ExitOnErr(clang::Interpreter::create(CI: std::move(CI), IEB: std::move(IEB)));
372 }
373
374 bool HasError = false;
375
376 for (const std::string &input : OptInputs) {
377 if (auto Err = Interp->ParseAndExecute(Code: input)) {
378 llvm::logAllUnhandledErrors(E: std::move(Err), OS&: llvm::errs(), ErrorBanner: "error: ");
379 HasError = true;
380 }
381 }
382
383 if (OptInputs.empty()) {
384 llvm::LineEditor LE("clang-repl");
385 std::string Input;
386 LE.setListCompleter(ReplListCompleter(CB, *Interp));
387 while (std::optional<std::string> Line = LE.readLine()) {
388 llvm::StringRef L = *Line;
389 L = L.trim();
390 if (L.ends_with(Suffix: "\\")) {
391 Input += L.drop_back(N: 1);
392 // If it is a preprocessor directive, new lines matter.
393 if (L.starts_with(Prefix: '#'))
394 Input += "\n";
395 LE.setPrompt("clang-repl... ");
396 continue;
397 }
398
399 Input += L;
400 // If we add more % commands, there should be better architecture than
401 // this.
402 if (Input == R"(%quit)") {
403 break;
404 }
405 if (Input == R"(%undo)") {
406 if (auto Err = Interp->Undo())
407 llvm::logAllUnhandledErrors(E: std::move(Err), OS&: llvm::errs(), ErrorBanner: "error: ");
408 } else if (Input == R"(%help)") {
409 llvm::outs() << "%help\t\tlist clang-repl %commands\n"
410 << "%undo\t\tundo the previous input\n"
411 << "%lib\t<path>\tlink a dynamic library\n"
412 << "%quit\t\texit clang-repl\n";
413 } else if (Input == R"(%lib)") {
414 auto Err = llvm::make_error<llvm::StringError>(
415 Args: "%lib expects 1 argument: the path to a dynamic library\n",
416 Args: std::error_code());
417 llvm::logAllUnhandledErrors(E: std::move(Err), OS&: llvm::errs(), ErrorBanner: "error: ");
418 } else if (Input.rfind(s: "%lib ", pos: 0) == 0) {
419 if (auto Err = Interp->LoadDynamicLibrary(name: Input.data() + 5))
420 llvm::logAllUnhandledErrors(E: std::move(Err), OS&: llvm::errs(), ErrorBanner: "error: ");
421 } else if (Input[0] == '%') {
422 auto Err = llvm::make_error<llvm::StringError>(
423 Args: llvm::formatv(
424 Fmt: "Invalid % command \"{0}\", use \"%help\" to list commands\n",
425 Vals&: Input),
426 Args: std::error_code());
427 llvm::logAllUnhandledErrors(E: std::move(Err), OS&: llvm::errs(), ErrorBanner: "error: ");
428 } else if (auto Err = Interp->ParseAndExecute(Code: Input)) {
429 llvm::logAllUnhandledErrors(E: std::move(Err), OS&: llvm::errs(), ErrorBanner: "error: ");
430 }
431
432 Input = "";
433 LE.setPrompt("clang-repl> ");
434 }
435 }
436
437 // Our error handler depends on the Diagnostics object, which we're
438 // potentially about to delete. Uninstall the handler now so that any
439 // later errors use the default handling behavior instead.
440 llvm::remove_fatal_error_handler();
441
442 return checkDiagErrors(CI: Interp->getCompilerInstance(), HasError);
443}
444