1//===- Job.cpp - Command to Execute ---------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "clang/Driver/Job.h"
10#include "clang/Basic/LLVM.h"
11#include "clang/Driver/Driver.h"
12#include "clang/Driver/InputInfo.h"
13#include "clang/Driver/Tool.h"
14#include "clang/Driver/ToolChain.h"
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/ADT/SmallString.h"
17#include "llvm/ADT/SmallVector.h"
18#include "llvm/ADT/StringExtras.h"
19#include "llvm/ADT/StringRef.h"
20#include "llvm/ADT/StringSet.h"
21#include "llvm/ADT/StringSwitch.h"
22#include "llvm/Support/CrashRecoveryContext.h"
23#include "llvm/Support/FileSystem.h"
24#include "llvm/Support/IOSandbox.h"
25#include "llvm/Support/Path.h"
26#include "llvm/Support/PrettyStackTrace.h"
27#include "llvm/Support/Program.h"
28#include "llvm/Support/raw_ostream.h"
29#include <cassert>
30#include <cstddef>
31#include <string>
32#include <system_error>
33#include <utility>
34
35using namespace clang;
36using namespace driver;
37
38Command::Command(const Action &Source, const Tool &Creator,
39 ResponseFileSupport ResponseSupport, const char *Executable,
40 const llvm::opt::ArgStringList &Arguments,
41 ArrayRef<InputInfo> Inputs, ArrayRef<InputInfo> Outputs,
42 const char *PrependArg)
43 : Source(Source), Creator(Creator), ResponseSupport(ResponseSupport),
44 Executable(Executable), PrependArg(PrependArg), Arguments(Arguments) {
45 for (const auto &II : Inputs)
46 if (II.isFilename())
47 InputInfoList.push_back(x: II);
48 for (const auto &II : Outputs)
49 if (II.isFilename())
50 OutputFilenames.push_back(x: II.getFilename());
51}
52
53/// Check if the compiler flag in question should be skipped when
54/// emitting a reproducer. Also track how many arguments it has and if the
55/// option is some kind of include path.
56static bool skipArgs(const char *Flag, bool HaveCrashVFS, int &SkipNum,
57 bool &IsInclude) {
58 SkipNum = 2;
59 // These flags are all of the form -Flag <Arg> and are treated as two
60 // arguments. Therefore, we need to skip the flag and the next argument.
61 bool ShouldSkip =
62 llvm::StringSwitch<bool>(Flag)
63 .Cases(CaseStrings: {"-MF", "-MT", "-MQ", "-serialize-diagnostic-file"}, Value: true)
64 .Cases(CaseStrings: {"-o", "-dependency-file"}, Value: true)
65 .Cases(CaseStrings: {"-fdebug-compilation-dir", "-diagnostic-log-file"}, Value: true)
66 .Cases(CaseStrings: {"-dwarf-debug-flags", "-ivfsoverlay"}, Value: true)
67 .Default(Value: false);
68 if (ShouldSkip)
69 return true;
70
71 // Some include flags shouldn't be skipped if we have a crash VFS
72 IsInclude =
73 llvm::StringSwitch<bool>(Flag)
74 .Cases(CaseStrings: {"-include", "-header-include-file"}, Value: true)
75 .Cases(CaseStrings: {"-idirafter", "-internal-isystem", "-iwithprefix"}, Value: true)
76 .Cases(CaseStrings: {"-internal-externc-isystem", "-iprefix"}, Value: true)
77 .Cases(CaseStrings: {"-iwithprefixbefore", "-isystem", "-iquote"}, Value: true)
78 .Cases(CaseStrings: {"-isysroot", "-I", "-F", "-resource-dir"}, Value: true)
79 .Cases(CaseStrings: {"-internal-iframework", "-iframework", "-include-pch"}, Value: true)
80 .Default(Value: false);
81 if (IsInclude)
82 return !HaveCrashVFS;
83
84 // The remaining flags are treated as a single argument.
85
86 // These flags are all of the form -Flag and have no second argument.
87 ShouldSkip = llvm::StringSwitch<bool>(Flag)
88 .Cases(CaseStrings: {"-M", "-MM", "-MG", "-MP", "-MD"}, Value: true)
89 .Case(S: "-MMD", Value: true)
90 .Default(Value: false);
91
92 // Match found.
93 SkipNum = 1;
94 if (ShouldSkip)
95 return true;
96
97 // These flags are treated as a single argument (e.g., -F<Dir>).
98 StringRef FlagRef(Flag);
99 IsInclude = FlagRef.starts_with(Prefix: "-F") || FlagRef.starts_with(Prefix: "-I");
100 if (IsInclude)
101 return !HaveCrashVFS;
102 if (FlagRef.starts_with(Prefix: "-fmodules-cache-path="))
103 return true;
104
105 SkipNum = 0;
106 return false;
107}
108
109void Command::writeResponseFile(raw_ostream &OS) const {
110 // In a file list, we only write the set of inputs to the response file
111 if (ResponseSupport.ResponseKind == ResponseFileSupport::RF_FileList) {
112 for (const auto *Arg : InputFileList) {
113 OS << Arg << '\n';
114 }
115 return;
116 }
117
118 // In regular response files, we send all arguments to the response file.
119 // Wrapping all arguments in double quotes ensures that both Unix tools and
120 // Windows tools understand the response file.
121 for (const auto *Arg : Arguments) {
122 OS << '"';
123
124 for (; *Arg != '\0'; Arg++) {
125 if (*Arg == '\"' || *Arg == '\\') {
126 OS << '\\';
127 }
128 OS << *Arg;
129 }
130
131 OS << "\" ";
132 }
133}
134
135void Command::buildArgvForResponseFile(
136 llvm::SmallVectorImpl<const char *> &Out) const {
137 // When not a file list, all arguments are sent to the response file.
138 // This leaves us to set the argv to a single parameter, requesting the tool
139 // to read the response file.
140 if (ResponseSupport.ResponseKind != ResponseFileSupport::RF_FileList) {
141 Out.push_back(Elt: Executable);
142 Out.push_back(Elt: ResponseFileFlag.c_str());
143 return;
144 }
145
146 llvm::StringSet<> Inputs(llvm::from_range, InputFileList);
147 Out.push_back(Elt: Executable);
148
149 if (PrependArg)
150 Out.push_back(Elt: PrependArg);
151
152 // In a file list, build args vector ignoring parameters that will go in the
153 // response file (elements of the InputFileList vector)
154 bool FirstInput = true;
155 for (const auto *Arg : Arguments) {
156 if (Inputs.count(Key: Arg) == 0) {
157 Out.push_back(Elt: Arg);
158 } else if (FirstInput) {
159 FirstInput = false;
160 Out.push_back(Elt: ResponseSupport.ResponseFlag);
161 Out.push_back(Elt: ResponseFile);
162 }
163 }
164}
165
166/// Rewrite relative include-like flag paths to absolute ones.
167static void
168rewriteIncludes(const llvm::ArrayRef<const char *> &Args, size_t Idx,
169 size_t NumArgs,
170 llvm::SmallVectorImpl<llvm::SmallString<128>> &IncFlags) {
171 using namespace llvm;
172 using namespace sys;
173
174 auto getAbsPath = [](StringRef InInc, SmallVectorImpl<char> &OutInc) -> bool {
175 if (path::is_absolute(path: InInc)) // Nothing to do here...
176 return false;
177 std::error_code EC = fs::current_path(result&: OutInc);
178 if (EC)
179 return false;
180 path::append(path&: OutInc, a: InInc);
181 return true;
182 };
183
184 SmallString<128> NewInc;
185 if (NumArgs == 1) {
186 StringRef FlagRef(Args[Idx + NumArgs - 1]);
187 assert((FlagRef.starts_with("-F") || FlagRef.starts_with("-I")) &&
188 "Expecting -I or -F");
189 StringRef Inc = FlagRef.substr(Start: 2);
190 if (getAbsPath(Inc, NewInc)) {
191 SmallString<128> NewArg(FlagRef.slice(Start: 0, End: 2));
192 NewArg += NewInc;
193 IncFlags.push_back(Elt: std::move(NewArg));
194 }
195 return;
196 }
197
198 assert(NumArgs == 2 && "Not expecting more than two arguments");
199 StringRef Inc(Args[Idx + NumArgs - 1]);
200 if (!getAbsPath(Inc, NewInc))
201 return;
202 IncFlags.push_back(Elt: SmallString<128>(Args[Idx]));
203 IncFlags.push_back(Elt: std::move(NewInc));
204}
205
206void Command::Print(raw_ostream &OS, const char *Terminator, bool Quote,
207 CrashReportInfo *CrashInfo) const {
208 // Always quote the exe.
209 OS << ' ';
210 llvm::sys::printArg(OS, Arg: Executable, /*Quote=*/true);
211
212 ArrayRef<const char *> Args = Arguments;
213 SmallVector<const char *, 128> ArgsRespFile;
214 if (ResponseFile != nullptr) {
215 buildArgvForResponseFile(Out&: ArgsRespFile);
216 Args = ArrayRef<const char *>(ArgsRespFile).slice(N: 1); // no executable name
217 } else if (PrependArg) {
218 OS << ' ';
219 llvm::sys::printArg(OS, Arg: PrependArg, /*Quote=*/true);
220 }
221
222 bool HaveCrashVFS = CrashInfo && !CrashInfo->VFSPath.empty();
223 for (size_t i = 0, e = Args.size(); i < e; ++i) {
224 const char *const Arg = Args[i];
225
226 if (CrashInfo) {
227 int NumArgs = 0;
228 bool IsInclude = false;
229 if (skipArgs(Flag: Arg, HaveCrashVFS, SkipNum&: NumArgs, IsInclude)) {
230 i += NumArgs - 1;
231 continue;
232 }
233
234 // Relative includes need to be expanded to absolute paths.
235 if (HaveCrashVFS && IsInclude) {
236 SmallVector<SmallString<128>, 2> NewIncFlags;
237 rewriteIncludes(Args, Idx: i, NumArgs, IncFlags&: NewIncFlags);
238 if (!NewIncFlags.empty()) {
239 for (auto &F : NewIncFlags) {
240 OS << ' ';
241 llvm::sys::printArg(OS, Arg: F.c_str(), Quote);
242 }
243 i += NumArgs - 1;
244 continue;
245 }
246 }
247
248 auto Found = llvm::find_if(Range: InputInfoList, P: [&Arg](const InputInfo &II) {
249 return II.getFilename() == Arg;
250 });
251 if (Found != InputInfoList.end() &&
252 (i == 0 || StringRef(Args[i - 1]) != "-main-file-name")) {
253 // Replace the input file name with the crashinfo's file name.
254 OS << ' ';
255 StringRef ShortName = llvm::sys::path::filename(path: CrashInfo->Filename);
256 llvm::sys::printArg(OS, Arg: ShortName.str(), Quote);
257 continue;
258 }
259 }
260
261 OS << ' ';
262 llvm::sys::printArg(OS, Arg, Quote);
263 }
264
265 if (CrashInfo && HaveCrashVFS) {
266 OS << ' ';
267 llvm::sys::printArg(OS, Arg: "-ivfsoverlay", Quote);
268 OS << ' ';
269 llvm::sys::printArg(OS, Arg: CrashInfo->VFSPath.str(), Quote);
270
271 // The leftover modules from the crash are stored in
272 // <name>.cache/vfs/modules
273 // Leave it untouched for pcm inspection and provide a clean/empty dir
274 // path to contain the future generated module cache:
275 // <name>.cache/vfs/repro-modules
276 SmallString<128> RelModCacheDir = llvm::sys::path::parent_path(
277 path: llvm::sys::path::parent_path(path: CrashInfo->VFSPath));
278 llvm::sys::path::append(path&: RelModCacheDir, a: "repro-modules");
279
280 std::string ModCachePath = "-fmodules-cache-path=";
281 ModCachePath.append(s: RelModCacheDir.c_str());
282
283 OS << ' ';
284 llvm::sys::printArg(OS, Arg: ModCachePath, Quote);
285 }
286
287 if (ResponseFile != nullptr) {
288 OS << "\n Arguments passed via response file:\n";
289 writeResponseFile(OS);
290 // Avoiding duplicated newline terminator, since FileLists are
291 // newline-separated.
292 if (ResponseSupport.ResponseKind != ResponseFileSupport::RF_FileList)
293 OS << "\n";
294 OS << " (end of response file)";
295 }
296
297 OS << Terminator;
298}
299
300void Command::setResponseFile(const char *FileName) {
301 ResponseFile = FileName;
302 ResponseFileFlag = ResponseSupport.ResponseFlag;
303 ResponseFileFlag += FileName;
304}
305
306void Command::setEnvironment(llvm::ArrayRef<const char *> NewEnvironment) {
307 Environment.reserve(n: NewEnvironment.size() + 1);
308 Environment.assign(first: NewEnvironment.begin(), last: NewEnvironment.end());
309 Environment.push_back(x: nullptr);
310}
311
312void Command::setRedirectFiles(
313 const std::vector<std::optional<std::string>> &Redirects) {
314 RedirectFiles = Redirects;
315}
316
317void Command::PrintFileNames() const {
318 if (PrintInputFilenames) {
319 for (const auto &Arg : InputInfoList)
320 llvm::outs() << llvm::sys::path::filename(path: Arg.getFilename()) << "\n";
321 llvm::outs().flush();
322 }
323}
324
325int Command::Execute(ArrayRef<std::optional<StringRef>> Redirects,
326 std::string *ErrMsg, bool *ExecutionFailed) const {
327 PrintFileNames();
328
329 SmallVector<const char *, 128> Argv;
330 if (ResponseFile == nullptr) {
331 Argv.push_back(Elt: Executable);
332 if (PrependArg)
333 Argv.push_back(Elt: PrependArg);
334 Argv.append(in_start: Arguments.begin(), in_end: Arguments.end());
335 Argv.push_back(Elt: nullptr);
336 } else {
337 // If the command is too large, we need to put arguments in a response file.
338 std::string RespContents;
339 llvm::raw_string_ostream SS(RespContents);
340
341 // Write file contents and build the Argv vector
342 writeResponseFile(OS&: SS);
343 buildArgvForResponseFile(Out&: Argv);
344 Argv.push_back(Elt: nullptr);
345
346 // Save the response file in the appropriate encoding
347 if (std::error_code EC = writeFileWithEncoding(
348 FileName: ResponseFile, Contents: RespContents, Encoding: ResponseSupport.ResponseEncoding)) {
349 if (ErrMsg)
350 *ErrMsg = EC.message();
351 if (ExecutionFailed)
352 *ExecutionFailed = true;
353 // Return -1 by convention (see llvm/include/llvm/Support/Program.h) to
354 // indicate the requested executable cannot be started.
355 return -1;
356 }
357 }
358
359 std::optional<ArrayRef<StringRef>> Env;
360 std::vector<StringRef> ArgvVectorStorage;
361 if (!Environment.empty()) {
362 assert(Environment.back() == nullptr &&
363 "Environment vector should be null-terminated by now");
364 ArgvVectorStorage = llvm::toStringRefArray(Strings: Environment.data());
365 Env = ArrayRef(ArgvVectorStorage);
366 }
367
368 auto Args = llvm::toStringRefArray(Strings: Argv.data());
369
370 // Use Job-specific redirect files if they are present.
371 if (!RedirectFiles.empty()) {
372 std::vector<std::optional<StringRef>> RedirectFilesOptional;
373 for (const auto &Ele : RedirectFiles)
374 if (Ele)
375 RedirectFilesOptional.push_back(x: std::optional<StringRef>(*Ele));
376 else
377 RedirectFilesOptional.push_back(x: std::nullopt);
378
379 return llvm::sys::ExecuteAndWait(Program: Executable, Args, Env,
380 Redirects: ArrayRef(RedirectFilesOptional),
381 /*secondsToWait=*/SecondsToWait: 0, /*memoryLimit=*/MemoryLimit: 0,
382 ErrMsg, ExecutionFailed, ProcStat: &ProcStat);
383 }
384
385 return llvm::sys::ExecuteAndWait(Program: Executable, Args, Env, Redirects,
386 /*secondsToWait*/ SecondsToWait: 0, /*memoryLimit*/ MemoryLimit: 0,
387 ErrMsg, ExecutionFailed, ProcStat: &ProcStat);
388}
389
390CC1Command::CC1Command(const Action &Source, const Tool &Creator,
391 ResponseFileSupport ResponseSupport,
392 const char *Executable,
393 const llvm::opt::ArgStringList &Arguments,
394 ArrayRef<InputInfo> Inputs, ArrayRef<InputInfo> Outputs,
395 const char *PrependArg)
396 : Command(Source, Creator, ResponseSupport, Executable, Arguments, Inputs,
397 Outputs, PrependArg) {
398 InProcess = true;
399}
400
401void CC1Command::Print(raw_ostream &OS, const char *Terminator, bool Quote,
402 CrashReportInfo *CrashInfo) const {
403 if (InProcess)
404 OS << " (in-process)\n";
405 Command::Print(OS, Terminator, Quote, CrashInfo);
406}
407
408int CC1Command::Execute(ArrayRef<std::optional<StringRef>> Redirects,
409 std::string *ErrMsg, bool *ExecutionFailed) const {
410 // FIXME: Currently, if there're more than one job, we disable
411 // -fintegrate-cc1. If we're no longer a integrated-cc1 job, fallback to
412 // out-of-process execution. See discussion in https://reviews.llvm.org/D74447
413 if (!InProcess)
414 return Command::Execute(Redirects, ErrMsg, ExecutionFailed);
415
416 PrintFileNames();
417
418 SmallVector<const char *, 128> Argv;
419 Argv.push_back(Elt: getExecutable());
420 Argv.append(in_start: getArguments().begin(), in_end: getArguments().end());
421 Argv.push_back(Elt: nullptr);
422 Argv.pop_back(); // The terminating null element shall not be part of the
423 // slice (main() behavior).
424
425 // This flag simply indicates that the program couldn't start, which isn't
426 // applicable here.
427 if (ExecutionFailed)
428 *ExecutionFailed = false;
429
430 // Enabling the sandbox here allows us to restore its previous state even when
431 // this cc1 invocation crashes.
432 auto EnableSandbox = llvm::sys::sandbox::scopedEnable();
433
434 llvm::CrashRecoveryContext CRC;
435 CRC.DumpStackAndCleanupOnFailure = true;
436
437 const void *PrettyState = llvm::SavePrettyStackState();
438 const Driver &D = getCreator().getToolChain().getDriver();
439
440 int R = 0;
441 // Enter ExecuteCC1Tool() instead of starting up a new process
442 if (!CRC.RunSafely(Fn: [&]() { R = D.CC1Main(Argv); })) {
443 llvm::RestorePrettyStackState(State: PrettyState);
444 return CRC.RetCode;
445 }
446 return R;
447}
448
449void CC1Command::setEnvironment(llvm::ArrayRef<const char *> NewEnvironment) {
450 // We don't support set a new environment when calling into ExecuteCC1Tool()
451 llvm_unreachable(
452 "The CC1Command doesn't support changing the environment vars!");
453}
454
455void JobList::Print(raw_ostream &OS, const char *Terminator, bool Quote,
456 CrashReportInfo *CrashInfo) const {
457 for (const auto &Job : *this)
458 Job.Print(OS, Terminator, Quote, CrashInfo);
459}
460
461void JobList::clear() { Jobs.clear(); }
462