1//===- DependencyScanningTool.cpp - clang-scan-deps service ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "clang/Tooling/DependencyScanningTool.h"
10#include "clang/Basic/AtomicLineLogger.h"
11#include "clang/Basic/Diagnostic.h"
12#include "clang/Basic/DiagnosticFrontend.h"
13#include "clang/DependencyScanning/DependencyScanningWorker.h"
14#include "clang/Driver/Compilation.h"
15#include "clang/Driver/Driver.h"
16#include "clang/Driver/Tool.h"
17#include "clang/Frontend/CompilerInstance.h"
18#include "clang/Frontend/Utils.h"
19#include "clang/Options/Options.h"
20#include "llvm/ADT/SmallVectorExtras.h"
21#include "llvm/ADT/iterator.h"
22#include "llvm/TargetParser/Host.h"
23#include <optional>
24
25using namespace clang;
26using namespace tooling;
27using namespace dependencies;
28
29namespace {
30/// Prints out all of the gathered dependencies into a string.
31class MakeDependencyPrinterConsumer : public DependencyConsumer {
32public:
33 void handleBuildCommand(Command) override {}
34
35 void
36 handleDependencyOutputOpts(const DependencyOutputOptions &Opts) override {
37 this->Opts = std::make_unique<DependencyOutputOptions>(args: Opts);
38 }
39
40 void handleFileDependency(StringRef File) override {
41 SmallString<128> NormalizedFile = File;
42 llvm::sys::path::remove_dots(path&: NormalizedFile, /*remove_dot_dot=*/true);
43 Dependencies.emplace_back(args: NormalizedFile.str());
44 }
45
46 // These are ignored for the make format as it can't support the full
47 // set of deps, and handleFileDependency handles enough for implicitly
48 // built modules to work.
49 void handlePrebuiltModuleDependency(PrebuiltModuleDep PMD) override {}
50 void handleModuleDependency(ModuleDeps MD) override {
51 MD.forEachFileDep(Cb: [this](StringRef File) {
52 DependenciesFromModules.push_back(x: std::string(File));
53 });
54 }
55 void handleDirectModuleDependency(ModuleID ID) override {}
56 void handleVisibleModule(std::string ModuleName) override {}
57 void handleContextHash(std::string Hash) override {}
58
59 void printDependencies(std::string &S) {
60 assert(Opts && "Handled dependency output options.");
61
62 class DependencyPrinter : public DependencyFileGenerator {
63 public:
64 DependencyPrinter(DependencyOutputOptions &Opts,
65 ArrayRef<std::string> Dependencies,
66 ArrayRef<std::string> ModuleDependencies)
67 : DependencyFileGenerator(Opts) {
68 for (const auto &Dep : Dependencies)
69 addDependency(Filename: Dep);
70 for (const auto &Dep : ModuleDependencies)
71 addDependency(Filename: Dep);
72 }
73
74 void printDependencies(std::string &S) {
75 llvm::raw_string_ostream OS(S);
76 outputDependencyFile(OS);
77 }
78 };
79
80 DependencyPrinter Generator(*Opts, Dependencies, DependenciesFromModules);
81 Generator.printDependencies(S);
82 }
83
84protected:
85 std::unique_ptr<DependencyOutputOptions> Opts;
86 std::vector<std::string> Dependencies;
87 std::vector<std::string> DependenciesFromModules;
88};
89} // anonymous namespace
90
91static std::pair<std::unique_ptr<driver::Driver>,
92 std::unique_ptr<driver::Compilation>>
93buildCompilation(ArrayRef<std::string> ArgStrs, DiagnosticsEngine &Diags,
94 IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
95 llvm::BumpPtrAllocator &Alloc, AtomicLineLogger &Logger) {
96 SmallVector<const char *, 256> Argv;
97 Argv.reserve(N: ArgStrs.size());
98 for (const std::string &Arg : ArgStrs)
99 Argv.push_back(Elt: Arg.c_str());
100
101 std::unique_ptr<driver::Driver> Driver = std::make_unique<driver::Driver>(
102 args&: Argv[0], args: llvm::sys::getDefaultTargetTriple(), args&: Diags,
103 args: "clang LLVM compiler", args&: FS);
104 Driver->setTitle("clang_based_tool");
105
106 bool CLMode = driver::IsClangCL(
107 DriverMode: driver::getDriverMode(ProgName: Argv[0], Args: ArrayRef(Argv).slice(N: 1)));
108
109 if (llvm::Error E =
110 driver::expandResponseFiles(Args&: Argv, ClangCLMode: CLMode, Alloc, FS: FS.get())) {
111 Diags.Report(DiagID: diag::err_drv_expand_response_file)
112 << llvm::toString(E: std::move(E));
113 return std::make_pair(x: nullptr, y: nullptr);
114 }
115
116 std::unique_ptr<driver::Compilation> Compilation(
117 Driver->BuildCompilation(Args: Argv));
118 if (!Compilation)
119 return std::make_pair(x: nullptr, y: nullptr);
120
121 if (Compilation->containsError())
122 return std::make_pair(x: nullptr, y: nullptr);
123
124 if (Compilation->getJobs().empty()) {
125 Diags.Report(DiagID: diag::err_fe_expected_compiler_job)
126 << llvm::join(R&: ArgStrs, Separator: " ");
127 return std::make_pair(x: nullptr, y: nullptr);
128 }
129
130 const llvm::opt::Arg *LogPathArg =
131 Compilation->getArgs().getLastArg(Ids: options::OPT_fdepscan_log_path);
132 StringRef LogPath =
133 LogPathArg ? StringRef(LogPathArg->getValue()).trim() : StringRef();
134
135 // Forbid -fdepscan-log-path="".
136 if (LogPathArg && LogPath.empty()) {
137 Diags.Report(DiagID: diag::err_drv_depscan_log_path_empty);
138 return std::make_pair(x: nullptr, y: nullptr);
139 }
140
141 if (!Logger.enable(RequestedLogPath: LogPath)) {
142 Diags.Report(DiagID: diag::err_drv_depscan_log_path_inconsistent)
143 << unsigned(!LogPath.empty()) << LogPath
144 << unsigned(!Logger.getLogPath().empty()) << Logger.getLogPath();
145 return std::make_pair(x: nullptr, y: nullptr);
146 }
147
148 return std::make_pair(x: std::move(Driver), y: std::move(Compilation));
149}
150
151/// Constructs the full frontend command line, including executable, for the
152/// given driver \c Cmd.
153static SmallVector<std::string, 0>
154buildCC1CommandLine(const driver::Command &Cmd) {
155 const auto &Args = Cmd.getArguments();
156 SmallVector<std::string, 0> Out;
157 Out.reserve(N: Args.size() + 1);
158 Out.emplace_back(Args: Cmd.getExecutable());
159 llvm::append_range(C&: Out, R: Args);
160 return Out;
161}
162
163static bool computeDependenciesForDriverCommandLine(
164 DependencyScanningWorker &Worker, StringRef WorkingDirectory,
165 ArrayRef<std::string> CommandLine, DependencyConsumer &Consumer,
166 DependencyActionController &Controller, DiagnosticConsumer &DiagConsumer,
167 IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS) {
168 auto FS = Worker.makeEffectiveVFS(WorkingDirectory, OverlayFS);
169
170 // Compilation holds a non-owning a reference to the Driver, hence we need to
171 // keep the Driver alive when we use Compilation. Arguments to commands may be
172 // owned by Alloc when expanded from response files.
173 llvm::BumpPtrAllocator Alloc;
174 auto DiagOpts = createScanningDiagOptions(CommandLine);
175 auto DiagEngine =
176 CompilerInstance::createDiagnostics(VFS&: *FS, Opts&: *DiagOpts, Client: &DiagConsumer,
177 /*ShouldOwnClient=*/false);
178 const auto [Driver, Compilation] = buildCompilation(
179 ArgStrs: CommandLine, Diags&: *DiagEngine, FS, Alloc, Logger&: Worker.getService().getLogger());
180 if (!Compilation)
181 return false;
182
183 SmallVector<SmallVector<std::string, 0>> FrontendCommandLines;
184 for (const auto &Cmd : Compilation->getJobs())
185 FrontendCommandLines.push_back(Elt: buildCC1CommandLine(Cmd));
186 SmallVector<ArrayRef<std::string>> FrontendCommandLinesView(
187 FrontendCommandLines.begin(), FrontendCommandLines.end());
188
189 return Worker.computeDependencies(WorkingDirectory, CommandLines: FrontendCommandLinesView,
190 DepConsumer&: Consumer, Controller, DiagConsumer,
191 OverlayFS: std::move(OverlayFS));
192}
193
194bool tooling::computeDependencies(
195 DependencyScanningWorker &Worker, StringRef WorkingDirectory,
196 ArrayRef<std::string> CommandLine, DependencyConsumer &Consumer,
197 DependencyActionController &Controller, DiagnosticConsumer &DiagConsumer,
198 IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS) {
199 const auto IsCC1Input = (CommandLine.size() >= 2 && CommandLine[1] == "-cc1");
200 return IsCC1Input ? Worker.computeDependencies(WorkingDirectory, CommandLines: CommandLine,
201 DepConsumer&: Consumer, Controller,
202 DiagConsumer, OverlayFS)
203 : computeDependenciesForDriverCommandLine(
204 Worker, WorkingDirectory, CommandLine, Consumer,
205 Controller, DiagConsumer, OverlayFS);
206}
207
208std::optional<std::string> DependencyScanningTool::getDependencyFile(
209 ArrayRef<std::string> CommandLine, StringRef CWD,
210 LookupModuleOutputCallback LookupModuleOutput,
211 DiagnosticConsumer &DiagConsumer) {
212 MakeDependencyPrinterConsumer DepConsumer;
213 CallbackActionController Controller(LookupModuleOutput);
214 if (!computeDependencies(Worker, WorkingDirectory: CWD, CommandLine, Consumer&: DepConsumer, Controller,
215 DiagConsumer))
216 return std::nullopt;
217 std::string Output;
218 DepConsumer.printDependencies(S&: Output);
219 return Output;
220}
221
222std::optional<P1689Rule> DependencyScanningTool::getP1689ModuleDependencyFile(
223 const CompileCommand &Command, StringRef CWD, std::string &MakeformatOutput,
224 std::string &MakeformatOutputPath, DiagnosticConsumer &DiagConsumer) {
225 class P1689ModuleDependencyPrinterConsumer
226 : public MakeDependencyPrinterConsumer {
227 public:
228 P1689ModuleDependencyPrinterConsumer(P1689Rule &Rule,
229 const CompileCommand &Command)
230 : Filename(Command.Filename), Rule(Rule) {
231 Rule.PrimaryOutput = Command.Output;
232 }
233
234 void handleProvidedAndRequiredStdCXXModules(
235 std::optional<P1689ModuleInfo> Provided,
236 std::vector<P1689ModuleInfo> Requires) override {
237 Rule.Provides = std::move(Provided);
238 if (Rule.Provides)
239 Rule.Provides->SourcePath = Filename.str();
240 Rule.Requires = std::move(Requires);
241 }
242
243 StringRef getMakeFormatDependencyOutputPath() {
244 if (Opts->OutputFormat != DependencyOutputFormat::Make)
245 return {};
246 return Opts->OutputFile;
247 }
248
249 private:
250 StringRef Filename;
251 P1689Rule &Rule;
252 };
253
254 class P1689ActionController : public DependencyActionController {
255 public:
256 void initializeScanInvocation(CompilerInvocation &ScanInvocation) override {
257 ScanInvocation.getPreprocessorOpts().DependencyScanningModuleMapImports =
258 true;
259 }
260
261 // The lookupModuleOutput is for clang modules. P1689 format don't need it.
262 std::string lookupModuleOutput(const ModuleDeps &,
263 ModuleOutputKind Kind) override {
264 return "";
265 }
266
267 std::unique_ptr<DependencyActionController> clone() const override {
268 return std::make_unique<P1689ActionController>();
269 }
270 };
271
272 P1689Rule Rule;
273 P1689ModuleDependencyPrinterConsumer Consumer(Rule, Command);
274 P1689ActionController Controller;
275 if (!computeDependencies(Worker, WorkingDirectory: CWD, CommandLine: Command.CommandLine, Consumer,
276 Controller, DiagConsumer))
277 return std::nullopt;
278
279 MakeformatOutputPath = Consumer.getMakeFormatDependencyOutputPath();
280 if (!MakeformatOutputPath.empty())
281 Consumer.printDependencies(S&: MakeformatOutput);
282 return Rule;
283}
284
285static std::pair<IntrusiveRefCntPtr<llvm::vfs::FileSystem>,
286 std::vector<std::string>>
287initVFSForTUBufferScanning(ArrayRef<std::string> CommandLine,
288 llvm::MemoryBufferRef TUBuffer) {
289 StringRef InputPath = TUBuffer.getBufferIdentifier();
290 auto InputBuf = llvm::MemoryBuffer::getMemBufferCopy(InputData: TUBuffer.getBuffer());
291
292 auto FS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
293 FS->addFile(Path: InputPath, ModificationTime: 0, Buffer: std::move(InputBuf));
294
295 std::vector<std::string> ModifiedCommandLine(CommandLine);
296 ModifiedCommandLine.emplace_back(args&: InputPath);
297
298 return std::make_pair(x: std::move(FS), y&: ModifiedCommandLine);
299}
300
301static std::pair<IntrusiveRefCntPtr<llvm::vfs::FileSystem>,
302 std::vector<std::string>>
303initVFSForByNameScanning(ArrayRef<std::string> CommandLine) {
304 // The fake input buffer is read-only, and it is used to produce unique source
305 // locations for the diagnostics. Therefore, sharing this global buffer across
306 // threads is ok.
307 static const std::string FakeInput(
308 DependencyScanningWorker::MaxNumOfByNameQueries, ' ');
309
310 StringRef InputPath =
311 llvm::sys::path::is_style_windows(S: llvm::sys::path::Style::native)
312 ? "Z:\\module-include.input"
313 : "/module-include.input";
314 auto InputBuf = llvm::MemoryBuffer::getMemBuffer(InputData: FakeInput, BufferName: InputPath);
315
316 auto FS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
317 FS->addFile(Path: InputPath, ModificationTime: 0, Buffer: std::move(InputBuf));
318
319 std::vector<std::string> ModifiedCommandLine(CommandLine);
320 ModifiedCommandLine.emplace_back(args&: InputPath);
321
322 return std::make_pair(x: std::move(FS), y&: ModifiedCommandLine);
323}
324
325std::optional<TranslationUnitDeps>
326DependencyScanningTool::getTranslationUnitDependencies(
327 ArrayRef<std::string> CommandLine, StringRef CWD,
328 DiagnosticConsumer &DiagConsumer,
329 const llvm::DenseSet<ModuleID> &AlreadySeen,
330 LookupModuleOutputCallback LookupModuleOutput,
331 std::optional<llvm::MemoryBufferRef> TUBuffer) {
332 FullDependencyConsumer Consumer(AlreadySeen);
333 CallbackActionController Controller(LookupModuleOutput);
334
335 // If we are scanning from a TUBuffer, create an overlay filesystem with the
336 // input as an in-memory file and add it to the command line.
337 IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS = nullptr;
338 std::vector<std::string> CommandLineWithTUBufferInput;
339 if (TUBuffer) {
340 std::tie(args&: OverlayFS, args&: CommandLineWithTUBufferInput) =
341 initVFSForTUBufferScanning(CommandLine, TUBuffer: *TUBuffer);
342 CommandLine = CommandLineWithTUBufferInput;
343 }
344
345 if (!computeDependencies(Worker, WorkingDirectory: CWD, CommandLine, Consumer, Controller,
346 DiagConsumer, OverlayFS: std::move(OverlayFS)))
347 return std::nullopt;
348 return Consumer.takeTranslationUnitDeps();
349}
350
351static std::optional<SmallVector<std::string, 0>>
352getFirstCC1CommandLine(ArrayRef<std::string> CommandLine,
353 DiagnosticsEngine &Diags,
354 llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
355 AtomicLineLogger &Logger) {
356 // Compilation holds a non-owning a reference to the Driver, hence we need to
357 // keep the Driver alive when we use Compilation. Arguments to commands may be
358 // owned by Alloc when expanded from response files.
359 llvm::BumpPtrAllocator Alloc;
360 const auto [Driver, Compilation] =
361 buildCompilation(ArgStrs: CommandLine, Diags, FS: std::move(FS), Alloc, Logger);
362 if (!Compilation)
363 return std::nullopt;
364
365 const auto IsClangCmd = [](const driver::Command &Cmd) {
366 return StringRef(Cmd.getCreator().getName()) == "clang";
367 };
368
369 const auto &Jobs = Compilation->getJobs();
370 if (const auto It = llvm::find_if(Range: Jobs, P: IsClangCmd); It != Jobs.end())
371 return buildCC1CommandLine(Cmd: *It);
372 return std::nullopt;
373}
374
375bool DependencyScanningTool::getByNameDependencies(
376 StringRef CWD, ArrayRef<std::string> CommandLine,
377 DiagnosticConsumer &DiagConsumer, DependencyActionController &Controller,
378 llvm::function_ref<std::optional<std::string>()> getNextName,
379 DependencyConsumer &DepConsumer) {
380 auto [OverlayFS, ModifiedCommandLine] = initVFSForByNameScanning(CommandLine);
381 auto FS = Worker.makeEffectiveVFS(WorkingDirectory: CWD, OverlayFS);
382 std::vector<std::string> CC1CommandLine;
383 if (ModifiedCommandLine.size() >= 2 && ModifiedCommandLine[1] == "-cc1") {
384 CC1CommandLine = std::move(ModifiedCommandLine);
385 } else {
386 // Driver-style (or ill-formed): lower to a cc1 command line, or diagnose.
387 auto DiagOpts = createScanningDiagOptions(CommandLine: ModifiedCommandLine);
388 auto DiagEngine =
389 CompilerInstance::createDiagnostics(VFS&: *FS, Opts&: *DiagOpts, Client: &DiagConsumer,
390 /*ShouldOwnClient=*/false);
391 auto MaybeFirstCC1 = getFirstCC1CommandLine(
392 CommandLine: ModifiedCommandLine, Diags&: *DiagEngine, FS, Logger&: Worker.getService().getLogger());
393 if (!MaybeFirstCC1)
394 return false;
395 CC1CommandLine.assign(first: MaybeFirstCC1->begin(), last: MaybeFirstCC1->end());
396 }
397
398 return Worker.computeDependenciesByName(CWD, CC1CommandLine,
399 OverlayFS: std::move(OverlayFS), DiagConsumer,
400 Controller, getNextName, DepConsumer);
401}
402