1//===- DependencyScanningTool.cpp - clang-scan-deps service ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "clang/Tooling/DependencyScanningTool.h"
10#include "clang/Basic/Diagnostic.h"
11#include "clang/Basic/DiagnosticFrontend.h"
12#include "clang/DependencyScanning/DependencyScannerImpl.h"
13#include "clang/Driver/Tool.h"
14#include "clang/Frontend/Utils.h"
15#include "llvm/ADT/SmallVectorExtras.h"
16#include "llvm/ADT/iterator.h"
17#include "llvm/Support/VirtualFileSystem.h"
18#include "llvm/TargetParser/Host.h"
19#include <optional>
20
21using namespace clang;
22using namespace tooling;
23using namespace dependencies;
24
25DependencyScanningTool::DependencyScanningTool(
26 DependencyScanningService &Service,
27 llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS)
28 : Worker(Service, std::move(FS)) {}
29
30namespace {
31/// Prints out all of the gathered dependencies into a string.
32class MakeDependencyPrinterConsumer : public DependencyConsumer {
33public:
34 void handleBuildCommand(Command) override {}
35
36 void
37 handleDependencyOutputOpts(const DependencyOutputOptions &Opts) override {
38 this->Opts = std::make_unique<DependencyOutputOptions>(args: Opts);
39 }
40
41 void handleFileDependency(StringRef File) override {
42 Dependencies.push_back(x: std::string(File));
43 }
44
45 // These are ignored for the make format as it can't support the full
46 // set of deps, and handleFileDependency handles enough for implicitly
47 // built modules to work.
48 void handlePrebuiltModuleDependency(PrebuiltModuleDep PMD) override {}
49 void handleModuleDependency(ModuleDeps MD) override {}
50 void handleDirectModuleDependency(ModuleID ID) override {}
51 void handleVisibleModule(std::string ModuleName) override {}
52 void handleContextHash(std::string Hash) override {}
53
54 void printDependencies(std::string &S) {
55 assert(Opts && "Handled dependency output options.");
56
57 class DependencyPrinter : public DependencyFileGenerator {
58 public:
59 DependencyPrinter(DependencyOutputOptions &Opts,
60 ArrayRef<std::string> Dependencies)
61 : DependencyFileGenerator(Opts) {
62 for (const auto &Dep : Dependencies)
63 addDependency(Filename: Dep);
64 }
65
66 void printDependencies(std::string &S) {
67 llvm::raw_string_ostream OS(S);
68 outputDependencyFile(OS);
69 }
70 };
71
72 DependencyPrinter Generator(*Opts, Dependencies);
73 Generator.printDependencies(S);
74 }
75
76protected:
77 std::unique_ptr<DependencyOutputOptions> Opts;
78 std::vector<std::string> Dependencies;
79};
80} // anonymous namespace
81
82static std::pair<std::unique_ptr<driver::Driver>,
83 std::unique_ptr<driver::Compilation>>
84buildCompilation(ArrayRef<std::string> ArgStrs, DiagnosticsEngine &Diags,
85 IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
86 llvm::BumpPtrAllocator &Alloc) {
87 SmallVector<const char *, 256> Argv;
88 Argv.reserve(N: ArgStrs.size());
89 for (const std::string &Arg : ArgStrs)
90 Argv.push_back(Elt: Arg.c_str());
91
92 std::unique_ptr<driver::Driver> Driver = std::make_unique<driver::Driver>(
93 args&: Argv[0], args: llvm::sys::getDefaultTargetTriple(), args&: Diags,
94 args: "clang LLVM compiler", args&: FS);
95 Driver->setTitle("clang_based_tool");
96
97 bool CLMode = driver::IsClangCL(
98 DriverMode: driver::getDriverMode(ProgName: Argv[0], Args: ArrayRef(Argv).slice(N: 1)));
99
100 if (llvm::Error E =
101 driver::expandResponseFiles(Args&: Argv, ClangCLMode: CLMode, Alloc, FS: FS.get())) {
102 Diags.Report(DiagID: diag::err_drv_expand_response_file)
103 << llvm::toString(E: std::move(E));
104 return std::make_pair(x: nullptr, y: nullptr);
105 }
106
107 std::unique_ptr<driver::Compilation> Compilation(
108 Driver->BuildCompilation(Args: Argv));
109 if (!Compilation)
110 return std::make_pair(x: nullptr, y: nullptr);
111
112 if (Compilation->containsError())
113 return std::make_pair(x: nullptr, y: nullptr);
114
115 if (Compilation->getJobs().empty()) {
116 Diags.Report(DiagID: diag::err_fe_expected_compiler_job)
117 << llvm::join(R&: ArgStrs, Separator: " ");
118 return std::make_pair(x: nullptr, y: nullptr);
119 }
120
121 return std::make_pair(x: std::move(Driver), y: std::move(Compilation));
122}
123
124/// Constructs the full frontend command line, including executable, for the
125/// given driver \c Cmd.
126static SmallVector<std::string, 0>
127buildCC1CommandLine(const driver::Command &Cmd) {
128 const auto &Args = Cmd.getArguments();
129 SmallVector<std::string, 0> Out;
130 Out.reserve(N: Args.size() + 1);
131 Out.emplace_back(Args: Cmd.getExecutable());
132 llvm::append_range(C&: Out, R: Args);
133 return Out;
134}
135
136static bool computeDependenciesForDriverCommandLine(
137 DependencyScanningWorker &Worker, StringRef WorkingDirectory,
138 ArrayRef<std::string> CommandLine, DependencyConsumer &Consumer,
139 DependencyActionController &Controller, DiagnosticConsumer &DiagConsumer,
140 IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> OverlayFS) {
141 IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS = nullptr;
142 if (OverlayFS) {
143 FS = OverlayFS;
144 } else {
145 FS = &Worker.getVFS();
146 FS->setCurrentWorkingDirectory(WorkingDirectory);
147 }
148
149 // Compilation holds a non-owning a reference to the Driver, hence we need to
150 // keep the Driver alive when we use Compilation. Arguments to commands may be
151 // owned by Alloc when expanded from response files.
152 llvm::BumpPtrAllocator Alloc;
153 auto DiagEngineWithDiagOpts =
154 DiagnosticsEngineWithDiagOpts(CommandLine, FS, DiagConsumer);
155 const auto [Driver, Compilation] = buildCompilation(
156 ArgStrs: CommandLine, Diags&: *DiagEngineWithDiagOpts.DiagEngine, FS, Alloc);
157 if (!Compilation)
158 return false;
159
160 SmallVector<SmallVector<std::string, 0>> FrontendCommandLines;
161 for (const auto &Cmd : Compilation->getJobs())
162 FrontendCommandLines.push_back(Elt: buildCC1CommandLine(Cmd));
163 SmallVector<ArrayRef<std::string>> FrontendCommandLinesView(
164 FrontendCommandLines.begin(), FrontendCommandLines.end());
165
166 return Worker.computeDependencies(WorkingDirectory, CommandLines: FrontendCommandLinesView,
167 DepConsumer&: Consumer, Controller, DiagConsumer,
168 OverlayFS);
169}
170
171static llvm::Error makeErrorFromDiagnosticsOS(
172 TextDiagnosticsPrinterWithOutput &DiagPrinterWithOS) {
173 return llvm::make_error<llvm::StringError>(
174 Args&: DiagPrinterWithOS.DiagnosticsOS.str(), Args: llvm::inconvertibleErrorCode());
175}
176
177bool tooling::computeDependencies(
178 DependencyScanningWorker &Worker, StringRef WorkingDirectory,
179 ArrayRef<std::string> CommandLine, DependencyConsumer &Consumer,
180 DependencyActionController &Controller, DiagnosticConsumer &DiagConsumer,
181 llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> OverlayFS) {
182 const auto IsCC1Input = (CommandLine.size() >= 2 && CommandLine[1] == "-cc1");
183 return IsCC1Input ? Worker.computeDependencies(WorkingDirectory, CommandLine,
184 DepConsumer&: Consumer, Controller,
185 DiagConsumer, OverlayFS)
186 : computeDependenciesForDriverCommandLine(
187 Worker, WorkingDirectory, CommandLine, Consumer,
188 Controller, DiagConsumer, OverlayFS);
189}
190
191std::optional<std::string>
192DependencyScanningTool::getDependencyFile(ArrayRef<std::string> CommandLine,
193 StringRef CWD,
194 DiagnosticConsumer &DiagConsumer) {
195 MakeDependencyPrinterConsumer DepConsumer;
196 CallbackActionController Controller(nullptr);
197 if (!computeDependencies(Worker, WorkingDirectory: CWD, CommandLine, Consumer&: DepConsumer, Controller,
198 DiagConsumer))
199 return std::nullopt;
200 std::string Output;
201 DepConsumer.printDependencies(S&: Output);
202 return Output;
203}
204
205std::optional<P1689Rule> DependencyScanningTool::getP1689ModuleDependencyFile(
206 const CompileCommand &Command, StringRef CWD, std::string &MakeformatOutput,
207 std::string &MakeformatOutputPath, DiagnosticConsumer &DiagConsumer) {
208 class P1689ModuleDependencyPrinterConsumer
209 : public MakeDependencyPrinterConsumer {
210 public:
211 P1689ModuleDependencyPrinterConsumer(P1689Rule &Rule,
212 const CompileCommand &Command)
213 : Filename(Command.Filename), Rule(Rule) {
214 Rule.PrimaryOutput = Command.Output;
215 }
216
217 void handleProvidedAndRequiredStdCXXModules(
218 std::optional<P1689ModuleInfo> Provided,
219 std::vector<P1689ModuleInfo> Requires) override {
220 Rule.Provides = Provided;
221 if (Rule.Provides)
222 Rule.Provides->SourcePath = Filename.str();
223 Rule.Requires = Requires;
224 }
225
226 StringRef getMakeFormatDependencyOutputPath() {
227 if (Opts->OutputFormat != DependencyOutputFormat::Make)
228 return {};
229 return Opts->OutputFile;
230 }
231
232 private:
233 StringRef Filename;
234 P1689Rule &Rule;
235 };
236
237 class P1689ActionController : public DependencyActionController {
238 public:
239 // The lookupModuleOutput is for clang modules. P1689 format don't need it.
240 std::string lookupModuleOutput(const ModuleDeps &,
241 ModuleOutputKind Kind) override {
242 return "";
243 }
244 };
245
246 P1689Rule Rule;
247 P1689ModuleDependencyPrinterConsumer Consumer(Rule, Command);
248 P1689ActionController Controller;
249 if (!computeDependencies(Worker, WorkingDirectory: CWD, CommandLine: Command.CommandLine, Consumer,
250 Controller, DiagConsumer))
251 return std::nullopt;
252
253 MakeformatOutputPath = Consumer.getMakeFormatDependencyOutputPath();
254 if (!MakeformatOutputPath.empty())
255 Consumer.printDependencies(S&: MakeformatOutput);
256 return Rule;
257}
258
259std::optional<TranslationUnitDeps>
260DependencyScanningTool::getTranslationUnitDependencies(
261 ArrayRef<std::string> CommandLine, StringRef CWD,
262 DiagnosticConsumer &DiagConsumer,
263 const llvm::DenseSet<ModuleID> &AlreadySeen,
264 LookupModuleOutputCallback LookupModuleOutput,
265 std::optional<llvm::MemoryBufferRef> TUBuffer) {
266 FullDependencyConsumer Consumer(AlreadySeen);
267 CallbackActionController Controller(LookupModuleOutput);
268
269 // If we are scanning from a TUBuffer, create an overlay filesystem with the
270 // input as an in-memory file and add it to the command line.
271 IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> OverlayFS = nullptr;
272 std::vector<std::string> CommandLineWithTUBufferInput;
273 if (TUBuffer) {
274 std::tie(args&: OverlayFS, args&: CommandLineWithTUBufferInput) =
275 initVFSForTUBufferScanning(BaseFS: &Worker.getVFS(), CommandLine, WorkingDirectory: CWD,
276 TUBuffer: *TUBuffer);
277 CommandLine = CommandLineWithTUBufferInput;
278 }
279
280 if (!computeDependencies(Worker, WorkingDirectory: CWD, CommandLine, Consumer, Controller,
281 DiagConsumer, OverlayFS))
282 return std::nullopt;
283 return Consumer.takeTranslationUnitDeps();
284}
285
286llvm::Expected<TranslationUnitDeps>
287DependencyScanningTool::getModuleDependencies(
288 StringRef ModuleName, ArrayRef<std::string> CommandLine, StringRef CWD,
289 const llvm::DenseSet<ModuleID> &AlreadySeen,
290 LookupModuleOutputCallback LookupModuleOutput) {
291 if (auto Error =
292 initializeCompilerInstanceWithContextOrError(CWD, CommandLine))
293 return Error;
294
295 auto Result = computeDependenciesByNameWithContextOrError(
296 ModuleName, AlreadySeen, LookupModuleOutput);
297
298 if (auto Error = finalizeCompilerInstanceWithContextOrError())
299 return Error;
300
301 return Result;
302}
303
304static std::optional<SmallVector<std::string, 0>> getFirstCC1CommandLine(
305 ArrayRef<std::string> CommandLine, DiagnosticsEngine &Diags,
306 llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> ScanFS) {
307 // Compilation holds a non-owning a reference to the Driver, hence we need to
308 // keep the Driver alive when we use Compilation. Arguments to commands may be
309 // owned by Alloc when expanded from response files.
310 llvm::BumpPtrAllocator Alloc;
311 const auto [Driver, Compilation] =
312 buildCompilation(ArgStrs: CommandLine, Diags, FS: ScanFS, Alloc);
313 if (!Compilation)
314 return std::nullopt;
315
316 const auto IsClangCmd = [](const driver::Command &Cmd) {
317 return StringRef(Cmd.getCreator().getName()) == "clang";
318 };
319
320 const auto &Jobs = Compilation->getJobs();
321 if (const auto It = llvm::find_if(Range: Jobs, P: IsClangCmd); It != Jobs.end())
322 return buildCC1CommandLine(Cmd: *It);
323 return std::nullopt;
324}
325
326bool DependencyScanningTool::initializeWorkerCIWithContextFromCommandline(
327 DependencyScanningWorker &Worker, StringRef CWD,
328 ArrayRef<std::string> CommandLine, DiagnosticConsumer &DC) {
329 if (CommandLine.size() >= 2 && CommandLine[1] == "-cc1") {
330 // The input command line is already a -cc1 invocation; initialize the
331 // compiler instance directly from it.
332 return Worker.initializeCompilerInstanceWithContext(CWD, CommandLine, DC);
333 }
334
335 // The input command line is either a driver-style command line, or
336 // ill-formed. In this case, we will first call the Driver to build a -cc1
337 // command line for this compilation or diagnose any ill-formed input.
338 auto [OverlayFS, ModifiedCommandLine] = initVFSForByNameScanning(
339 BaseFS: &Worker.getVFS(), CommandLine, WorkingDirectory: CWD, ModuleName: "ScanningByName");
340 auto DiagEngineWithCmdAndOpts =
341 std::make_unique<DiagnosticsEngineWithDiagOpts>(args&: ModifiedCommandLine,
342 args&: OverlayFS, args&: DC);
343
344 const auto MaybeFirstCC1 = getFirstCC1CommandLine(
345 CommandLine: ModifiedCommandLine, Diags&: *DiagEngineWithCmdAndOpts->DiagEngine, ScanFS: OverlayFS);
346 if (!MaybeFirstCC1)
347 return false;
348
349 return Worker.initializeCompilerInstanceWithContext(
350 CWD, CommandLine: *MaybeFirstCC1, DiagEngineWithCmdAndOpts: std::move(DiagEngineWithCmdAndOpts), OverlayFS);
351}
352
353llvm::Error
354DependencyScanningTool::initializeCompilerInstanceWithContextOrError(
355 StringRef CWD, ArrayRef<std::string> CommandLine) {
356 DiagPrinterWithOS =
357 std::make_unique<TextDiagnosticsPrinterWithOutput>(args&: CommandLine);
358
359 bool Result = initializeWorkerCIWithContextFromCommandline(
360 Worker, CWD, CommandLine, DC&: DiagPrinterWithOS->DiagPrinter);
361
362 if (Result)
363 return llvm::Error::success();
364 return makeErrorFromDiagnosticsOS(DiagPrinterWithOS&: *DiagPrinterWithOS);
365}
366
367llvm::Expected<TranslationUnitDeps>
368DependencyScanningTool::computeDependenciesByNameWithContextOrError(
369 StringRef ModuleName, const llvm::DenseSet<ModuleID> &AlreadySeen,
370 LookupModuleOutputCallback LookupModuleOutput) {
371 FullDependencyConsumer Consumer(AlreadySeen);
372 CallbackActionController Controller(LookupModuleOutput);
373 // We need to clear the DiagnosticOutput so that each by-name lookup
374 // has a clean diagnostics buffer.
375 DiagPrinterWithOS->DiagnosticOutput.clear();
376 if (Worker.computeDependenciesByNameWithContext(ModuleName, Consumer,
377 Controller))
378 return Consumer.takeTranslationUnitDeps();
379 return makeErrorFromDiagnosticsOS(DiagPrinterWithOS&: *DiagPrinterWithOS);
380}
381
382llvm::Error
383DependencyScanningTool::finalizeCompilerInstanceWithContextOrError() {
384 if (Worker.finalizeCompilerInstanceWithContext())
385 return llvm::Error::success();
386 return makeErrorFromDiagnosticsOS(DiagPrinterWithOS&: *DiagPrinterWithOS);
387}
388