1//===- DependencyScanningTool.cpp - clang-scan-deps service ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "clang/Tooling/DependencyScanningTool.h"
10#include "clang/Basic/Diagnostic.h"
11#include "clang/Basic/DiagnosticFrontend.h"
12#include "clang/DependencyScanning/DependencyScanningWorker.h"
13#include "clang/Driver/Compilation.h"
14#include "clang/Driver/Driver.h"
15#include "clang/Driver/Tool.h"
16#include "clang/Frontend/CompilerInstance.h"
17#include "clang/Frontend/Utils.h"
18#include "llvm/ADT/SmallVectorExtras.h"
19#include "llvm/ADT/iterator.h"
20#include "llvm/TargetParser/Host.h"
21#include <optional>
22
23using namespace clang;
24using namespace tooling;
25using namespace dependencies;
26
27namespace {
28/// Prints out all of the gathered dependencies into a string.
29class MakeDependencyPrinterConsumer : public DependencyConsumer {
30public:
31 void handleBuildCommand(Command) override {}
32
33 void
34 handleDependencyOutputOpts(const DependencyOutputOptions &Opts) override {
35 this->Opts = std::make_unique<DependencyOutputOptions>(args: Opts);
36 }
37
38 void handleFileDependency(StringRef File) override {
39 SmallString<128> NormalizedFile = File;
40 llvm::sys::path::remove_dots(path&: NormalizedFile, /*remove_dot_dot=*/true);
41 Dependencies.emplace_back(args: NormalizedFile.str());
42 }
43
44 // These are ignored for the make format as it can't support the full
45 // set of deps, and handleFileDependency handles enough for implicitly
46 // built modules to work.
47 void handlePrebuiltModuleDependency(PrebuiltModuleDep PMD) override {}
48 void handleModuleDependency(ModuleDeps MD) override {
49 MD.forEachFileDep(Cb: [this](StringRef File) {
50 DependenciesFromModules.push_back(x: std::string(File));
51 });
52 }
53 void handleDirectModuleDependency(ModuleID ID) override {}
54 void handleVisibleModule(std::string ModuleName) override {}
55 void handleContextHash(std::string Hash) override {}
56
57 void printDependencies(std::string &S) {
58 assert(Opts && "Handled dependency output options.");
59
60 class DependencyPrinter : public DependencyFileGenerator {
61 public:
62 DependencyPrinter(DependencyOutputOptions &Opts,
63 ArrayRef<std::string> Dependencies,
64 ArrayRef<std::string> ModuleDependencies)
65 : DependencyFileGenerator(Opts) {
66 for (const auto &Dep : Dependencies)
67 addDependency(Filename: Dep);
68 for (const auto &Dep : ModuleDependencies)
69 addDependency(Filename: Dep);
70 }
71
72 void printDependencies(std::string &S) {
73 llvm::raw_string_ostream OS(S);
74 outputDependencyFile(OS);
75 }
76 };
77
78 DependencyPrinter Generator(*Opts, Dependencies, DependenciesFromModules);
79 Generator.printDependencies(S);
80 }
81
82protected:
83 std::unique_ptr<DependencyOutputOptions> Opts;
84 std::vector<std::string> Dependencies;
85 std::vector<std::string> DependenciesFromModules;
86};
87} // anonymous namespace
88
89static std::pair<std::unique_ptr<driver::Driver>,
90 std::unique_ptr<driver::Compilation>>
91buildCompilation(ArrayRef<std::string> ArgStrs, DiagnosticsEngine &Diags,
92 IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
93 llvm::BumpPtrAllocator &Alloc) {
94 SmallVector<const char *, 256> Argv;
95 Argv.reserve(N: ArgStrs.size());
96 for (const std::string &Arg : ArgStrs)
97 Argv.push_back(Elt: Arg.c_str());
98
99 std::unique_ptr<driver::Driver> Driver = std::make_unique<driver::Driver>(
100 args&: Argv[0], args: llvm::sys::getDefaultTargetTriple(), args&: Diags,
101 args: "clang LLVM compiler", args&: FS);
102 Driver->setTitle("clang_based_tool");
103
104 bool CLMode = driver::IsClangCL(
105 DriverMode: driver::getDriverMode(ProgName: Argv[0], Args: ArrayRef(Argv).slice(N: 1)));
106
107 if (llvm::Error E =
108 driver::expandResponseFiles(Args&: Argv, ClangCLMode: CLMode, Alloc, FS: FS.get())) {
109 Diags.Report(DiagID: diag::err_drv_expand_response_file)
110 << llvm::toString(E: std::move(E));
111 return std::make_pair(x: nullptr, y: nullptr);
112 }
113
114 std::unique_ptr<driver::Compilation> Compilation(
115 Driver->BuildCompilation(Args: Argv));
116 if (!Compilation)
117 return std::make_pair(x: nullptr, y: nullptr);
118
119 if (Compilation->containsError())
120 return std::make_pair(x: nullptr, y: nullptr);
121
122 if (Compilation->getJobs().empty()) {
123 Diags.Report(DiagID: diag::err_fe_expected_compiler_job)
124 << llvm::join(R&: ArgStrs, Separator: " ");
125 return std::make_pair(x: nullptr, y: nullptr);
126 }
127
128 return std::make_pair(x: std::move(Driver), y: std::move(Compilation));
129}
130
131/// Constructs the full frontend command line, including executable, for the
132/// given driver \c Cmd.
133static SmallVector<std::string, 0>
134buildCC1CommandLine(const driver::Command &Cmd) {
135 const auto &Args = Cmd.getArguments();
136 SmallVector<std::string, 0> Out;
137 Out.reserve(N: Args.size() + 1);
138 Out.emplace_back(Args: Cmd.getExecutable());
139 llvm::append_range(C&: Out, R: Args);
140 return Out;
141}
142
143static bool computeDependenciesForDriverCommandLine(
144 DependencyScanningWorker &Worker, StringRef WorkingDirectory,
145 ArrayRef<std::string> CommandLine, DependencyConsumer &Consumer,
146 DependencyActionController &Controller, DiagnosticConsumer &DiagConsumer,
147 IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS) {
148 auto FS = Worker.makeEffectiveVFS(WorkingDirectory, OverlayFS);
149
150 // Compilation holds a non-owning a reference to the Driver, hence we need to
151 // keep the Driver alive when we use Compilation. Arguments to commands may be
152 // owned by Alloc when expanded from response files.
153 llvm::BumpPtrAllocator Alloc;
154 auto DiagOpts = createScanningDiagOptions(CommandLine);
155 auto DiagEngine =
156 CompilerInstance::createDiagnostics(VFS&: *FS, Opts&: *DiagOpts, Client: &DiagConsumer,
157 /*ShouldOwnClient=*/false);
158 const auto [Driver, Compilation] =
159 buildCompilation(ArgStrs: CommandLine, Diags&: *DiagEngine, FS, Alloc);
160 if (!Compilation)
161 return false;
162
163 SmallVector<SmallVector<std::string, 0>> FrontendCommandLines;
164 for (const auto &Cmd : Compilation->getJobs())
165 FrontendCommandLines.push_back(Elt: buildCC1CommandLine(Cmd));
166 SmallVector<ArrayRef<std::string>> FrontendCommandLinesView(
167 FrontendCommandLines.begin(), FrontendCommandLines.end());
168
169 return Worker.computeDependencies(WorkingDirectory, CommandLines: FrontendCommandLinesView,
170 DepConsumer&: Consumer, Controller, DiagConsumer,
171 OverlayFS: std::move(OverlayFS));
172}
173
174bool tooling::computeDependencies(
175 DependencyScanningWorker &Worker, StringRef WorkingDirectory,
176 ArrayRef<std::string> CommandLine, DependencyConsumer &Consumer,
177 DependencyActionController &Controller, DiagnosticConsumer &DiagConsumer,
178 IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS) {
179 const auto IsCC1Input = (CommandLine.size() >= 2 && CommandLine[1] == "-cc1");
180 return IsCC1Input ? Worker.computeDependencies(WorkingDirectory, CommandLines: CommandLine,
181 DepConsumer&: Consumer, Controller,
182 DiagConsumer, OverlayFS)
183 : computeDependenciesForDriverCommandLine(
184 Worker, WorkingDirectory, CommandLine, Consumer,
185 Controller, DiagConsumer, OverlayFS);
186}
187
188std::optional<std::string> DependencyScanningTool::getDependencyFile(
189 ArrayRef<std::string> CommandLine, StringRef CWD,
190 LookupModuleOutputCallback LookupModuleOutput,
191 DiagnosticConsumer &DiagConsumer) {
192 MakeDependencyPrinterConsumer DepConsumer;
193 CallbackActionController Controller(LookupModuleOutput);
194 if (!computeDependencies(Worker, WorkingDirectory: CWD, CommandLine, Consumer&: DepConsumer, Controller,
195 DiagConsumer))
196 return std::nullopt;
197 std::string Output;
198 DepConsumer.printDependencies(S&: Output);
199 return Output;
200}
201
202std::optional<P1689Rule> DependencyScanningTool::getP1689ModuleDependencyFile(
203 const CompileCommand &Command, StringRef CWD, std::string &MakeformatOutput,
204 std::string &MakeformatOutputPath, DiagnosticConsumer &DiagConsumer) {
205 class P1689ModuleDependencyPrinterConsumer
206 : public MakeDependencyPrinterConsumer {
207 public:
208 P1689ModuleDependencyPrinterConsumer(P1689Rule &Rule,
209 const CompileCommand &Command)
210 : Filename(Command.Filename), Rule(Rule) {
211 Rule.PrimaryOutput = Command.Output;
212 }
213
214 void handleProvidedAndRequiredStdCXXModules(
215 std::optional<P1689ModuleInfo> Provided,
216 std::vector<P1689ModuleInfo> Requires) override {
217 Rule.Provides = std::move(Provided);
218 if (Rule.Provides)
219 Rule.Provides->SourcePath = Filename.str();
220 Rule.Requires = std::move(Requires);
221 }
222
223 StringRef getMakeFormatDependencyOutputPath() {
224 if (Opts->OutputFormat != DependencyOutputFormat::Make)
225 return {};
226 return Opts->OutputFile;
227 }
228
229 private:
230 StringRef Filename;
231 P1689Rule &Rule;
232 };
233
234 class P1689ActionController : public DependencyActionController {
235 public:
236 // The lookupModuleOutput is for clang modules. P1689 format don't need it.
237 std::string lookupModuleOutput(const ModuleDeps &,
238 ModuleOutputKind Kind) override {
239 return "";
240 }
241
242 std::unique_ptr<DependencyActionController> clone() const override {
243 return std::make_unique<P1689ActionController>();
244 }
245 };
246
247 P1689Rule Rule;
248 P1689ModuleDependencyPrinterConsumer Consumer(Rule, Command);
249 P1689ActionController Controller;
250 if (!computeDependencies(Worker, WorkingDirectory: CWD, CommandLine: Command.CommandLine, Consumer,
251 Controller, DiagConsumer))
252 return std::nullopt;
253
254 MakeformatOutputPath = Consumer.getMakeFormatDependencyOutputPath();
255 if (!MakeformatOutputPath.empty())
256 Consumer.printDependencies(S&: MakeformatOutput);
257 return Rule;
258}
259
260static std::pair<IntrusiveRefCntPtr<llvm::vfs::FileSystem>,
261 std::vector<std::string>>
262initVFSForTUBufferScanning(ArrayRef<std::string> CommandLine,
263 llvm::MemoryBufferRef TUBuffer) {
264 StringRef InputPath = TUBuffer.getBufferIdentifier();
265 auto InputBuf = llvm::MemoryBuffer::getMemBufferCopy(InputData: TUBuffer.getBuffer());
266
267 auto FS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
268 FS->addFile(Path: InputPath, ModificationTime: 0, Buffer: std::move(InputBuf));
269
270 std::vector<std::string> ModifiedCommandLine(CommandLine);
271 ModifiedCommandLine.emplace_back(args&: InputPath);
272
273 return std::make_pair(x: std::move(FS), y&: ModifiedCommandLine);
274}
275
276static std::pair<IntrusiveRefCntPtr<llvm::vfs::FileSystem>,
277 std::vector<std::string>>
278initVFSForByNameScanning(ArrayRef<std::string> CommandLine) {
279 // The fake input buffer is read-only, and it is used to produce unique source
280 // locations for the diagnostics. Therefore, sharing this global buffer across
281 // threads is ok.
282 static const std::string FakeInput(
283 DependencyScanningWorker::MaxNumOfByNameQueries, ' ');
284
285 StringRef InputPath =
286 llvm::sys::path::is_style_windows(S: llvm::sys::path::Style::native)
287 ? "Z:\\module-include.input"
288 : "/module-include.input";
289 auto InputBuf = llvm::MemoryBuffer::getMemBuffer(InputData: FakeInput, BufferName: InputPath);
290
291 auto FS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
292 FS->addFile(Path: InputPath, ModificationTime: 0, Buffer: std::move(InputBuf));
293
294 std::vector<std::string> ModifiedCommandLine(CommandLine);
295 ModifiedCommandLine.emplace_back(args&: InputPath);
296
297 return std::make_pair(x: std::move(FS), y&: ModifiedCommandLine);
298}
299
300std::optional<TranslationUnitDeps>
301DependencyScanningTool::getTranslationUnitDependencies(
302 ArrayRef<std::string> CommandLine, StringRef CWD,
303 DiagnosticConsumer &DiagConsumer,
304 const llvm::DenseSet<ModuleID> &AlreadySeen,
305 LookupModuleOutputCallback LookupModuleOutput,
306 std::optional<llvm::MemoryBufferRef> TUBuffer) {
307 FullDependencyConsumer Consumer(AlreadySeen);
308 CallbackActionController Controller(LookupModuleOutput);
309
310 // If we are scanning from a TUBuffer, create an overlay filesystem with the
311 // input as an in-memory file and add it to the command line.
312 IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS = nullptr;
313 std::vector<std::string> CommandLineWithTUBufferInput;
314 if (TUBuffer) {
315 std::tie(args&: OverlayFS, args&: CommandLineWithTUBufferInput) =
316 initVFSForTUBufferScanning(CommandLine, TUBuffer: *TUBuffer);
317 CommandLine = CommandLineWithTUBufferInput;
318 }
319
320 if (!computeDependencies(Worker, WorkingDirectory: CWD, CommandLine, Consumer, Controller,
321 DiagConsumer, OverlayFS: std::move(OverlayFS)))
322 return std::nullopt;
323 return Consumer.takeTranslationUnitDeps();
324}
325
326static std::optional<SmallVector<std::string, 0>>
327getFirstCC1CommandLine(ArrayRef<std::string> CommandLine,
328 DiagnosticsEngine &Diags,
329 llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS) {
330 // Compilation holds a non-owning a reference to the Driver, hence we need to
331 // keep the Driver alive when we use Compilation. Arguments to commands may be
332 // owned by Alloc when expanded from response files.
333 llvm::BumpPtrAllocator Alloc;
334 const auto [Driver, Compilation] =
335 buildCompilation(ArgStrs: CommandLine, Diags, FS: std::move(FS), Alloc);
336 if (!Compilation)
337 return std::nullopt;
338
339 const auto IsClangCmd = [](const driver::Command &Cmd) {
340 return StringRef(Cmd.getCreator().getName()) == "clang";
341 };
342
343 const auto &Jobs = Compilation->getJobs();
344 if (const auto It = llvm::find_if(Range: Jobs, P: IsClangCmd); It != Jobs.end())
345 return buildCC1CommandLine(Cmd: *It);
346 return std::nullopt;
347}
348
349bool DependencyScanningTool::getByNameDependencies(
350 StringRef CWD, ArrayRef<std::string> CommandLine,
351 DiagnosticConsumer &DiagConsumer, DependencyActionController &Controller,
352 llvm::function_ref<std::optional<std::string>()> getNextName,
353 DependencyConsumer &DepConsumer) {
354 auto [OverlayFS, ModifiedCommandLine] = initVFSForByNameScanning(CommandLine);
355 auto FS = Worker.makeEffectiveVFS(WorkingDirectory: CWD, OverlayFS);
356 std::vector<std::string> CC1CommandLine;
357 if (ModifiedCommandLine.size() >= 2 && ModifiedCommandLine[1] == "-cc1") {
358 CC1CommandLine = std::move(ModifiedCommandLine);
359 } else {
360 // Driver-style (or ill-formed): lower to a cc1 command line, or diagnose.
361 auto DiagOpts = createScanningDiagOptions(CommandLine: ModifiedCommandLine);
362 auto DiagEngine =
363 CompilerInstance::createDiagnostics(VFS&: *FS, Opts&: *DiagOpts, Client: &DiagConsumer,
364 /*ShouldOwnClient=*/false);
365 auto MaybeFirstCC1 =
366 getFirstCC1CommandLine(CommandLine: ModifiedCommandLine, Diags&: *DiagEngine, FS);
367 if (!MaybeFirstCC1)
368 return false;
369 CC1CommandLine.assign(first: MaybeFirstCC1->begin(), last: MaybeFirstCC1->end());
370 }
371
372 return Worker.computeDependenciesByName(CWD, CC1CommandLine,
373 OverlayFS: std::move(OverlayFS), DiagConsumer,
374 Controller, getNextName, DepConsumer);
375}
376