1//===- DependencyScanningWorker.cpp - clang-scan-deps worker --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h"
10#include "clang/Basic/DiagnosticDriver.h"
11#include "clang/Basic/DiagnosticFrontend.h"
12#include "clang/Basic/DiagnosticSerialization.h"
13#include "clang/Driver/Compilation.h"
14#include "clang/Driver/Driver.h"
15#include "clang/Driver/Job.h"
16#include "clang/Driver/Tool.h"
17#include "clang/Frontend/CompilerInstance.h"
18#include "clang/Frontend/CompilerInvocation.h"
19#include "clang/Frontend/FrontendActions.h"
20#include "clang/Frontend/TextDiagnosticPrinter.h"
21#include "clang/Frontend/Utils.h"
22#include "clang/Lex/PreprocessorOptions.h"
23#include "clang/Serialization/ObjectFilePCHContainerReader.h"
24#include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
25#include "clang/Tooling/DependencyScanning/InProcessModuleCache.h"
26#include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
27#include "clang/Tooling/Tooling.h"
28#include "llvm/ADT/IntrusiveRefCntPtr.h"
29#include "llvm/Support/Allocator.h"
30#include "llvm/Support/Error.h"
31#include "llvm/Support/MemoryBuffer.h"
32#include "llvm/TargetParser/Host.h"
33#include <optional>
34
35using namespace clang;
36using namespace tooling;
37using namespace dependencies;
38
39namespace {
40
41/// Forwards the gatherered dependencies to the consumer.
42class DependencyConsumerForwarder : public DependencyFileGenerator {
43public:
44 DependencyConsumerForwarder(std::unique_ptr<DependencyOutputOptions> Opts,
45 StringRef WorkingDirectory, DependencyConsumer &C)
46 : DependencyFileGenerator(*Opts), WorkingDirectory(WorkingDirectory),
47 Opts(std::move(Opts)), C(C) {}
48
49 void finishedMainFile(DiagnosticsEngine &Diags) override {
50 C.handleDependencyOutputOpts(Opts: *Opts);
51 llvm::SmallString<256> CanonPath;
52 for (const auto &File : getDependencies()) {
53 CanonPath = File;
54 llvm::sys::path::remove_dots(path&: CanonPath, /*remove_dot_dot=*/true);
55 llvm::sys::fs::make_absolute(current_directory: WorkingDirectory, path&: CanonPath);
56 C.handleFileDependency(Filename: CanonPath);
57 }
58 }
59
60private:
61 StringRef WorkingDirectory;
62 std::unique_ptr<DependencyOutputOptions> Opts;
63 DependencyConsumer &C;
64};
65
66static bool checkHeaderSearchPaths(const HeaderSearchOptions &HSOpts,
67 const HeaderSearchOptions &ExistingHSOpts,
68 DiagnosticsEngine *Diags,
69 const LangOptions &LangOpts) {
70 if (LangOpts.Modules) {
71 if (HSOpts.VFSOverlayFiles != ExistingHSOpts.VFSOverlayFiles) {
72 if (Diags) {
73 Diags->Report(DiagID: diag::warn_pch_vfsoverlay_mismatch);
74 auto VFSNote = [&](int Type, ArrayRef<std::string> VFSOverlays) {
75 if (VFSOverlays.empty()) {
76 Diags->Report(DiagID: diag::note_pch_vfsoverlay_empty) << Type;
77 } else {
78 std::string Files = llvm::join(R&: VFSOverlays, Separator: "\n");
79 Diags->Report(DiagID: diag::note_pch_vfsoverlay_files) << Type << Files;
80 }
81 };
82 VFSNote(0, HSOpts.VFSOverlayFiles);
83 VFSNote(1, ExistingHSOpts.VFSOverlayFiles);
84 }
85 }
86 }
87 return false;
88}
89
90using PrebuiltModuleFilesT = decltype(HeaderSearchOptions::PrebuiltModuleFiles);
91
92/// A listener that collects the imported modules and the input
93/// files. While visiting, collect vfsoverlays and file inputs that determine
94/// whether prebuilt modules fully resolve in stable directories.
95class PrebuiltModuleListener : public ASTReaderListener {
96public:
97 PrebuiltModuleListener(PrebuiltModuleFilesT &PrebuiltModuleFiles,
98 llvm::SmallVector<std::string> &NewModuleFiles,
99 PrebuiltModulesAttrsMap &PrebuiltModulesASTMap,
100 const HeaderSearchOptions &HSOpts,
101 const LangOptions &LangOpts, DiagnosticsEngine &Diags,
102 const ArrayRef<StringRef> StableDirs)
103 : PrebuiltModuleFiles(PrebuiltModuleFiles),
104 NewModuleFiles(NewModuleFiles),
105 PrebuiltModulesASTMap(PrebuiltModulesASTMap), ExistingHSOpts(HSOpts),
106 ExistingLangOpts(LangOpts), Diags(Diags), StableDirs(StableDirs) {}
107
108 bool needsImportVisitation() const override { return true; }
109 bool needsInputFileVisitation() override { return true; }
110 bool needsSystemInputFileVisitation() override { return true; }
111
112 /// Accumulate the modules are transitively depended on by the initial
113 /// prebuilt module.
114 void visitImport(StringRef ModuleName, StringRef Filename) override {
115 if (PrebuiltModuleFiles.insert(x: {ModuleName.str(), Filename.str()}).second)
116 NewModuleFiles.push_back(Elt: Filename.str());
117
118 auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(Key: Filename);
119 PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second;
120 if (PrebuiltMapEntry.second)
121 PrebuiltModule.setInStableDir(!StableDirs.empty());
122
123 if (auto It = PrebuiltModulesASTMap.find(Key: CurrentFile);
124 It != PrebuiltModulesASTMap.end() && CurrentFile != Filename)
125 PrebuiltModule.addDependent(ModuleFile: It->getKey());
126 }
127
128 /// For each input file discovered, check whether it's external path is in a
129 /// stable directory. Traversal is stopped if the current module is not
130 /// considered stable.
131 bool visitInputFile(StringRef FilenameAsRequested, StringRef Filename,
132 bool isSystem, bool isOverridden,
133 bool isExplicitModule) override {
134 if (StableDirs.empty())
135 return false;
136 auto PrebuiltEntryIt = PrebuiltModulesASTMap.find(Key: CurrentFile);
137 if ((PrebuiltEntryIt == PrebuiltModulesASTMap.end()) ||
138 (!PrebuiltEntryIt->second.isInStableDir()))
139 return false;
140
141 PrebuiltEntryIt->second.setInStableDir(
142 isPathInStableDir(Directories: StableDirs, Input: Filename));
143 return PrebuiltEntryIt->second.isInStableDir();
144 }
145
146 /// Update which module that is being actively traversed.
147 void visitModuleFile(StringRef Filename,
148 serialization::ModuleKind Kind) override {
149 // If the CurrentFile is not
150 // considered stable, update any of it's transitive dependents.
151 auto PrebuiltEntryIt = PrebuiltModulesASTMap.find(Key: CurrentFile);
152 if ((PrebuiltEntryIt != PrebuiltModulesASTMap.end()) &&
153 !PrebuiltEntryIt->second.isInStableDir())
154 PrebuiltEntryIt->second.updateDependentsNotInStableDirs(
155 PrebuiltModulesMap&: PrebuiltModulesASTMap);
156 CurrentFile = Filename;
157 }
158
159 /// Check the header search options for a given module when considering
160 /// if the module comes from stable directories.
161 bool ReadHeaderSearchOptions(const HeaderSearchOptions &HSOpts,
162 StringRef ModuleFilename,
163 StringRef SpecificModuleCachePath,
164 bool Complain) override {
165
166 auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(Key: CurrentFile);
167 PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second;
168 if (PrebuiltMapEntry.second)
169 PrebuiltModule.setInStableDir(!StableDirs.empty());
170
171 if (PrebuiltModule.isInStableDir())
172 PrebuiltModule.setInStableDir(areOptionsInStableDir(Directories: StableDirs, HSOpts));
173
174 return false;
175 }
176
177 /// Accumulate vfsoverlays used to build these prebuilt modules.
178 bool ReadHeaderSearchPaths(const HeaderSearchOptions &HSOpts,
179 bool Complain) override {
180
181 auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(Key: CurrentFile);
182 PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second;
183 if (PrebuiltMapEntry.second)
184 PrebuiltModule.setInStableDir(!StableDirs.empty());
185
186 PrebuiltModule.setVFS(
187 llvm::StringSet<>(llvm::from_range, HSOpts.VFSOverlayFiles));
188
189 return checkHeaderSearchPaths(
190 HSOpts, ExistingHSOpts, Diags: Complain ? &Diags : nullptr, LangOpts: ExistingLangOpts);
191 }
192
193private:
194 PrebuiltModuleFilesT &PrebuiltModuleFiles;
195 llvm::SmallVector<std::string> &NewModuleFiles;
196 PrebuiltModulesAttrsMap &PrebuiltModulesASTMap;
197 const HeaderSearchOptions &ExistingHSOpts;
198 const LangOptions &ExistingLangOpts;
199 DiagnosticsEngine &Diags;
200 std::string CurrentFile;
201 const ArrayRef<StringRef> StableDirs;
202};
203
204/// Visit the given prebuilt module and collect all of the modules it
205/// transitively imports and contributing input files.
206static bool visitPrebuiltModule(StringRef PrebuiltModuleFilename,
207 CompilerInstance &CI,
208 PrebuiltModuleFilesT &ModuleFiles,
209 PrebuiltModulesAttrsMap &PrebuiltModulesASTMap,
210 DiagnosticsEngine &Diags,
211 const ArrayRef<StringRef> StableDirs) {
212 // List of module files to be processed.
213 llvm::SmallVector<std::string> Worklist;
214
215 PrebuiltModuleListener Listener(ModuleFiles, Worklist, PrebuiltModulesASTMap,
216 CI.getHeaderSearchOpts(), CI.getLangOpts(),
217 Diags, StableDirs);
218
219 Listener.visitModuleFile(Filename: PrebuiltModuleFilename,
220 Kind: serialization::MK_ExplicitModule);
221 if (ASTReader::readASTFileControlBlock(
222 Filename: PrebuiltModuleFilename, FileMgr&: CI.getFileManager(), ModCache: CI.getModuleCache(),
223 PCHContainerRdr: CI.getPCHContainerReader(),
224 /*FindModuleFileExtensions=*/false, Listener,
225 /*ValidateDiagnosticOptions=*/false, ClientLoadCapabilities: ASTReader::ARR_OutOfDate))
226 return true;
227
228 while (!Worklist.empty()) {
229 Listener.visitModuleFile(Filename: Worklist.back(), Kind: serialization::MK_ExplicitModule);
230 if (ASTReader::readASTFileControlBlock(
231 Filename: Worklist.pop_back_val(), FileMgr&: CI.getFileManager(), ModCache: CI.getModuleCache(),
232 PCHContainerRdr: CI.getPCHContainerReader(),
233 /*FindModuleFileExtensions=*/false, Listener,
234 /*ValidateDiagnosticOptions=*/false))
235 return true;
236 }
237 return false;
238}
239
240/// Transform arbitrary file name into an object-like file name.
241static std::string makeObjFileName(StringRef FileName) {
242 SmallString<128> ObjFileName(FileName);
243 llvm::sys::path::replace_extension(path&: ObjFileName, extension: "o");
244 return std::string(ObjFileName);
245}
246
247/// Deduce the dependency target based on the output file and input files.
248static std::string
249deduceDepTarget(const std::string &OutputFile,
250 const SmallVectorImpl<FrontendInputFile> &InputFiles) {
251 if (OutputFile != "-")
252 return OutputFile;
253
254 if (InputFiles.empty() || !InputFiles.front().isFile())
255 return "clang-scan-deps\\ dependency";
256
257 return makeObjFileName(FileName: InputFiles.front().getFile());
258}
259
260/// Sanitize diagnostic options for dependency scan.
261static void sanitizeDiagOpts(DiagnosticOptions &DiagOpts) {
262 // Don't print 'X warnings and Y errors generated'.
263 DiagOpts.ShowCarets = false;
264 // Don't write out diagnostic file.
265 DiagOpts.DiagnosticSerializationFile.clear();
266 // Don't emit warnings except for scanning specific warnings.
267 // TODO: It would be useful to add a more principled way to ignore all
268 // warnings that come from source code. The issue is that we need to
269 // ignore warnings that could be surpressed by
270 // `#pragma clang diagnostic`, while still allowing some scanning
271 // warnings for things we're not ready to turn into errors yet.
272 // See `test/ClangScanDeps/diagnostic-pragmas.c` for an example.
273 llvm::erase_if(C&: DiagOpts.Warnings, P: [](StringRef Warning) {
274 return llvm::StringSwitch<bool>(Warning)
275 .Cases(S0: "pch-vfs-diff", S1: "error=pch-vfs-diff", Value: false)
276 .StartsWith(S: "no-error=", Value: false)
277 .Default(Value: true);
278 });
279}
280
281// Clang implements -D and -U by splatting text into a predefines buffer. This
282// allows constructs such as `-DFඞ=3 "-D F\u{0D9E} 4 3 2”` to be accepted and
283// define the same macro, or adding C++ style comments before the macro name.
284//
285// This function checks that the first non-space characters in the macro
286// obviously form an identifier that can be uniqued on without lexing. Failing
287// to do this could lead to changing the final definition of a macro.
288//
289// We could set up a preprocessor and actually lex the name, but that's very
290// heavyweight for a situation that will almost never happen in practice.
291static std::optional<StringRef> getSimpleMacroName(StringRef Macro) {
292 StringRef Name = Macro.split(Separator: "=").first.ltrim(Chars: " \t");
293 std::size_t I = 0;
294
295 auto FinishName = [&]() -> std::optional<StringRef> {
296 StringRef SimpleName = Name.slice(Start: 0, End: I);
297 if (SimpleName.empty())
298 return std::nullopt;
299 return SimpleName;
300 };
301
302 for (; I != Name.size(); ++I) {
303 switch (Name[I]) {
304 case '(': // Start of macro parameter list
305 case ' ': // End of macro name
306 case '\t':
307 return FinishName();
308 case '_':
309 continue;
310 default:
311 if (llvm::isAlnum(C: Name[I]))
312 continue;
313 return std::nullopt;
314 }
315 }
316 return FinishName();
317}
318
319static void canonicalizeDefines(PreprocessorOptions &PPOpts) {
320 using MacroOpt = std::pair<StringRef, std::size_t>;
321 std::vector<MacroOpt> SimpleNames;
322 SimpleNames.reserve(n: PPOpts.Macros.size());
323 std::size_t Index = 0;
324 for (const auto &M : PPOpts.Macros) {
325 auto SName = getSimpleMacroName(Macro: M.first);
326 // Skip optimizing if we can't guarantee we can preserve relative order.
327 if (!SName)
328 return;
329 SimpleNames.emplace_back(args&: *SName, args&: Index);
330 ++Index;
331 }
332
333 llvm::stable_sort(Range&: SimpleNames, C: llvm::less_first());
334 // Keep the last instance of each macro name by going in reverse
335 auto NewEnd = std::unique(
336 first: SimpleNames.rbegin(), last: SimpleNames.rend(),
337 binary_pred: [](const MacroOpt &A, const MacroOpt &B) { return A.first == B.first; });
338 SimpleNames.erase(first: SimpleNames.begin(), last: NewEnd.base());
339
340 // Apply permutation.
341 decltype(PPOpts.Macros) NewMacros;
342 NewMacros.reserve(n: SimpleNames.size());
343 for (std::size_t I = 0, E = SimpleNames.size(); I != E; ++I) {
344 std::size_t OriginalIndex = SimpleNames[I].second;
345 // We still emit undefines here as they may be undefining a predefined macro
346 NewMacros.push_back(x: std::move(PPOpts.Macros[OriginalIndex]));
347 }
348 std::swap(x&: PPOpts.Macros, y&: NewMacros);
349}
350
351class ScanningDependencyDirectivesGetter : public DependencyDirectivesGetter {
352 DependencyScanningWorkerFilesystem *DepFS;
353
354public:
355 ScanningDependencyDirectivesGetter(FileManager &FileMgr) : DepFS(nullptr) {
356 FileMgr.getVirtualFileSystem().visit(Callback: [&](llvm::vfs::FileSystem &FS) {
357 auto *DFS = llvm::dyn_cast<DependencyScanningWorkerFilesystem>(Val: &FS);
358 if (DFS) {
359 assert(!DepFS && "Found multiple scanning VFSs");
360 DepFS = DFS;
361 }
362 });
363 assert(DepFS && "Did not find scanning VFS");
364 }
365
366 std::unique_ptr<DependencyDirectivesGetter>
367 cloneFor(FileManager &FileMgr) override {
368 return std::make_unique<ScanningDependencyDirectivesGetter>(args&: FileMgr);
369 }
370
371 std::optional<ArrayRef<dependency_directives_scan::Directive>>
372 operator()(FileEntryRef File) override {
373 return DepFS->getDirectiveTokens(Path: File.getName());
374 }
375};
376
377/// A clang tool that runs the preprocessor in a mode that's optimized for
378/// dependency scanning for the given compiler invocation.
379class DependencyScanningAction : public tooling::ToolAction {
380public:
381 DependencyScanningAction(
382 DependencyScanningService &Service, StringRef WorkingDirectory,
383 DependencyConsumer &Consumer, DependencyActionController &Controller,
384 llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS,
385 bool DisableFree, std::optional<StringRef> ModuleName = std::nullopt)
386 : Service(Service), WorkingDirectory(WorkingDirectory),
387 Consumer(Consumer), Controller(Controller), DepFS(std::move(DepFS)),
388 DisableFree(DisableFree), ModuleName(ModuleName) {}
389
390 bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation,
391 FileManager *DriverFileMgr,
392 std::shared_ptr<PCHContainerOperations> PCHContainerOps,
393 DiagnosticConsumer *DiagConsumer) override {
394 // Make a deep copy of the original Clang invocation.
395 CompilerInvocation OriginalInvocation(*Invocation);
396 // Restore the value of DisableFree, which may be modified by Tooling.
397 OriginalInvocation.getFrontendOpts().DisableFree = DisableFree;
398 if (any(Val: Service.getOptimizeArgs() & ScanningOptimizations::Macros))
399 canonicalizeDefines(PPOpts&: OriginalInvocation.getPreprocessorOpts());
400
401 if (Scanned) {
402 // Scanning runs once for the first -cc1 invocation in a chain of driver
403 // jobs. For any dependent jobs, reuse the scanning result and just
404 // update the LastCC1Arguments to correspond to the new invocation.
405 // FIXME: to support multi-arch builds, each arch requires a separate scan
406 setLastCC1Arguments(std::move(OriginalInvocation));
407 return true;
408 }
409
410 Scanned = true;
411
412 // Create a compiler instance to handle the actual work.
413 auto ModCache = makeInProcessModuleCache(Entries&: Service.getModuleCacheEntries());
414 ScanInstanceStorage.emplace(args: std::move(Invocation),
415 args: std::move(PCHContainerOps), args: ModCache.get());
416 CompilerInstance &ScanInstance = *ScanInstanceStorage;
417 ScanInstance.setBuildingModule(false);
418
419 // Create the compiler's actual diagnostics engine.
420 sanitizeDiagOpts(DiagOpts&: ScanInstance.getDiagnosticOpts());
421 assert(!DiagConsumerFinished && "attempt to reuse finished consumer");
422 ScanInstance.createDiagnostics(VFS&: DriverFileMgr->getVirtualFileSystem(),
423 Client: DiagConsumer, /*ShouldOwnClient=*/false);
424 if (!ScanInstance.hasDiagnostics())
425 return false;
426
427 ScanInstance.getPreprocessorOpts().AllowPCHWithDifferentModulesCachePath =
428 true;
429
430 if (ScanInstance.getHeaderSearchOpts().ModulesValidateOncePerBuildSession)
431 ScanInstance.getHeaderSearchOpts().BuildSessionTimestamp =
432 Service.getBuildSessionTimestamp();
433
434 ScanInstance.getFrontendOpts().GenerateGlobalModuleIndex = false;
435 ScanInstance.getFrontendOpts().UseGlobalModuleIndex = false;
436 // This will prevent us compiling individual modules asynchronously since
437 // FileManager is not thread-safe, but it does improve performance for now.
438 ScanInstance.getFrontendOpts().ModulesShareFileManager = true;
439 ScanInstance.getHeaderSearchOpts().ModuleFormat = "raw";
440 ScanInstance.getHeaderSearchOpts().ModulesIncludeVFSUsage =
441 any(Val: Service.getOptimizeArgs() & ScanningOptimizations::VFS);
442
443 // Support for virtual file system overlays.
444 auto FS = createVFSFromCompilerInvocation(
445 CI: ScanInstance.getInvocation(), Diags&: ScanInstance.getDiagnostics(),
446 BaseFS: DriverFileMgr->getVirtualFileSystemPtr());
447
448 // Create a new FileManager to match the invocation's FileSystemOptions.
449 auto *FileMgr = ScanInstance.createFileManager(VFS: FS);
450
451 // Use the dependency scanning optimized file system if requested to do so.
452 if (DepFS) {
453 StringRef ModulesCachePath =
454 ScanInstance.getHeaderSearchOpts().ModuleCachePath;
455
456 DepFS->resetBypassedPathPrefix();
457 if (!ModulesCachePath.empty())
458 DepFS->setBypassedPathPrefix(ModulesCachePath);
459
460 ScanInstance.setDependencyDirectivesGetter(
461 std::make_unique<ScanningDependencyDirectivesGetter>(args&: *FileMgr));
462 }
463
464 ScanInstance.createSourceManager(FileMgr&: *FileMgr);
465
466 // Create a collection of stable directories derived from the ScanInstance
467 // for determining whether module dependencies would fully resolve from
468 // those directories.
469 llvm::SmallVector<StringRef> StableDirs;
470 const StringRef Sysroot = ScanInstance.getHeaderSearchOpts().Sysroot;
471 if (!Sysroot.empty() &&
472 (llvm::sys::path::root_directory(path: Sysroot) != Sysroot))
473 StableDirs = {Sysroot, ScanInstance.getHeaderSearchOpts().ResourceDir};
474
475 // Store a mapping of prebuilt module files and their properties like header
476 // search options. This will prevent the implicit build to create duplicate
477 // modules and will force reuse of the existing prebuilt module files
478 // instead.
479 PrebuiltModulesAttrsMap PrebuiltModulesASTMap;
480
481 if (!ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty())
482 if (visitPrebuiltModule(
483 PrebuiltModuleFilename: ScanInstance.getPreprocessorOpts().ImplicitPCHInclude,
484 CI&: ScanInstance,
485 ModuleFiles&: ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles,
486 PrebuiltModulesASTMap, Diags&: ScanInstance.getDiagnostics(), StableDirs))
487 return false;
488
489 // Create the dependency collector that will collect the produced
490 // dependencies.
491 //
492 // This also moves the existing dependency output options from the
493 // invocation to the collector. The options in the invocation are reset,
494 // which ensures that the compiler won't create new dependency collectors,
495 // and thus won't write out the extra '.d' files to disk.
496 auto Opts = std::make_unique<DependencyOutputOptions>();
497 std::swap(a&: *Opts, b&: ScanInstance.getInvocation().getDependencyOutputOpts());
498 // We need at least one -MT equivalent for the generator of make dependency
499 // files to work.
500 if (Opts->Targets.empty())
501 Opts->Targets = {
502 deduceDepTarget(OutputFile: ScanInstance.getFrontendOpts().OutputFile,
503 InputFiles: ScanInstance.getFrontendOpts().Inputs)};
504 Opts->IncludeSystemHeaders = true;
505
506 switch (Service.getFormat()) {
507 case ScanningOutputFormat::Make:
508 ScanInstance.addDependencyCollector(
509 Listener: std::make_shared<DependencyConsumerForwarder>(
510 args: std::move(Opts), args&: WorkingDirectory, args&: Consumer));
511 break;
512 case ScanningOutputFormat::P1689:
513 case ScanningOutputFormat::Full:
514 MDC = std::make_shared<ModuleDepCollector>(
515 args&: Service, args: std::move(Opts), args&: ScanInstance, args&: Consumer, args&: Controller,
516 args&: OriginalInvocation, args: std::move(PrebuiltModulesASTMap), args&: StableDirs);
517 ScanInstance.addDependencyCollector(Listener: MDC);
518 break;
519 }
520
521 // Consider different header search and diagnostic options to create
522 // different modules. This avoids the unsound aliasing of module PCMs.
523 //
524 // TODO: Implement diagnostic bucketing to reduce the impact of strict
525 // context hashing.
526 ScanInstance.getHeaderSearchOpts().ModulesStrictContextHash = true;
527 ScanInstance.getHeaderSearchOpts().ModulesSerializeOnlyPreprocessor = true;
528 ScanInstance.getHeaderSearchOpts().ModulesSkipDiagnosticOptions = true;
529 ScanInstance.getHeaderSearchOpts().ModulesSkipHeaderSearchPaths = true;
530 ScanInstance.getHeaderSearchOpts().ModulesSkipPragmaDiagnosticMappings =
531 true;
532 ScanInstance.getHeaderSearchOpts().ModulesForceValidateUserHeaders = false;
533
534 // Avoid some checks and module map parsing when loading PCM files.
535 ScanInstance.getPreprocessorOpts().ModulesCheckRelocated = false;
536
537 std::unique_ptr<FrontendAction> Action;
538
539 if (Service.getFormat() == ScanningOutputFormat::P1689)
540 Action = std::make_unique<PreprocessOnlyAction>();
541 else if (ModuleName)
542 Action = std::make_unique<GetDependenciesByModuleNameAction>(args&: *ModuleName);
543 else
544 Action = std::make_unique<ReadPCHAndPreprocessAction>();
545
546 if (ScanInstance.getDiagnostics().hasErrorOccurred())
547 return false;
548
549 const bool Result = ScanInstance.ExecuteAction(Act&: *Action);
550
551 // ExecuteAction is responsible for calling finish.
552 DiagConsumerFinished = true;
553
554 if (Result)
555 setLastCC1Arguments(std::move(OriginalInvocation));
556
557 // Propagate the statistics to the parent FileManager.
558 DriverFileMgr->AddStats(Other: ScanInstance.getFileManager());
559
560 return Result;
561 }
562
563 bool hasScanned() const { return Scanned; }
564 bool hasDiagConsumerFinished() const { return DiagConsumerFinished; }
565
566 /// Take the cc1 arguments corresponding to the most recent invocation used
567 /// with this action. Any modifications implied by the discovered dependencies
568 /// will have already been applied.
569 std::vector<std::string> takeLastCC1Arguments() {
570 std::vector<std::string> Result;
571 std::swap(x&: Result, y&: LastCC1Arguments); // Reset LastCC1Arguments to empty.
572 return Result;
573 }
574
575private:
576 void setLastCC1Arguments(CompilerInvocation &&CI) {
577 if (MDC)
578 MDC->applyDiscoveredDependencies(CI);
579 LastCC1Arguments = CI.getCC1CommandLine();
580 }
581
582 DependencyScanningService &Service;
583 StringRef WorkingDirectory;
584 DependencyConsumer &Consumer;
585 DependencyActionController &Controller;
586 llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS;
587 bool DisableFree;
588 std::optional<StringRef> ModuleName;
589 std::optional<CompilerInstance> ScanInstanceStorage;
590 std::shared_ptr<ModuleDepCollector> MDC;
591 std::vector<std::string> LastCC1Arguments;
592 bool Scanned = false;
593 bool DiagConsumerFinished = false;
594};
595
596} // end anonymous namespace
597
598DependencyScanningWorker::DependencyScanningWorker(
599 DependencyScanningService &Service,
600 llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS)
601 : Service(Service) {
602 PCHContainerOps = std::make_shared<PCHContainerOperations>();
603 // We need to read object files from PCH built outside the scanner.
604 PCHContainerOps->registerReader(
605 Reader: std::make_unique<ObjectFilePCHContainerReader>());
606 // The scanner itself writes only raw ast files.
607 PCHContainerOps->registerWriter(Writer: std::make_unique<RawPCHContainerWriter>());
608
609 if (Service.shouldTraceVFS())
610 FS = llvm::makeIntrusiveRefCnt<llvm::vfs::TracingFileSystem>(A: std::move(FS));
611
612 switch (Service.getMode()) {
613 case ScanningMode::DependencyDirectivesScan:
614 DepFS =
615 new DependencyScanningWorkerFilesystem(Service.getSharedCache(), FS);
616 BaseFS = DepFS;
617 break;
618 case ScanningMode::CanonicalPreprocessing:
619 DepFS = nullptr;
620 BaseFS = FS;
621 break;
622 }
623}
624
625static std::unique_ptr<DiagnosticOptions>
626createDiagOptions(const std::vector<std::string> &CommandLine) {
627 std::vector<const char *> CLI;
628 for (const std::string &Arg : CommandLine)
629 CLI.push_back(x: Arg.c_str());
630 auto DiagOpts = CreateAndPopulateDiagOpts(Argv: CLI);
631 sanitizeDiagOpts(DiagOpts&: *DiagOpts);
632 return DiagOpts;
633}
634
635llvm::Error DependencyScanningWorker::computeDependencies(
636 StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
637 DependencyConsumer &Consumer, DependencyActionController &Controller,
638 std::optional<llvm::MemoryBufferRef> TUBuffer) {
639 // Capture the emitted diagnostics and report them to the client
640 // in the case of a failure.
641 std::string DiagnosticOutput;
642 llvm::raw_string_ostream DiagnosticsOS(DiagnosticOutput);
643 auto DiagOpts = createDiagOptions(CommandLine);
644 TextDiagnosticPrinter DiagPrinter(DiagnosticsOS, *DiagOpts);
645
646 if (computeDependencies(WorkingDirectory, CommandLine, DepConsumer&: Consumer, Controller,
647 DiagConsumer&: DiagPrinter, TUBuffer))
648 return llvm::Error::success();
649 return llvm::make_error<llvm::StringError>(Args&: DiagnosticsOS.str(),
650 Args: llvm::inconvertibleErrorCode());
651}
652
653llvm::Error DependencyScanningWorker::computeDependencies(
654 StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
655 DependencyConsumer &Consumer, DependencyActionController &Controller,
656 StringRef ModuleName) {
657 // Capture the emitted diagnostics and report them to the client
658 // in the case of a failure.
659 std::string DiagnosticOutput;
660 llvm::raw_string_ostream DiagnosticsOS(DiagnosticOutput);
661 auto DiagOpts = createDiagOptions(CommandLine);
662 TextDiagnosticPrinter DiagPrinter(DiagnosticsOS, *DiagOpts);
663
664 if (computeDependencies(WorkingDirectory, CommandLine, DepConsumer&: Consumer, Controller,
665 DiagConsumer&: DiagPrinter, ModuleName))
666 return llvm::Error::success();
667 return llvm::make_error<llvm::StringError>(Args&: DiagnosticsOS.str(),
668 Args: llvm::inconvertibleErrorCode());
669}
670
671static bool forEachDriverJob(
672 ArrayRef<std::string> ArgStrs, DiagnosticsEngine &Diags, FileManager &FM,
673 llvm::function_ref<bool(const driver::Command &Cmd)> Callback) {
674 SmallVector<const char *, 256> Argv;
675 Argv.reserve(N: ArgStrs.size());
676 for (const std::string &Arg : ArgStrs)
677 Argv.push_back(Elt: Arg.c_str());
678
679 llvm::vfs::FileSystem *FS = &FM.getVirtualFileSystem();
680
681 std::unique_ptr<driver::Driver> Driver = std::make_unique<driver::Driver>(
682 args&: Argv[0], args: llvm::sys::getDefaultTargetTriple(), args&: Diags,
683 args: "clang LLVM compiler", args&: FS);
684 Driver->setTitle("clang_based_tool");
685
686 llvm::BumpPtrAllocator Alloc;
687 bool CLMode = driver::IsClangCL(
688 DriverMode: driver::getDriverMode(ProgName: Argv[0], Args: ArrayRef(Argv).slice(N: 1)));
689
690 if (llvm::Error E = driver::expandResponseFiles(Args&: Argv, ClangCLMode: CLMode, Alloc, FS)) {
691 Diags.Report(DiagID: diag::err_drv_expand_response_file)
692 << llvm::toString(E: std::move(E));
693 return false;
694 }
695
696 const std::unique_ptr<driver::Compilation> Compilation(
697 Driver->BuildCompilation(Args: llvm::ArrayRef(Argv)));
698 if (!Compilation)
699 return false;
700
701 if (Compilation->containsError())
702 return false;
703
704 for (const driver::Command &Job : Compilation->getJobs()) {
705 if (!Callback(Job))
706 return false;
707 }
708 return true;
709}
710
711static bool createAndRunToolInvocation(
712 std::vector<std::string> CommandLine, DependencyScanningAction &Action,
713 FileManager &FM,
714 std::shared_ptr<clang::PCHContainerOperations> &PCHContainerOps,
715 DiagnosticsEngine &Diags, DependencyConsumer &Consumer) {
716
717 // Save executable path before providing CommandLine to ToolInvocation
718 std::string Executable = CommandLine[0];
719 ToolInvocation Invocation(std::move(CommandLine), &Action, &FM,
720 PCHContainerOps);
721 Invocation.setDiagnosticConsumer(Diags.getClient());
722 Invocation.setDiagnosticOptions(&Diags.getDiagnosticOptions());
723 if (!Invocation.run())
724 return false;
725
726 std::vector<std::string> Args = Action.takeLastCC1Arguments();
727 Consumer.handleBuildCommand(Cmd: {.Executable: std::move(Executable), .Arguments: std::move(Args)});
728 return true;
729}
730
731bool DependencyScanningWorker::scanDependencies(
732 StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
733 DependencyConsumer &Consumer, DependencyActionController &Controller,
734 DiagnosticConsumer &DC, llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
735 std::optional<StringRef> ModuleName) {
736 auto FileMgr =
737 llvm::makeIntrusiveRefCnt<FileManager>(A: FileSystemOptions{}, A&: FS);
738
739 std::vector<const char *> CCommandLine(CommandLine.size(), nullptr);
740 llvm::transform(Range: CommandLine, d_first: CCommandLine.begin(),
741 F: [](const std::string &Str) { return Str.c_str(); });
742 auto DiagOpts = CreateAndPopulateDiagOpts(Argv: CCommandLine);
743 sanitizeDiagOpts(DiagOpts&: *DiagOpts);
744 IntrusiveRefCntPtr<DiagnosticsEngine> Diags =
745 CompilerInstance::createDiagnostics(VFS&: FileMgr->getVirtualFileSystem(),
746 Opts&: *DiagOpts, Client: &DC,
747 /*ShouldOwnClient=*/false);
748
749 // Although `Diagnostics` are used only for command-line parsing, the
750 // custom `DiagConsumer` might expect a `SourceManager` to be present.
751 SourceManager SrcMgr(*Diags, *FileMgr);
752 Diags->setSourceManager(&SrcMgr);
753 // DisableFree is modified by Tooling for running
754 // in-process; preserve the original value, which is
755 // always true for a driver invocation.
756 bool DisableFree = true;
757 DependencyScanningAction Action(Service, WorkingDirectory, Consumer,
758 Controller, DepFS, DisableFree, ModuleName);
759
760 bool Success = false;
761 if (CommandLine[1] == "-cc1") {
762 Success = createAndRunToolInvocation(CommandLine, Action, FM&: *FileMgr,
763 PCHContainerOps, Diags&: *Diags, Consumer);
764 } else {
765 Success = forEachDriverJob(
766 ArgStrs: CommandLine, Diags&: *Diags, FM&: *FileMgr, Callback: [&](const driver::Command &Cmd) {
767 if (StringRef(Cmd.getCreator().getName()) != "clang") {
768 // Non-clang command. Just pass through to the dependency
769 // consumer.
770 Consumer.handleBuildCommand(
771 Cmd: {.Executable: Cmd.getExecutable(),
772 .Arguments: {Cmd.getArguments().begin(), Cmd.getArguments().end()}});
773 return true;
774 }
775
776 // Insert -cc1 comand line options into Argv
777 std::vector<std::string> Argv;
778 Argv.push_back(x: Cmd.getExecutable());
779 llvm::append_range(C&: Argv, R: Cmd.getArguments());
780
781 // Create an invocation that uses the underlying file
782 // system to ensure that any file system requests that
783 // are made by the driver do not go through the
784 // dependency scanning filesystem.
785 return createAndRunToolInvocation(CommandLine: std::move(Argv), Action, FM&: *FileMgr,
786 PCHContainerOps, Diags&: *Diags, Consumer);
787 });
788 }
789
790 if (Success && !Action.hasScanned())
791 Diags->Report(DiagID: diag::err_fe_expected_compiler_job)
792 << llvm::join(R: CommandLine, Separator: " ");
793
794 // Ensure finish() is called even if we never reached ExecuteAction().
795 if (!Action.hasDiagConsumerFinished())
796 DC.finish();
797
798 return Success && Action.hasScanned();
799}
800
801bool DependencyScanningWorker::computeDependencies(
802 StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
803 DependencyConsumer &Consumer, DependencyActionController &Controller,
804 DiagnosticConsumer &DC, std::optional<llvm::MemoryBufferRef> TUBuffer) {
805 // Reset what might have been modified in the previous worker invocation.
806 BaseFS->setCurrentWorkingDirectory(WorkingDirectory);
807
808 std::optional<std::vector<std::string>> ModifiedCommandLine;
809 llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> ModifiedFS;
810
811 // If we're scanning based on a module name alone, we don't expect the client
812 // to provide us with an input file. However, the driver really wants to have
813 // one. Let's just make it up to make the driver happy.
814 if (TUBuffer) {
815 auto OverlayFS =
816 llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(A&: BaseFS);
817 auto InMemoryFS =
818 llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
819 InMemoryFS->setCurrentWorkingDirectory(WorkingDirectory);
820 auto InputPath = TUBuffer->getBufferIdentifier();
821 InMemoryFS->addFile(
822 Path: InputPath, ModificationTime: 0,
823 Buffer: llvm::MemoryBuffer::getMemBufferCopy(InputData: TUBuffer->getBuffer()));
824 llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> InMemoryOverlay =
825 InMemoryFS;
826
827 OverlayFS->pushOverlay(FS: InMemoryOverlay);
828 ModifiedFS = OverlayFS;
829 ModifiedCommandLine = CommandLine;
830 ModifiedCommandLine->emplace_back(args&: InputPath);
831 }
832
833 const std::vector<std::string> &FinalCommandLine =
834 ModifiedCommandLine ? *ModifiedCommandLine : CommandLine;
835 auto &FinalFS = ModifiedFS ? ModifiedFS : BaseFS;
836
837 return scanDependencies(WorkingDirectory, CommandLine: FinalCommandLine, Consumer,
838 Controller, DC, FS: FinalFS, /*ModuleName=*/std::nullopt);
839}
840
841bool DependencyScanningWorker::computeDependencies(
842 StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
843 DependencyConsumer &Consumer, DependencyActionController &Controller,
844 DiagnosticConsumer &DC, StringRef ModuleName) {
845 // Reset what might have been modified in the previous worker invocation.
846 BaseFS->setCurrentWorkingDirectory(WorkingDirectory);
847
848 // If we're scanning based on a module name alone, we don't expect the client
849 // to provide us with an input file. However, the driver really wants to have
850 // one. Let's just make it up to make the driver happy.
851 auto OverlayFS =
852 llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(A&: BaseFS);
853 auto InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
854 InMemoryFS->setCurrentWorkingDirectory(WorkingDirectory);
855 SmallString<128> FakeInputPath;
856 // TODO: We should retry the creation if the path already exists.
857 llvm::sys::fs::createUniquePath(Model: ModuleName + "-%%%%%%%%.input", ResultPath&: FakeInputPath,
858 /*MakeAbsolute=*/false);
859 InMemoryFS->addFile(Path: FakeInputPath, ModificationTime: 0, Buffer: llvm::MemoryBuffer::getMemBuffer(InputData: ""));
860 llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> InMemoryOverlay = InMemoryFS;
861
862 OverlayFS->pushOverlay(FS: InMemoryOverlay);
863 auto ModifiedCommandLine = CommandLine;
864 ModifiedCommandLine.emplace_back(args&: FakeInputPath);
865
866 return scanDependencies(WorkingDirectory, CommandLine: ModifiedCommandLine, Consumer,
867 Controller, DC, FS: OverlayFS, ModuleName);
868}
869
870DependencyActionController::~DependencyActionController() {}
871