1//===- DependencyScannerImpl.cpp - Implements module dependency scanning --===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "clang/DependencyScanning/DependencyScannerImpl.h"
10#include "clang/Basic/DiagnosticFrontend.h"
11#include "clang/Basic/DiagnosticSerialization.h"
12#include "clang/DependencyScanning/DependencyScanningFilesystem.h"
13#include "clang/DependencyScanning/DependencyScanningService.h"
14#include "clang/DependencyScanning/DependencyScanningWorker.h"
15#include "clang/Frontend/FrontendActions.h"
16#include "llvm/ADT/IntrusiveRefCntPtr.h"
17#include "llvm/ADT/ScopeExit.h"
18#include "llvm/Option/Option.h"
19#include "llvm/Support/AdvisoryLock.h"
20#include "llvm/Support/CrashRecoveryContext.h"
21#include "llvm/Support/VirtualFileSystem.h"
22#include "llvm/TargetParser/Host.h"
23
24#include <mutex>
25#include <thread>
26
27using namespace clang;
28using namespace dependencies;
29
30namespace {
31/// Forwards the gatherered dependencies to the consumer.
32class DependencyConsumerForwarder : public DependencyFileGenerator {
33public:
34 DependencyConsumerForwarder(std::unique_ptr<DependencyOutputOptions> Opts,
35 StringRef WorkingDirectory, DependencyConsumer &C)
36 : DependencyFileGenerator(*Opts), WorkingDirectory(WorkingDirectory),
37 Opts(std::move(Opts)), C(C) {}
38
39 void finishedMainFile(DiagnosticsEngine &Diags) override {
40 C.handleDependencyOutputOpts(Opts: *Opts);
41 llvm::SmallString<256> CanonPath;
42 for (const auto &File : getDependencies()) {
43 CanonPath = File;
44 llvm::sys::path::remove_dots(path&: CanonPath, /*remove_dot_dot=*/true);
45 llvm::sys::path::make_absolute(current_directory: WorkingDirectory, path&: CanonPath);
46 C.handleFileDependency(Filename: CanonPath);
47 }
48 }
49
50private:
51 StringRef WorkingDirectory;
52 std::unique_ptr<DependencyOutputOptions> Opts;
53 DependencyConsumer &C;
54};
55
56static bool checkHeaderSearchPaths(const HeaderSearchOptions &HSOpts,
57 const HeaderSearchOptions &ExistingHSOpts,
58 DiagnosticsEngine *Diags,
59 const LangOptions &LangOpts) {
60 if (LangOpts.Modules) {
61 if (HSOpts.VFSOverlayFiles != ExistingHSOpts.VFSOverlayFiles) {
62 if (Diags) {
63 Diags->Report(DiagID: diag::warn_pch_vfsoverlay_mismatch);
64 auto VFSNote = [&](int Type, ArrayRef<std::string> VFSOverlays) {
65 if (VFSOverlays.empty()) {
66 Diags->Report(DiagID: diag::note_pch_vfsoverlay_empty) << Type;
67 } else {
68 std::string Files = llvm::join(R&: VFSOverlays, Separator: "\n");
69 Diags->Report(DiagID: diag::note_pch_vfsoverlay_files) << Type << Files;
70 }
71 };
72 VFSNote(0, HSOpts.VFSOverlayFiles);
73 VFSNote(1, ExistingHSOpts.VFSOverlayFiles);
74 }
75 }
76 }
77 return false;
78}
79
80using PrebuiltModuleFilesT = decltype(HeaderSearchOptions::PrebuiltModuleFiles);
81
82/// A listener that collects the imported modules and the input
83/// files. While visiting, collect vfsoverlays and file inputs that determine
84/// whether prebuilt modules fully resolve in stable directories.
85class PrebuiltModuleListener : public ASTReaderListener {
86public:
87 PrebuiltModuleListener(PrebuiltModuleFilesT &PrebuiltModuleFiles,
88 llvm::SmallVector<std::string> &NewModuleFiles,
89 PrebuiltModulesAttrsMap &PrebuiltModulesASTMap,
90 const HeaderSearchOptions &HSOpts,
91 const LangOptions &LangOpts, DiagnosticsEngine &Diags,
92 const ArrayRef<StringRef> StableDirs)
93 : PrebuiltModuleFiles(PrebuiltModuleFiles),
94 NewModuleFiles(NewModuleFiles),
95 PrebuiltModulesASTMap(PrebuiltModulesASTMap), ExistingHSOpts(HSOpts),
96 ExistingLangOpts(LangOpts), Diags(Diags), StableDirs(StableDirs) {}
97
98 bool needsImportVisitation() const override { return true; }
99 bool needsInputFileVisitation() override { return true; }
100 bool needsSystemInputFileVisitation() override { return true; }
101
102 /// Accumulate the modules are transitively depended on by the initial
103 /// prebuilt module.
104 void visitImport(StringRef ModuleName, StringRef Filename) override {
105 if (PrebuiltModuleFiles.insert(x: {ModuleName.str(), Filename.str()}).second)
106 NewModuleFiles.push_back(Elt: Filename.str());
107
108 auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(Key: Filename);
109 PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second;
110 if (PrebuiltMapEntry.second)
111 PrebuiltModule.setInStableDir(!StableDirs.empty());
112
113 if (auto It = PrebuiltModulesASTMap.find(Key: CurrentFile);
114 It != PrebuiltModulesASTMap.end() && CurrentFile != Filename)
115 PrebuiltModule.addDependent(ModuleFile: It->getKey());
116 }
117
118 /// For each input file discovered, check whether it's external path is in a
119 /// stable directory. Traversal is stopped if the current module is not
120 /// considered stable.
121 bool visitInputFileAsRequested(StringRef FilenameAsRequested,
122 StringRef Filename, bool isSystem,
123 bool isOverridden, time_t StoredTime,
124 bool isExplicitModule) override {
125 if (StableDirs.empty())
126 return false;
127 auto PrebuiltEntryIt = PrebuiltModulesASTMap.find(Key: CurrentFile);
128 if ((PrebuiltEntryIt == PrebuiltModulesASTMap.end()) ||
129 (!PrebuiltEntryIt->second.isInStableDir()))
130 return false;
131
132 PrebuiltEntryIt->second.setInStableDir(
133 isPathInStableDir(Directories: StableDirs, Input: Filename));
134 return PrebuiltEntryIt->second.isInStableDir();
135 }
136
137 /// Update which module that is being actively traversed.
138 void visitModuleFile(ModuleFileName Filename,
139 serialization::ModuleKind Kind) override {
140 // If the CurrentFile is not
141 // considered stable, update any of it's transitive dependents.
142 auto PrebuiltEntryIt = PrebuiltModulesASTMap.find(Key: CurrentFile);
143 if ((PrebuiltEntryIt != PrebuiltModulesASTMap.end()) &&
144 !PrebuiltEntryIt->second.isInStableDir())
145 PrebuiltEntryIt->second.updateDependentsNotInStableDirs(
146 PrebuiltModulesMap&: PrebuiltModulesASTMap);
147 CurrentFile = Filename.str();
148 }
149
150 /// Check the header search options for a given module when considering
151 /// if the module comes from stable directories.
152 bool ReadHeaderSearchOptions(const HeaderSearchOptions &HSOpts,
153 StringRef ModuleFilename, StringRef ContextHash,
154 bool Complain) override {
155
156 auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(Key: CurrentFile);
157 PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second;
158 if (PrebuiltMapEntry.second)
159 PrebuiltModule.setInStableDir(!StableDirs.empty());
160
161 if (PrebuiltModule.isInStableDir())
162 PrebuiltModule.setInStableDir(areOptionsInStableDir(Directories: StableDirs, HSOpts));
163
164 return false;
165 }
166
167 /// Accumulate vfsoverlays used to build these prebuilt modules.
168 bool ReadHeaderSearchPaths(const HeaderSearchOptions &HSOpts,
169 bool Complain) override {
170
171 auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(Key: CurrentFile);
172 PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second;
173 if (PrebuiltMapEntry.second)
174 PrebuiltModule.setInStableDir(!StableDirs.empty());
175
176 PrebuiltModule.setVFS(
177 llvm::StringSet<>(llvm::from_range, HSOpts.VFSOverlayFiles));
178
179 return checkHeaderSearchPaths(
180 HSOpts, ExistingHSOpts, Diags: Complain ? &Diags : nullptr, LangOpts: ExistingLangOpts);
181 }
182
183private:
184 PrebuiltModuleFilesT &PrebuiltModuleFiles;
185 llvm::SmallVector<std::string> &NewModuleFiles;
186 PrebuiltModulesAttrsMap &PrebuiltModulesASTMap;
187 const HeaderSearchOptions &ExistingHSOpts;
188 const LangOptions &ExistingLangOpts;
189 DiagnosticsEngine &Diags;
190 std::string CurrentFile;
191 const ArrayRef<StringRef> StableDirs;
192};
193
194/// Visit the given prebuilt module and collect all of the modules it
195/// transitively imports and contributing input files.
196static bool visitPrebuiltModule(StringRef PrebuiltModuleFilename,
197 CompilerInstance &CI,
198 PrebuiltModuleFilesT &ModuleFiles,
199 PrebuiltModulesAttrsMap &PrebuiltModulesASTMap,
200 DiagnosticsEngine &Diags,
201 const ArrayRef<StringRef> StableDirs) {
202 // List of module files to be processed.
203 llvm::SmallVector<std::string> Worklist;
204
205 PrebuiltModuleListener Listener(ModuleFiles, Worklist, PrebuiltModulesASTMap,
206 CI.getHeaderSearchOpts(), CI.getLangOpts(),
207 Diags, StableDirs);
208
209 Listener.visitModuleFile(Filename: ModuleFileName::makeExplicit(Name: PrebuiltModuleFilename),
210 Kind: serialization::MK_ExplicitModule);
211 if (ASTReader::readASTFileControlBlock(
212 Filename: PrebuiltModuleFilename, FileMgr&: CI.getFileManager(), ModCache: CI.getModuleCache(),
213 PCHContainerRdr: CI.getPCHContainerReader(),
214 /*FindModuleFileExtensions=*/false, Listener,
215 /*ValidateDiagnosticOptions=*/false, ClientLoadCapabilities: ASTReader::ARR_OutOfDate))
216 return true;
217
218 while (!Worklist.empty()) {
219 // FIXME: This is assuming the PCH only refers to explicitly-built modules,
220 // which technically is not guaranteed. To remove the assumption, we'd need
221 // to also rework how the module files are handled to the scan, specifically
222 // change the values of HeaderSearchOptions::PrebuiltModuleFiles from plain
223 // paths to ModuleFileName.
224 Listener.visitModuleFile(Filename: ModuleFileName::makeExplicit(Name: Worklist.back()),
225 Kind: serialization::MK_ExplicitModule);
226 if (ASTReader::readASTFileControlBlock(
227 Filename: Worklist.pop_back_val(), FileMgr&: CI.getFileManager(), ModCache: CI.getModuleCache(),
228 PCHContainerRdr: CI.getPCHContainerReader(),
229 /*FindModuleFileExtensions=*/false, Listener,
230 /*ValidateDiagnosticOptions=*/false))
231 return true;
232 }
233 return false;
234}
235
236/// Transform arbitrary file name into an object-like file name.
237static std::string makeObjFileName(StringRef FileName) {
238 SmallString<128> ObjFileName(FileName);
239 llvm::sys::path::replace_extension(path&: ObjFileName, extension: "o");
240 return std::string(ObjFileName);
241}
242
243/// Deduce the dependency target based on the output file and input files.
244static std::string
245deduceDepTarget(const std::string &OutputFile,
246 const SmallVectorImpl<FrontendInputFile> &InputFiles) {
247 if (OutputFile != "-")
248 return OutputFile;
249
250 if (InputFiles.empty() || !InputFiles.front().isFile())
251 return "clang-scan-deps\\ dependency";
252
253 return makeObjFileName(FileName: InputFiles.front().getFile());
254}
255
256// Clang implements -D and -U by splatting text into a predefines buffer. This
257// allows constructs such as `-DFඞ=3 "-D F\u{0D9E} 4 3 2”` to be accepted and
258// define the same macro, or adding C++ style comments before the macro name.
259//
260// This function checks that the first non-space characters in the macro
261// obviously form an identifier that can be uniqued on without lexing. Failing
262// to do this could lead to changing the final definition of a macro.
263//
264// We could set up a preprocessor and actually lex the name, but that's very
265// heavyweight for a situation that will almost never happen in practice.
266static std::optional<StringRef> getSimpleMacroName(StringRef Macro) {
267 StringRef Name = Macro.split(Separator: "=").first.ltrim(Chars: " \t");
268 std::size_t I = 0;
269
270 auto FinishName = [&]() -> std::optional<StringRef> {
271 StringRef SimpleName = Name.slice(Start: 0, End: I);
272 if (SimpleName.empty())
273 return std::nullopt;
274 return SimpleName;
275 };
276
277 for (; I != Name.size(); ++I) {
278 switch (Name[I]) {
279 case '(': // Start of macro parameter list
280 case ' ': // End of macro name
281 case '\t':
282 return FinishName();
283 case '_':
284 continue;
285 default:
286 if (llvm::isAlnum(C: Name[I]))
287 continue;
288 return std::nullopt;
289 }
290 }
291 return FinishName();
292}
293} // namespace
294
295void dependencies::canonicalizeDefines(PreprocessorOptions &PPOpts) {
296 using MacroOpt = std::pair<StringRef, std::size_t>;
297 std::vector<MacroOpt> SimpleNames;
298 SimpleNames.reserve(n: PPOpts.Macros.size());
299 std::size_t Index = 0;
300 for (const auto &M : PPOpts.Macros) {
301 auto SName = getSimpleMacroName(Macro: M.first);
302 // Skip optimizing if we can't guarantee we can preserve relative order.
303 if (!SName)
304 return;
305 SimpleNames.emplace_back(args&: *SName, args&: Index);
306 ++Index;
307 }
308
309 llvm::stable_sort(Range&: SimpleNames, C: llvm::less_first());
310 // Keep the last instance of each macro name by going in reverse
311 auto NewEnd = std::unique(
312 first: SimpleNames.rbegin(), last: SimpleNames.rend(),
313 binary_pred: [](const MacroOpt &A, const MacroOpt &B) { return A.first == B.first; });
314 SimpleNames.erase(first: SimpleNames.begin(), last: NewEnd.base());
315
316 // Apply permutation.
317 decltype(PPOpts.Macros) NewMacros;
318 NewMacros.reserve(n: SimpleNames.size());
319 for (std::size_t I = 0, E = SimpleNames.size(); I != E; ++I) {
320 std::size_t OriginalIndex = SimpleNames[I].second;
321 // We still emit undefines here as they may be undefining a predefined macro
322 NewMacros.push_back(x: std::move(PPOpts.Macros[OriginalIndex]));
323 }
324 std::swap(x&: PPOpts.Macros, y&: NewMacros);
325}
326
327namespace {
328class ScanningDependencyDirectivesGetter : public DependencyDirectivesGetter {
329 DependencyScanningWorkerFilesystem *DepFS;
330
331public:
332 ScanningDependencyDirectivesGetter(FileManager &FileMgr) : DepFS(nullptr) {
333 FileMgr.getVirtualFileSystem().visit(Callback: [&](llvm::vfs::FileSystem &FS) {
334 auto *DFS = llvm::dyn_cast<DependencyScanningWorkerFilesystem>(Val: &FS);
335 if (DFS) {
336 assert(!DepFS && "Found multiple scanning VFSs");
337 DepFS = DFS;
338 }
339 });
340 assert(DepFS && "Did not find scanning VFS");
341 }
342
343 std::unique_ptr<DependencyDirectivesGetter>
344 cloneFor(FileManager &FileMgr) override {
345 return std::make_unique<ScanningDependencyDirectivesGetter>(args&: FileMgr);
346 }
347
348 std::optional<ArrayRef<dependency_directives_scan::Directive>>
349 operator()(FileEntryRef File) override {
350 return DepFS->getDirectiveTokens(Path: File.getName());
351 }
352};
353
354/// Sanitize diagnostic options for dependency scan.
355void sanitizeDiagOpts(DiagnosticOptions &DiagOpts) {
356 // Don't print 'X warnings and Y errors generated'.
357 DiagOpts.ShowCarets = false;
358 // Don't write out diagnostic file.
359 DiagOpts.DiagnosticSerializationFile.clear();
360 // Don't emit warnings except for scanning specific warnings.
361 // TODO: It would be useful to add a more principled way to ignore all
362 // warnings that come from source code. The issue is that we need to
363 // ignore warnings that could be surpressed by
364 // `#pragma clang diagnostic`, while still allowing some scanning
365 // warnings for things we're not ready to turn into errors yet.
366 // See `test/ClangScanDeps/diagnostic-pragmas.c` for an example.
367 llvm::erase_if(C&: DiagOpts.Warnings, P: [](StringRef Warning) {
368 return llvm::StringSwitch<bool>(Warning)
369 .Cases(CaseStrings: {"pch-vfs-diff", "error=pch-vfs-diff"}, Value: false)
370 .StartsWith(S: "no-error=", Value: false)
371 .Default(Value: true);
372 });
373}
374} // namespace
375
376std::unique_ptr<DiagnosticOptions>
377dependencies::createDiagOptions(ArrayRef<std::string> CommandLine) {
378 std::vector<const char *> CLI;
379 for (const std::string &Arg : CommandLine)
380 CLI.push_back(x: Arg.c_str());
381 auto DiagOpts = CreateAndPopulateDiagOpts(Argv: CLI);
382 sanitizeDiagOpts(DiagOpts&: *DiagOpts);
383 return DiagOpts;
384}
385
386DiagnosticsEngineWithDiagOpts::DiagnosticsEngineWithDiagOpts(
387 ArrayRef<std::string> CommandLine,
388 IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS, DiagnosticConsumer &DC) {
389 std::vector<const char *> CCommandLine(CommandLine.size(), nullptr);
390 llvm::transform(Range&: CommandLine, d_first: CCommandLine.begin(),
391 F: [](const std::string &Str) { return Str.c_str(); });
392 DiagOpts = CreateAndPopulateDiagOpts(Argv: CCommandLine);
393 sanitizeDiagOpts(DiagOpts&: *DiagOpts);
394 DiagEngine = CompilerInstance::createDiagnostics(VFS&: *FS, Opts&: *DiagOpts, Client: &DC,
395 /*ShouldOwnClient=*/false);
396}
397
398std::unique_ptr<CompilerInvocation>
399dependencies::createCompilerInvocation(ArrayRef<std::string> CommandLine,
400 DiagnosticsEngine &Diags) {
401 llvm::opt::ArgStringList Argv;
402 for (const std::string &Str : ArrayRef(CommandLine).drop_front())
403 Argv.push_back(Elt: Str.c_str());
404
405 auto Invocation = std::make_unique<CompilerInvocation>();
406 if (!CompilerInvocation::CreateFromArgs(Res&: *Invocation, CommandLineArgs: Argv, Diags)) {
407 // FIXME: Should we just go on like cc1_main does?
408 return nullptr;
409 }
410 return Invocation;
411}
412
413void dependencies::initializeScanCompilerInstance(
414 CompilerInstance &ScanInstance,
415 IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
416 DiagnosticConsumer *DiagConsumer, DependencyScanningService &Service,
417 IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS) {
418 ScanInstance.setBuildingModule(false);
419 ScanInstance.createVirtualFileSystem(BaseFS: FS, DC: DiagConsumer);
420 ScanInstance.createDiagnostics(Client: DiagConsumer, /*ShouldOwnClient=*/false);
421 if (Service.getOpts().Format == ScanningOutputFormat::P1689)
422 ScanInstance.getDiagnostics().setIgnoreAllWarnings(true);
423 ScanInstance.createFileManager();
424 ScanInstance.createSourceManager();
425
426 // Use DepFS for getting the dependency directives if requested to do so.
427 if (Service.getOpts().Mode == ScanningMode::DependencyDirectivesScan) {
428 DepFS->resetBypassedPathPrefix();
429 SmallString<256> ModulesCachePath;
430 normalizeModuleCachePath(FileMgr&: ScanInstance.getFileManager(),
431 Path: ScanInstance.getHeaderSearchOpts().ModuleCachePath,
432 NormalizedPath&: ModulesCachePath);
433 if (!ModulesCachePath.empty())
434 DepFS->setBypassedPathPrefix(ModulesCachePath);
435
436 ScanInstance.setDependencyDirectivesGetter(
437 std::make_unique<ScanningDependencyDirectivesGetter>(
438 args&: ScanInstance.getFileManager()));
439 }
440}
441
442std::shared_ptr<CompilerInvocation> dependencies::createScanCompilerInvocation(
443 const CompilerInvocation &Invocation,
444 const DependencyScanningService &Service,
445 DependencyActionController &Controller) {
446 auto ScanInvocation = std::make_shared<CompilerInvocation>(args: Invocation);
447
448 sanitizeDiagOpts(DiagOpts&: ScanInvocation->getDiagnosticOpts());
449
450 ScanInvocation->getPreprocessorOpts().AllowPCHWithDifferentModulesCachePath =
451 true;
452
453 if (ScanInvocation->getHeaderSearchOpts().ModulesValidateOncePerBuildSession)
454 ScanInvocation->getHeaderSearchOpts().BuildSessionTimestamp =
455 Service.getOpts().BuildSessionTimestamp;
456
457 ScanInvocation->getFrontendOpts().DisableFree = false;
458 ScanInvocation->getFrontendOpts().GenerateGlobalModuleIndex = false;
459 ScanInvocation->getFrontendOpts().UseGlobalModuleIndex = false;
460 ScanInvocation->getFrontendOpts().GenReducedBMI = false;
461 ScanInvocation->getFrontendOpts().ModuleOutputPath.clear();
462 // This will prevent us compiling individual modules asynchronously since
463 // FileManager is not thread-safe, but it does improve performance for now.
464 ScanInvocation->getFrontendOpts().ModulesShareFileManager = true;
465 ScanInvocation->getHeaderSearchOpts().ModuleFormat = "raw";
466 ScanInvocation->getHeaderSearchOpts().ModulesIncludeVFSUsage =
467 any(Val: Service.getOpts().OptimizeArgs & ScanningOptimizations::VFS);
468
469 // Consider different header search and diagnostic options to create
470 // different modules. This avoids the unsound aliasing of module PCMs.
471 //
472 // TODO: Implement diagnostic bucketing to reduce the impact of strict
473 // context hashing.
474 ScanInvocation->getHeaderSearchOpts().ModulesStrictContextHash = true;
475 ScanInvocation->getHeaderSearchOpts().ModulesSerializeOnlyPreprocessor = true;
476 ScanInvocation->getHeaderSearchOpts().ModulesSkipDiagnosticOptions = true;
477 ScanInvocation->getHeaderSearchOpts().ModulesSkipHeaderSearchPaths = true;
478 ScanInvocation->getHeaderSearchOpts().ModulesSkipPragmaDiagnosticMappings =
479 true;
480 ScanInvocation->getHeaderSearchOpts().ModulesForceValidateUserHeaders = false;
481
482 // Ensure that the scanner does not create new dependency collectors,
483 // and thus won't write out the extra '.d' files to disk.
484 ScanInvocation->getDependencyOutputOpts() = {};
485
486 Controller.initializeScanInvocation(ScanInvocation&: *ScanInvocation);
487
488 return ScanInvocation;
489}
490
491llvm::SmallVector<StringRef>
492dependencies::getInitialStableDirs(const CompilerInstance &ScanInstance) {
493 // Create a collection of stable directories derived from the ScanInstance
494 // for determining whether module dependencies would fully resolve from
495 // those directories.
496 llvm::SmallVector<StringRef> StableDirs;
497 const StringRef Sysroot = ScanInstance.getHeaderSearchOpts().Sysroot;
498 if (!Sysroot.empty() && (llvm::sys::path::root_directory(path: Sysroot) != Sysroot))
499 StableDirs = {Sysroot, ScanInstance.getHeaderSearchOpts().ResourceDir};
500 return StableDirs;
501}
502
503std::optional<PrebuiltModulesAttrsMap>
504dependencies::computePrebuiltModulesASTMap(
505 CompilerInstance &ScanInstance, llvm::SmallVector<StringRef> &StableDirs) {
506 // Store a mapping of prebuilt module files and their properties like header
507 // search options. This will prevent the implicit build to create duplicate
508 // modules and will force reuse of the existing prebuilt module files
509 // instead.
510 PrebuiltModulesAttrsMap PrebuiltModulesASTMap;
511
512 if (!ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty())
513 if (visitPrebuiltModule(
514 PrebuiltModuleFilename: ScanInstance.getPreprocessorOpts().ImplicitPCHInclude, CI&: ScanInstance,
515 ModuleFiles&: ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles,
516 PrebuiltModulesASTMap, Diags&: ScanInstance.getDiagnostics(), StableDirs))
517 return {};
518
519 return PrebuiltModulesASTMap;
520}
521
522std::unique_ptr<DependencyOutputOptions>
523dependencies::createDependencyOutputOptions(
524 const CompilerInvocation &Invocation) {
525 auto Opts = std::make_unique<DependencyOutputOptions>(
526 args: Invocation.getDependencyOutputOpts());
527 // We need at least one -MT equivalent for the generator of make dependency
528 // files to work.
529 if (Opts->Targets.empty())
530 Opts->Targets = {deduceDepTarget(OutputFile: Invocation.getFrontendOpts().OutputFile,
531 InputFiles: Invocation.getFrontendOpts().Inputs)};
532 Opts->IncludeSystemHeaders = true;
533
534 return Opts;
535}
536
537std::shared_ptr<ModuleDepCollector>
538dependencies::initializeScanInstanceDependencyCollector(
539 CompilerInstance &ScanInstance,
540 std::unique_ptr<DependencyOutputOptions> DepOutputOpts,
541 StringRef WorkingDirectory, DependencyConsumer &Consumer,
542 DependencyScanningService &Service, CompilerInvocation &Inv,
543 DependencyActionController &Controller,
544 PrebuiltModulesAttrsMap PrebuiltModulesASTMap,
545 llvm::SmallVector<StringRef> &StableDirs) {
546 std::shared_ptr<ModuleDepCollector> MDC;
547 switch (Service.getOpts().Format) {
548 case ScanningOutputFormat::Make:
549 ScanInstance.addDependencyCollector(
550 Listener: std::make_shared<DependencyConsumerForwarder>(
551 args: std::move(DepOutputOpts), args&: WorkingDirectory, args&: Consumer));
552 break;
553 case ScanningOutputFormat::P1689:
554 case ScanningOutputFormat::Full:
555 MDC = std::make_shared<ModuleDepCollector>(
556 args&: Service, args: std::move(DepOutputOpts), args&: ScanInstance, args&: Consumer, args&: Controller,
557 args&: Inv, args: std::move(PrebuiltModulesASTMap), args&: StableDirs);
558 ScanInstance.addDependencyCollector(Listener: MDC);
559 break;
560 }
561
562 return MDC;
563}
564
565/// Manages (and terminates) the asynchronous compilation of modules.
566class AsyncModuleCompiles {
567 std::mutex Mutex;
568 bool Stop = false;
569 // FIXME: Have the service own a thread pool and use that instead.
570 std::vector<std::thread> Compiles;
571
572public:
573 /// Registers the module compilation, unless this instance is about to be
574 /// destroyed.
575 void add(llvm::unique_function<void()> Compile) {
576 std::lock_guard<std::mutex> Lock(Mutex);
577 if (!Stop)
578 Compiles.emplace_back(args: std::move(Compile));
579 }
580
581 ~AsyncModuleCompiles() {
582 {
583 // Prevent registration of further module compiles.
584 std::lock_guard<std::mutex> Lock(Mutex);
585 Stop = true;
586 }
587
588 // Wait for outstanding module compiles to finish.
589 for (std::thread &Compile : Compiles)
590 Compile.join();
591 }
592};
593
594struct SingleModuleWithAsyncModuleCompiles : PreprocessOnlyAction {
595 DependencyScanningService &Service;
596 DependencyActionController &Controller;
597 AsyncModuleCompiles &Compiles;
598
599 SingleModuleWithAsyncModuleCompiles(DependencyScanningService &Service,
600 DependencyActionController &Controller,
601 AsyncModuleCompiles &Compiles)
602 : Service(Service), Controller(Controller), Compiles(Compiles) {}
603
604 bool BeginSourceFileAction(CompilerInstance &CI) override;
605};
606
607/// The preprocessor callback that takes care of initiating an asynchronous
608/// module compilation if needed.
609struct AsyncModuleCompile : PPCallbacks {
610 CompilerInstance &CI;
611 DependencyScanningService &Service;
612 DependencyActionController &Controller;
613 AsyncModuleCompiles &Compiles;
614
615 AsyncModuleCompile(CompilerInstance &CI, DependencyScanningService &Service,
616 DependencyActionController &Controller,
617 AsyncModuleCompiles &Compiles)
618 : CI(CI), Service(Service), Controller(Controller), Compiles(Compiles) {}
619
620 void moduleLoadSkipped(Module *M) override {
621 M = M->getTopLevelModule();
622
623 HeaderSearch &HS = CI.getPreprocessor().getHeaderSearchInfo();
624 ModuleCache &ModCache = CI.getModuleCache();
625 ModuleFileName ModuleFileName = HS.getCachedModuleFileName(Module: M);
626
627 uint64_t Timestamp = ModCache.getModuleTimestamp(ModuleFilename: ModuleFileName);
628 // Someone else already built/validated the PCM.
629 if (Timestamp > CI.getHeaderSearchOpts().BuildSessionTimestamp)
630 return;
631
632 if (!CI.getASTReader())
633 CI.createASTReader();
634 SmallVector<ASTReader::ImportedModule, 0> Imported;
635 // Only calling ReadASTCore() to avoid the expensive eager deserialization
636 // of the clang::Module objects in ReadAST().
637 // FIXME: Consider doing this in the new thread depending on how expensive
638 // the read turns out to be.
639 switch (CI.getASTReader()->ReadASTCore(
640 FileName: ModuleFileName, Type: serialization::MK_ImplicitModule, ImportLoc: SourceLocation(),
641 ImportedBy: nullptr, Loaded&: Imported, ExpectedSize: {}, ExpectedModTime: {}, ExpectedSignature: {},
642 ClientLoadCapabilities: ASTReader::ARR_OutOfDate | ASTReader::ARR_Missing |
643 ASTReader::ARR_TreatModuleWithErrorsAsOutOfDate)) {
644 case ASTReader::Success:
645 // We successfully read a valid, up-to-date PCM.
646 // FIXME: This could update the timestamp. Regular calls to
647 // ASTReader::ReadAST() would do so unless they encountered corrupted
648 // AST block, corrupted extension block, or did not read the expected
649 // top-level module.
650 return;
651 case ASTReader::OutOfDate:
652 case ASTReader::Missing:
653 // The most interesting case.
654 break;
655 default:
656 // Let the regular scan diagnose this.
657 return;
658 }
659
660 ModCache.prepareForGetLock(ModuleFilename: ModuleFileName);
661 auto Lock = ModCache.getLock(ModuleFilename: ModuleFileName);
662 bool Owned;
663 llvm::Error LockErr = Lock->tryLock().moveInto(Value&: Owned);
664 // Someone else is building the PCM right now.
665 if (!LockErr && !Owned)
666 return;
667 // We should build the PCM.
668 IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS =
669 llvm::makeIntrusiveRefCnt<DependencyScanningWorkerFilesystem>(
670 A&: Service.getSharedCache(), A: Service.getOpts().MakeVFS());
671 VFS = createVFSFromCompilerInvocation(CI: CI.getInvocation(),
672 Diags&: CI.getDiagnostics(), BaseFS: std::move(VFS));
673 auto DC = std::make_unique<DiagnosticConsumer>();
674 auto MC = makeInProcessModuleCache(Entries&: Service.getModuleCacheEntries());
675 CompilerInstance::ThreadSafeCloneConfig CloneConfig(std::move(VFS), *DC,
676 std::move(MC));
677 auto ModCI1 = CI.cloneForModuleCompile(ImportLoc: SourceLocation(), Module: M, ModuleFileName,
678 ThreadSafeConfig: CloneConfig);
679 auto ModCI2 = CI.cloneForModuleCompile(ImportLoc: SourceLocation(), Module: M, ModuleFileName,
680 ThreadSafeConfig: CloneConfig);
681
682 auto ModController = Controller.clone();
683
684 // Note: This lock belongs to a module cache that might not outlive the
685 // thread. This works, because the in-process lock only refers to an object
686 // managed by the service, which does outlive the thread.
687 Compiles.add(Compile: [Lock = std::move(Lock), ModCI1 = std::move(ModCI1),
688 ModCI2 = std::move(ModCI2), DC = std::move(DC),
689 ModController = std::move(ModController), Service = &Service,
690 Compiles = &Compiles] {
691 llvm::CrashRecoveryContext CRC;
692 (void)CRC.RunSafely(Fn: [&] {
693 // Quickly discovers and compiles modules for the real scan below.
694 SingleModuleWithAsyncModuleCompiles Action1(*Service, *ModController,
695 *Compiles);
696 (void)ModCI1->ExecuteAction(Act&: Action1);
697 // The real scan below.
698 ModCI2->getPreprocessorOpts().SingleModuleParseMode = false;
699 GenerateModuleFromModuleMapAction Action2;
700 (void)ModCI2->ExecuteAction(Act&: Action2);
701 });
702 });
703 }
704};
705
706/// Runs the preprocessor on a TU with single-module-parse-mode and compiles
707/// modules asynchronously without blocking or importing them.
708struct SingleTUWithAsyncModuleCompiles : PreprocessOnlyAction {
709 DependencyScanningService &Service;
710 DependencyActionController &Controller;
711 AsyncModuleCompiles &Compiles;
712
713 SingleTUWithAsyncModuleCompiles(DependencyScanningService &Service,
714 DependencyActionController &Controller,
715 AsyncModuleCompiles &Compiles)
716 : Service(Service), Controller(Controller), Compiles(Compiles) {}
717
718 bool BeginSourceFileAction(CompilerInstance &CI) override {
719 CI.getInvocation().getPreprocessorOpts().SingleModuleParseMode = true;
720 CI.getPreprocessor().addPPCallbacks(C: std::make_unique<AsyncModuleCompile>(
721 args&: CI, args&: Service, args&: Controller, args&: Compiles));
722 return true;
723 }
724};
725
726bool SingleModuleWithAsyncModuleCompiles::BeginSourceFileAction(
727 CompilerInstance &CI) {
728 CI.getInvocation().getPreprocessorOpts().SingleModuleParseMode = true;
729 CI.getPreprocessor().addPPCallbacks(
730 C: std::make_unique<AsyncModuleCompile>(args&: CI, args&: Service, args&: Controller, args&: Compiles));
731 return true;
732}
733
734bool DependencyScanningAction::runInvocation(
735 std::string Executable,
736 std::unique_ptr<CompilerInvocation> OriginalInvocation,
737 IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
738 std::shared_ptr<PCHContainerOperations> PCHContainerOps,
739 DiagnosticConsumer *DiagConsumer) {
740 // Making sure that we canonicalize the defines early to avoid unnecessary
741 // variants in both the scanner and in the resulting explicit command lines.
742 if (any(Val: Service.getOpts().OptimizeArgs & ScanningOptimizations::Macros))
743 canonicalizeDefines(PPOpts&: OriginalInvocation->getPreprocessorOpts());
744
745 if (Scanned) {
746 CompilerInstance &ScanInstance = *ScanInstanceStorage;
747
748 // Scanning runs once for the first -cc1 invocation in a chain of driver
749 // jobs. For any dependent jobs, reuse the scanning result and just
750 // update the new invocation.
751 // FIXME: to support multi-arch builds, each arch requires a separate scan
752 if (MDC)
753 MDC->applyDiscoveredDependencies(CI&: *OriginalInvocation);
754
755 if (!Controller.finalize(ScanInstance, NewInvocation&: *OriginalInvocation))
756 return false;
757
758 Consumer.handleBuildCommand(
759 Cmd: {.Executable: Executable, .Arguments: OriginalInvocation->getCC1CommandLine()});
760 return true;
761 }
762
763 Scanned = true;
764
765 // Create a compiler instance to handle the actual work.
766 auto ScanInvocation =
767 createScanCompilerInvocation(Invocation: *OriginalInvocation, Service, Controller);
768
769 // Quickly discovers and compiles modules for the real scan below.
770 std::optional<AsyncModuleCompiles> AsyncCompiles;
771 if (Service.getOpts().AsyncScanModules) {
772 auto ModCache = makeInProcessModuleCache(Entries&: Service.getModuleCacheEntries());
773 auto ScanInstanceStorage = std::make_unique<CompilerInstance>(
774 args: std::make_shared<CompilerInvocation>(args&: *ScanInvocation), args&: PCHContainerOps,
775 args: std::move(ModCache));
776 CompilerInstance &ScanInstance = *ScanInstanceStorage;
777
778 DiagnosticConsumer DiagConsumer;
779 initializeScanCompilerInstance(ScanInstance, FS, DiagConsumer: &DiagConsumer, Service,
780 DepFS);
781
782 // FIXME: Do this only once.
783 SmallVector<StringRef> StableDirs = getInitialStableDirs(ScanInstance);
784 auto MaybePrebuiltModulesASTMap =
785 computePrebuiltModulesASTMap(ScanInstance, StableDirs);
786 if (!MaybePrebuiltModulesASTMap)
787 return false;
788
789 // Normally this would be handled by GeneratePCHAction
790 if (ScanInstance.getFrontendOpts().ProgramAction == frontend::GeneratePCH)
791 ScanInstance.getLangOpts().CompilingPCH = true;
792
793 AsyncCompiles.emplace();
794 SingleTUWithAsyncModuleCompiles Action(Service, Controller, *AsyncCompiles);
795 (void)ScanInstance.ExecuteAction(Act&: Action);
796 }
797
798 auto ModCache = makeInProcessModuleCache(Entries&: Service.getModuleCacheEntries());
799 ScanInstanceStorage.emplace(args: std::move(ScanInvocation),
800 args: std::move(PCHContainerOps), args: std::move(ModCache));
801 CompilerInstance &ScanInstance = *ScanInstanceStorage;
802
803 initializeScanCompilerInstance(ScanInstance, FS, DiagConsumer, Service,
804 DepFS);
805
806 llvm::SmallVector<StringRef> StableDirs = getInitialStableDirs(ScanInstance);
807 auto MaybePrebuiltModulesASTMap =
808 computePrebuiltModulesASTMap(ScanInstance, StableDirs);
809 if (!MaybePrebuiltModulesASTMap)
810 return false;
811
812 auto DepOutputOpts = createDependencyOutputOptions(Invocation: *OriginalInvocation);
813
814 MDC = initializeScanInstanceDependencyCollector(
815 ScanInstance, DepOutputOpts: std::move(DepOutputOpts), WorkingDirectory, Consumer,
816 Service, Inv&: *OriginalInvocation, Controller, PrebuiltModulesASTMap: *MaybePrebuiltModulesASTMap,
817 StableDirs);
818
819 if (ScanInstance.getDiagnostics().hasErrorOccurred())
820 return false;
821
822 if (!Controller.initialize(ScanInstance, NewInvocation&: *OriginalInvocation))
823 return false;
824
825 ReadPCHAndPreprocessAction Action;
826 const bool Result = ScanInstance.ExecuteAction(Act&: Action);
827
828 if (Result) {
829 if (MDC)
830 MDC->applyDiscoveredDependencies(CI&: *OriginalInvocation);
831
832 if (!Controller.finalize(ScanInstance, NewInvocation&: *OriginalInvocation))
833 return false;
834
835 Consumer.handleBuildCommand(
836 Cmd: {.Executable: Executable, .Arguments: OriginalInvocation->getCC1CommandLine()});
837 }
838
839 return Result;
840}
841