1//===- DependencyScannerImpl.cpp - Implements module dependency scanning --===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "clang/DependencyScanning/DependencyScannerImpl.h"
10#include "clang/Basic/DiagnosticFrontend.h"
11#include "clang/Basic/DiagnosticSerialization.h"
12#include "clang/DependencyScanning/DependencyActionController.h"
13#include "clang/DependencyScanning/DependencyConsumer.h"
14#include "clang/DependencyScanning/DependencyScanningFilesystem.h"
15#include "clang/DependencyScanning/DependencyScanningService.h"
16#include "clang/DependencyScanning/DependencyScanningWorker.h"
17#include "clang/Frontend/FrontendActions.h"
18#include "llvm/ADT/IntrusiveRefCntPtr.h"
19#include "llvm/ADT/ScopeExit.h"
20#include "llvm/Option/Option.h"
21#include "llvm/Support/AdvisoryLock.h"
22#include "llvm/Support/CrashRecoveryContext.h"
23#include "llvm/Support/VirtualFileSystem.h"
24#include "llvm/TargetParser/Host.h"
25
26#include <mutex>
27#include <thread>
28
29using namespace clang;
30using namespace dependencies;
31
32static bool checkHeaderSearchPaths(const HeaderSearchOptions &HSOpts,
33 const HeaderSearchOptions &ExistingHSOpts,
34 DiagnosticsEngine *Diags,
35 const LangOptions &LangOpts) {
36 if (LangOpts.Modules) {
37 if (HSOpts.VFSOverlayFiles != ExistingHSOpts.VFSOverlayFiles) {
38 if (Diags) {
39 Diags->Report(DiagID: diag::warn_pch_vfsoverlay_mismatch);
40 auto VFSNote = [&](int Type, ArrayRef<std::string> VFSOverlays) {
41 if (VFSOverlays.empty()) {
42 Diags->Report(DiagID: diag::note_pch_vfsoverlay_empty) << Type;
43 } else {
44 std::string Files = llvm::join(R&: VFSOverlays, Separator: "\n");
45 Diags->Report(DiagID: diag::note_pch_vfsoverlay_files) << Type << Files;
46 }
47 };
48 VFSNote(0, HSOpts.VFSOverlayFiles);
49 VFSNote(1, ExistingHSOpts.VFSOverlayFiles);
50 }
51 }
52 }
53 return false;
54}
55
56namespace {
57
58using PrebuiltModuleFilesT = decltype(HeaderSearchOptions::PrebuiltModuleFiles);
59
60/// A listener that collects the imported modules and the input
61/// files. While visiting, collect vfsoverlays and file inputs that determine
62/// whether prebuilt modules fully resolve in stable directories.
63class PrebuiltModuleListener : public ASTReaderListener {
64public:
65 PrebuiltModuleListener(PrebuiltModuleFilesT &PrebuiltModuleFiles,
66 llvm::SmallVector<std::string> &NewModuleFiles,
67 PrebuiltModulesAttrsMap &PrebuiltModulesASTMap,
68 const HeaderSearchOptions &HSOpts,
69 const LangOptions &LangOpts, DiagnosticsEngine &Diags,
70 const ArrayRef<StringRef> StableDirs)
71 : PrebuiltModuleFiles(PrebuiltModuleFiles),
72 NewModuleFiles(NewModuleFiles),
73 PrebuiltModulesASTMap(PrebuiltModulesASTMap), ExistingHSOpts(HSOpts),
74 ExistingLangOpts(LangOpts), Diags(Diags), StableDirs(StableDirs) {}
75
76 bool needsImportVisitation() const override { return true; }
77 bool needsInputFileVisitation() override { return true; }
78 bool needsSystemInputFileVisitation() override { return true; }
79
80 /// Accumulate the modules are transitively depended on by the initial
81 /// prebuilt module.
82 void visitImport(StringRef ModuleName, StringRef Filename) override {
83 if (PrebuiltModuleFiles.insert(x: {ModuleName.str(), Filename.str()}).second)
84 NewModuleFiles.push_back(Elt: Filename.str());
85
86 auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(Key: Filename);
87 PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second;
88 if (PrebuiltMapEntry.second)
89 PrebuiltModule.setInStableDir(!StableDirs.empty());
90
91 if (auto It = PrebuiltModulesASTMap.find(Key: CurrentFile);
92 It != PrebuiltModulesASTMap.end() && CurrentFile != Filename)
93 PrebuiltModule.addDependent(ModuleFile: It->getKey());
94 }
95
96 /// For each input file discovered, check whether it's external path is in a
97 /// stable directory. Traversal is stopped if the current module is not
98 /// considered stable.
99 bool visitInputFileAsRequested(StringRef FilenameAsRequested,
100 StringRef Filename, bool isSystem,
101 bool isOverridden, time_t StoredTime,
102 bool isExplicitModule) override {
103 if (StableDirs.empty())
104 return false;
105 auto PrebuiltEntryIt = PrebuiltModulesASTMap.find(Key: CurrentFile);
106 if ((PrebuiltEntryIt == PrebuiltModulesASTMap.end()) ||
107 (!PrebuiltEntryIt->second.isInStableDir()))
108 return false;
109
110 PrebuiltEntryIt->second.setInStableDir(
111 isPathInStableDir(Directories: StableDirs, Input: Filename));
112 return PrebuiltEntryIt->second.isInStableDir();
113 }
114
115 /// Update which module that is being actively traversed.
116 void visitModuleFile(ModuleFileName Filename, serialization::ModuleKind Kind,
117 bool DirectlyImported) override {
118 // If the CurrentFile is not
119 // considered stable, update any of it's transitive dependents.
120 auto PrebuiltEntryIt = PrebuiltModulesASTMap.find(Key: CurrentFile);
121 if ((PrebuiltEntryIt != PrebuiltModulesASTMap.end()) &&
122 !PrebuiltEntryIt->second.isInStableDir())
123 PrebuiltEntryIt->second.updateDependentsNotInStableDirs(
124 PrebuiltModulesMap&: PrebuiltModulesASTMap);
125 CurrentFile = Filename.str();
126 }
127
128 /// Check the header search options for a given module when considering
129 /// if the module comes from stable directories.
130 bool ReadHeaderSearchOptions(const HeaderSearchOptions &HSOpts,
131 StringRef ModuleFilename, StringRef ContextHash,
132 bool Complain) override {
133
134 auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(Key: CurrentFile);
135 PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second;
136 if (PrebuiltMapEntry.second)
137 PrebuiltModule.setInStableDir(!StableDirs.empty());
138
139 if (PrebuiltModule.isInStableDir())
140 PrebuiltModule.setInStableDir(areOptionsInStableDir(Directories: StableDirs, HSOpts));
141
142 return false;
143 }
144
145 /// Accumulate vfsoverlays used to build these prebuilt modules.
146 bool ReadHeaderSearchPaths(const HeaderSearchOptions &HSOpts,
147 bool Complain) override {
148
149 auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(Key: CurrentFile);
150 PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second;
151 if (PrebuiltMapEntry.second)
152 PrebuiltModule.setInStableDir(!StableDirs.empty());
153
154 PrebuiltModule.setVFS(
155 llvm::StringSet<>(llvm::from_range, HSOpts.VFSOverlayFiles));
156
157 return checkHeaderSearchPaths(
158 HSOpts, ExistingHSOpts, Diags: Complain ? &Diags : nullptr, LangOpts: ExistingLangOpts);
159 }
160
161private:
162 PrebuiltModuleFilesT &PrebuiltModuleFiles;
163 llvm::SmallVector<std::string> &NewModuleFiles;
164 PrebuiltModulesAttrsMap &PrebuiltModulesASTMap;
165 const HeaderSearchOptions &ExistingHSOpts;
166 const LangOptions &ExistingLangOpts;
167 DiagnosticsEngine &Diags;
168 std::string CurrentFile;
169 const ArrayRef<StringRef> StableDirs;
170};
171
172/// Visit the given prebuilt module and collect all of the modules it
173/// transitively imports and contributing input files.
174static bool visitPrebuiltModule(StringRef PrebuiltModuleFilename,
175 CompilerInstance &CI,
176 PrebuiltModuleFilesT &ModuleFiles,
177 PrebuiltModulesAttrsMap &PrebuiltModulesASTMap,
178 DiagnosticsEngine &Diags,
179 const ArrayRef<StringRef> StableDirs) {
180 // List of module files to be processed.
181 llvm::SmallVector<std::string> Worklist;
182
183 PrebuiltModuleListener Listener(ModuleFiles, Worklist, PrebuiltModulesASTMap,
184 CI.getHeaderSearchOpts(), CI.getLangOpts(),
185 Diags, StableDirs);
186
187 Listener.visitModuleFile(Filename: ModuleFileName::makeExplicit(Name: PrebuiltModuleFilename),
188 Kind: serialization::MK_ExplicitModule,
189 /*DirectlyImported=*/true);
190 if (ASTReader::readASTFileControlBlock(
191 Filename: PrebuiltModuleFilename, FileMgr&: CI.getFileManager(), ModCache: CI.getModuleCache(),
192 PCHContainerRdr: CI.getPCHContainerReader(),
193 /*FindModuleFileExtensions=*/false, Listener,
194 /*ValidateDiagnosticOptions=*/false, ClientLoadCapabilities: ASTReader::ARR_OutOfDate))
195 return true;
196
197 while (!Worklist.empty()) {
198 // FIXME: This is assuming the PCH only refers to explicitly-built modules,
199 // which technically is not guaranteed. To remove the assumption, we'd need
200 // to also rework how the module files are handled to the scan, specifically
201 // change the values of HeaderSearchOptions::PrebuiltModuleFiles from plain
202 // paths to ModuleFileName.
203 Listener.visitModuleFile(Filename: ModuleFileName::makeExplicit(Name: Worklist.back()),
204 Kind: serialization::MK_ExplicitModule,
205 /*DirectlyImported=*/false);
206 if (ASTReader::readASTFileControlBlock(
207 Filename: Worklist.pop_back_val(), FileMgr&: CI.getFileManager(), ModCache: CI.getModuleCache(),
208 PCHContainerRdr: CI.getPCHContainerReader(),
209 /*FindModuleFileExtensions=*/false, Listener,
210 /*ValidateDiagnosticOptions=*/false))
211 return true;
212 }
213 return false;
214}
215
216/// Transform arbitrary file name into an object-like file name.
217static std::string makeObjFileName(StringRef FileName) {
218 SmallString<128> ObjFileName(FileName);
219 llvm::sys::path::replace_extension(path&: ObjFileName, extension: "o");
220 return std::string(ObjFileName);
221}
222
223/// Deduce the dependency target based on the output file and input files.
224static std::string
225deduceDepTarget(const std::string &OutputFile,
226 const SmallVectorImpl<FrontendInputFile> &InputFiles) {
227 if (OutputFile != "-")
228 return OutputFile;
229
230 if (InputFiles.empty() || !InputFiles.front().isFile())
231 return "clang-scan-deps\\ dependency";
232
233 return makeObjFileName(FileName: InputFiles.front().getFile());
234}
235
236// Clang implements -D and -U by splatting text into a predefines buffer. This
237// allows constructs such as `-DFඞ=3 "-D F\u{0D9E} 4 3 2”` to be accepted and
238// define the same macro, or adding C++ style comments before the macro name.
239//
240// This function checks that the first non-space characters in the macro
241// obviously form an identifier that can be uniqued on without lexing. Failing
242// to do this could lead to changing the final definition of a macro.
243//
244// We could set up a preprocessor and actually lex the name, but that's very
245// heavyweight for a situation that will almost never happen in practice.
246static std::optional<StringRef> getSimpleMacroName(StringRef Macro) {
247 StringRef Name = Macro.split(Separator: "=").first.ltrim(Chars: " \t");
248 std::size_t I = 0;
249
250 auto FinishName = [&]() -> std::optional<StringRef> {
251 StringRef SimpleName = Name.slice(Start: 0, End: I);
252 if (SimpleName.empty())
253 return std::nullopt;
254 return SimpleName;
255 };
256
257 for (; I != Name.size(); ++I) {
258 switch (Name[I]) {
259 case '(': // Start of macro parameter list
260 case ' ': // End of macro name
261 case '\t':
262 return FinishName();
263 case '_':
264 continue;
265 default:
266 if (llvm::isAlnum(C: Name[I]))
267 continue;
268 return std::nullopt;
269 }
270 }
271 return FinishName();
272}
273} // namespace
274
275void dependencies::canonicalizeDefines(PreprocessorOptions &PPOpts) {
276 using MacroOpt = std::pair<StringRef, std::size_t>;
277 std::vector<MacroOpt> SimpleNames;
278 SimpleNames.reserve(n: PPOpts.Macros.size());
279 std::size_t Index = 0;
280 for (const auto &M : PPOpts.Macros) {
281 auto SName = getSimpleMacroName(Macro: M.first);
282 // Skip optimizing if we can't guarantee we can preserve relative order.
283 if (!SName)
284 return;
285 SimpleNames.emplace_back(args&: *SName, args&: Index);
286 ++Index;
287 }
288
289 llvm::stable_sort(Range&: SimpleNames, C: llvm::less_first());
290 // Keep the last instance of each macro name by going in reverse
291 auto NewEnd = std::unique(
292 first: SimpleNames.rbegin(), last: SimpleNames.rend(),
293 binary_pred: [](const MacroOpt &A, const MacroOpt &B) { return A.first == B.first; });
294 SimpleNames.erase(first: SimpleNames.begin(), last: NewEnd.base());
295
296 // Apply permutation.
297 decltype(PPOpts.Macros) NewMacros;
298 NewMacros.reserve(n: SimpleNames.size());
299 for (std::size_t I = 0, E = SimpleNames.size(); I != E; ++I) {
300 std::size_t OriginalIndex = SimpleNames[I].second;
301 // We still emit undefines here as they may be undefining a predefined macro
302 NewMacros.push_back(x: std::move(PPOpts.Macros[OriginalIndex]));
303 }
304 std::swap(x&: PPOpts.Macros, y&: NewMacros);
305}
306
307namespace {
308class ScanningDependencyDirectivesGetter : public DependencyDirectivesGetter {
309 DependencyScanningWorkerFilesystem *DepFS;
310
311public:
312 ScanningDependencyDirectivesGetter(FileManager &FileMgr) : DepFS(nullptr) {
313 FileMgr.getVirtualFileSystem().visit(Callback: [&](llvm::vfs::FileSystem &FS) {
314 auto *DFS = llvm::dyn_cast<DependencyScanningWorkerFilesystem>(Val: &FS);
315 if (DFS) {
316 assert(!DepFS && "Found multiple scanning VFSs");
317 DepFS = DFS;
318 }
319 });
320 assert(DepFS && "Did not find scanning VFS");
321 }
322
323 std::unique_ptr<DependencyDirectivesGetter>
324 cloneFor(FileManager &FileMgr) override {
325 return std::make_unique<ScanningDependencyDirectivesGetter>(args&: FileMgr);
326 }
327
328 std::optional<ArrayRef<dependency_directives_scan::Directive>>
329 operator()(FileEntryRef File) override {
330 return DepFS->getDirectiveTokens(Path: File.getName());
331 }
332};
333
334/// Sanitize diagnostic options for dependency scan.
335void sanitizeDiagOpts(DiagnosticOptions &DiagOpts) {
336 // Don't print 'X warnings and Y errors generated'.
337 DiagOpts.ShowCarets = false;
338 // Don't write out diagnostic file.
339 DiagOpts.DiagnosticSerializationFile.clear();
340 // Don't emit warnings except for scanning specific warnings.
341 // TODO: It would be useful to add a more principled way to ignore all
342 // warnings that come from source code. The issue is that we need to
343 // ignore warnings that could be surpressed by
344 // `#pragma clang diagnostic`, while still allowing some scanning
345 // warnings for things we're not ready to turn into errors yet.
346 // See `test/ClangScanDeps/diagnostic-pragmas.c` for an example.
347 llvm::erase_if(C&: DiagOpts.Warnings, P: [](StringRef Warning) {
348 return llvm::StringSwitch<bool>(Warning)
349 .Cases(CaseStrings: {"pch-vfs-diff", "error=pch-vfs-diff"}, Value: false)
350 .StartsWith(S: "no-error=", Value: false)
351 .Default(Value: true);
352 });
353}
354} // namespace
355
356std::unique_ptr<DiagnosticOptions>
357dependencies::createDiagOptions(ArrayRef<std::string> CommandLine) {
358 std::vector<const char *> CLI;
359 for (const std::string &Arg : CommandLine)
360 CLI.push_back(x: Arg.c_str());
361 auto DiagOpts = CreateAndPopulateDiagOpts(Argv: CLI);
362 sanitizeDiagOpts(DiagOpts&: *DiagOpts);
363 return DiagOpts;
364}
365
366DiagnosticsEngineWithDiagOpts::DiagnosticsEngineWithDiagOpts(
367 ArrayRef<std::string> CommandLine,
368 IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS, DiagnosticConsumer &DC) {
369 std::vector<const char *> CCommandLine(CommandLine.size(), nullptr);
370 llvm::transform(Range&: CommandLine, d_first: CCommandLine.begin(),
371 F: [](const std::string &Str) { return Str.c_str(); });
372 DiagOpts = CreateAndPopulateDiagOpts(Argv: CCommandLine);
373 sanitizeDiagOpts(DiagOpts&: *DiagOpts);
374 DiagEngine = CompilerInstance::createDiagnostics(VFS&: *FS, Opts&: *DiagOpts, Client: &DC,
375 /*ShouldOwnClient=*/false);
376}
377
378std::unique_ptr<CompilerInvocation>
379dependencies::createCompilerInvocation(ArrayRef<std::string> CommandLine,
380 DiagnosticsEngine &Diags) {
381 llvm::opt::ArgStringList Argv;
382 for (const std::string &Str : ArrayRef(CommandLine).drop_front())
383 Argv.push_back(Elt: Str.c_str());
384
385 auto Invocation = std::make_unique<CompilerInvocation>();
386 if (!CompilerInvocation::CreateFromArgs(Res&: *Invocation, CommandLineArgs: Argv, Diags)) {
387 // FIXME: Should we just go on like cc1_main does?
388 return nullptr;
389 }
390 return Invocation;
391}
392
393void dependencies::initializeScanCompilerInstance(
394 CompilerInstance &ScanInstance,
395 IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
396 DiagnosticConsumer *DiagConsumer, DependencyScanningService &Service,
397 IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS) {
398 ScanInstance.setBuildingModule(false);
399 ScanInstance.createVirtualFileSystem(BaseFS: FS, DC: DiagConsumer);
400 ScanInstance.createDiagnostics(Client: DiagConsumer, /*ShouldOwnClient=*/false);
401 if (!Service.getOpts().EmitWarnings)
402 ScanInstance.getDiagnostics().setIgnoreAllWarnings(true);
403 ScanInstance.createFileManager();
404 ScanInstance.createSourceManager();
405
406 // Use DepFS for getting the dependency directives if requested to do so.
407 if (Service.getOpts().Mode == ScanningMode::DependencyDirectivesScan)
408 ScanInstance.setDependencyDirectivesGetter(
409 std::make_unique<ScanningDependencyDirectivesGetter>(
410 args&: ScanInstance.getFileManager()));
411}
412
413std::shared_ptr<CompilerInvocation> dependencies::createScanCompilerInvocation(
414 const CompilerInvocation &Invocation,
415 const DependencyScanningService &Service,
416 DependencyActionController &Controller) {
417 auto ScanInvocation = std::make_shared<CompilerInvocation>(args: Invocation);
418
419 sanitizeDiagOpts(DiagOpts&: ScanInvocation->getDiagnosticOpts());
420
421 ScanInvocation->getPreprocessorOpts().AllowPCHWithDifferentModulesCachePath =
422 true;
423
424 if (ScanInvocation->getHeaderSearchOpts().ModulesValidateOncePerBuildSession)
425 ScanInvocation->getHeaderSearchOpts().BuildSessionTimestamp =
426 Service.getOpts().BuildSessionTimestamp;
427
428 ScanInvocation->getFrontendOpts().DisableFree = false;
429 ScanInvocation->getFrontendOpts().GenerateGlobalModuleIndex = false;
430 ScanInvocation->getFrontendOpts().UseGlobalModuleIndex = false;
431 ScanInvocation->getFrontendOpts().GenReducedBMI = false;
432 ScanInvocation->getFrontendOpts().ModuleOutputPath.clear();
433 // This will prevent us compiling individual modules asynchronously since
434 // FileManager is not thread-safe, but it does improve performance for now.
435 ScanInvocation->getFrontendOpts().ModulesShareFileManager = true;
436 ScanInvocation->getHeaderSearchOpts().ModuleFormat = "raw";
437 ScanInvocation->getHeaderSearchOpts().ModulesIncludeVFSUsage =
438 any(Val: Service.getOpts().OptimizeArgs & ScanningOptimizations::VFS);
439
440 // Consider different header search and diagnostic options to create
441 // different modules. This avoids the unsound aliasing of module PCMs.
442 //
443 // TODO: Implement diagnostic bucketing to reduce the impact of strict
444 // context hashing.
445 ScanInvocation->getHeaderSearchOpts().ModulesStrictContextHash = true;
446 ScanInvocation->getHeaderSearchOpts().ModulesSerializeOnlyPreprocessor = true;
447 ScanInvocation->getHeaderSearchOpts().ModulesSkipDiagnosticOptions = true;
448 ScanInvocation->getHeaderSearchOpts().ModulesSkipHeaderSearchPaths = true;
449 ScanInvocation->getHeaderSearchOpts().ModulesSkipPragmaDiagnosticMappings =
450 true;
451 ScanInvocation->getHeaderSearchOpts().ModulesForceValidateUserHeaders = false;
452
453 // FIXME: Do this even with PCHs by marking the option as something like
454 // "preprocessor benign" in LangOptions.def so that it passes the
455 // compatibility checks in ASTReader.
456 if (ScanInvocation->getPreprocessorOpts().ImplicitPCHInclude.empty()) {
457 // Application extension only affects the handling of availability
458 // attributes, which cannot change the dependencies.
459 ScanInvocation->getLangOpts().AppExt = false;
460 }
461
462 // Ensure that the scanner does not create new dependency collectors,
463 // and thus won't write out the extra '.d' files to disk.
464 ScanInvocation->getDependencyOutputOpts() = {};
465
466 Controller.initializeScanInvocation(ScanInvocation&: *ScanInvocation);
467
468 return ScanInvocation;
469}
470
471llvm::SmallVector<StringRef>
472dependencies::getInitialStableDirs(const CompilerInstance &ScanInstance) {
473 // Create a collection of stable directories derived from the ScanInstance
474 // for determining whether module dependencies would fully resolve from
475 // those directories.
476 llvm::SmallVector<StringRef> StableDirs;
477 const StringRef Sysroot = ScanInstance.getHeaderSearchOpts().Sysroot;
478 if (!Sysroot.empty() && (llvm::sys::path::root_directory(path: Sysroot) != Sysroot))
479 StableDirs = {Sysroot, ScanInstance.getHeaderSearchOpts().ResourceDir};
480 return StableDirs;
481}
482
483std::optional<PrebuiltModulesAttrsMap>
484dependencies::computePrebuiltModulesASTMap(
485 CompilerInstance &ScanInstance, llvm::SmallVector<StringRef> &StableDirs) {
486 // Store a mapping of prebuilt module files and their properties like header
487 // search options. This will prevent the implicit build to create duplicate
488 // modules and will force reuse of the existing prebuilt module files
489 // instead.
490 PrebuiltModulesAttrsMap PrebuiltModulesASTMap;
491
492 if (!ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty())
493 if (visitPrebuiltModule(
494 PrebuiltModuleFilename: ScanInstance.getPreprocessorOpts().ImplicitPCHInclude, CI&: ScanInstance,
495 ModuleFiles&: ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles,
496 PrebuiltModulesASTMap, Diags&: ScanInstance.getDiagnostics(), StableDirs))
497 return {};
498
499 return PrebuiltModulesASTMap;
500}
501
502std::unique_ptr<DependencyOutputOptions>
503dependencies::createDependencyOutputOptions(
504 const CompilerInvocation &Invocation) {
505 auto Opts = std::make_unique<DependencyOutputOptions>(
506 args: Invocation.getDependencyOutputOpts());
507 // We need at least one -MT equivalent for the generator of make dependency
508 // files to work.
509 if (Opts->Targets.empty())
510 Opts->Targets = {deduceDepTarget(OutputFile: Invocation.getFrontendOpts().OutputFile,
511 InputFiles: Invocation.getFrontendOpts().Inputs)};
512 Opts->IncludeSystemHeaders = true;
513
514 return Opts;
515}
516
517std::shared_ptr<ModuleDepCollector>
518dependencies::initializeScanInstanceDependencyCollector(
519 CompilerInstance &ScanInstance,
520 std::unique_ptr<DependencyOutputOptions> DepOutputOpts,
521 DependencyScanningService &Service, CompilerInvocation &Inv,
522 DependencyActionController &Controller,
523 PrebuiltModulesAttrsMap PrebuiltModulesASTMap,
524 SmallVector<StringRef> &StableDirs) {
525 auto MDC = std::make_shared<ModuleDepCollector>(
526 args&: Service, args: std::move(DepOutputOpts), args&: ScanInstance, args&: Controller, args&: Inv,
527 args: std::move(PrebuiltModulesASTMap), args&: StableDirs);
528 ScanInstance.addDependencyCollector(Listener: MDC);
529 return MDC;
530}
531
532/// Manages (and terminates) the asynchronous compilation of modules.
533class AsyncModuleCompiles {
534 std::mutex Mutex;
535 bool Stop = false;
536 // FIXME: Have the service own a thread pool and use that instead.
537 std::vector<std::thread> Compiles;
538
539public:
540 /// Registers the module compilation, unless this instance is about to be
541 /// destroyed.
542 void add(llvm::unique_function<void()> Compile) {
543 std::lock_guard<std::mutex> Lock(Mutex);
544 if (!Stop)
545 Compiles.emplace_back(args: std::move(Compile));
546 }
547
548 ~AsyncModuleCompiles() {
549 {
550 // Prevent registration of further module compiles.
551 std::lock_guard<std::mutex> Lock(Mutex);
552 Stop = true;
553 }
554
555 // Wait for outstanding module compiles to finish.
556 for (std::thread &Compile : Compiles)
557 Compile.join();
558 }
559};
560
561struct SingleModuleWithAsyncModuleCompiles : PreprocessOnlyAction {
562 DependencyScanningService &Service;
563 DependencyActionController &Controller;
564 AsyncModuleCompiles &Compiles;
565
566 SingleModuleWithAsyncModuleCompiles(DependencyScanningService &Service,
567 DependencyActionController &Controller,
568 AsyncModuleCompiles &Compiles)
569 : Service(Service), Controller(Controller), Compiles(Compiles) {}
570
571 bool BeginSourceFileAction(CompilerInstance &CI) override;
572};
573
574/// The preprocessor callback that takes care of initiating an asynchronous
575/// module compilation if needed.
576struct AsyncModuleCompile : PPCallbacks {
577 CompilerInstance &CI;
578 DependencyScanningService &Service;
579 DependencyActionController &Controller;
580 AsyncModuleCompiles &Compiles;
581
582 AsyncModuleCompile(CompilerInstance &CI, DependencyScanningService &Service,
583 DependencyActionController &Controller,
584 AsyncModuleCompiles &Compiles)
585 : CI(CI), Service(Service), Controller(Controller), Compiles(Compiles) {}
586
587 void moduleLoadSkipped(Module *M) override {
588 M = M->getTopLevelModule();
589
590 HeaderSearch &HS = CI.getPreprocessor().getHeaderSearchInfo();
591 ModuleCache &ModCache = CI.getModuleCache();
592 ModuleFileName ModuleFileName = HS.getCachedModuleFileName(Module: M);
593
594 uint64_t Timestamp = ModCache.getModuleTimestamp(ModuleFilename: ModuleFileName);
595 // Someone else already built/validated the PCM.
596 if (Timestamp > CI.getHeaderSearchOpts().BuildSessionTimestamp)
597 return;
598
599 if (!CI.getASTReader())
600 CI.createASTReader();
601 SmallVector<ASTReader::ImportedModule, 0> Imported;
602 // Only calling ReadASTCore() to avoid the expensive eager deserialization
603 // of the clang::Module objects in ReadAST().
604 // FIXME: Consider doing this in the new thread depending on how expensive
605 // the read turns out to be.
606 switch (CI.getASTReader()->ReadASTCore(
607 FileName: ModuleFileName, Type: serialization::MK_ImplicitModule, ImportLoc: SourceLocation(),
608 ImportedBy: nullptr, Loaded&: Imported, ExpectedSize: {}, ExpectedModTime: {}, ExpectedSignature: {},
609 ClientLoadCapabilities: ASTReader::ARR_OutOfDate | ASTReader::ARR_Missing |
610 ASTReader::ARR_TreatModuleWithErrorsAsOutOfDate)) {
611 case ASTReader::Success:
612 // We successfully read a valid, up-to-date PCM.
613 // FIXME: This could update the timestamp. Regular calls to
614 // ASTReader::ReadAST() would do so unless they encountered corrupted
615 // AST block, corrupted extension block, or did not read the expected
616 // top-level module.
617 return;
618 case ASTReader::OutOfDate:
619 case ASTReader::Missing:
620 // The most interesting case.
621 break;
622 default:
623 // Let the regular scan diagnose this.
624 return;
625 }
626
627 auto Lock = ModCache.getLock(ModuleFilename: ModuleFileName);
628 bool Owned;
629 llvm::Error LockErr = Lock->tryLock().moveInto(Value&: Owned);
630 // Someone else is building the PCM right now.
631 if (!LockErr && !Owned)
632 return;
633 // We should build the PCM.
634 IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS =
635 llvm::makeIntrusiveRefCnt<DependencyScanningWorkerFilesystem>(
636 A&: Service, A: Service.getOpts().MakeVFS());
637 VFS = createVFSFromCompilerInvocation(CI: CI.getInvocation(),
638 Diags&: CI.getDiagnostics(), BaseFS: std::move(VFS));
639 auto DC = std::make_unique<DiagnosticConsumer>();
640 auto MC = makeInProcessModuleCache(Entries&: Service.getModuleCacheEntries(),
641 Logger&: Service.getLogger());
642 CompilerInstance::ThreadSafeCloneConfig CloneConfig(std::move(VFS), *DC,
643 std::move(MC));
644 auto ModCI1 = CI.cloneForModuleCompile(ImportLoc: SourceLocation(), Module: M, ModuleFileName,
645 ThreadSafeConfig: CloneConfig);
646 auto ModCI2 = CI.cloneForModuleCompile(ImportLoc: SourceLocation(), Module: M, ModuleFileName,
647 ThreadSafeConfig: CloneConfig);
648
649 auto ModController = Controller.clone();
650
651 // Note: This lock belongs to a module cache that might not outlive the
652 // thread. This works, because the in-process lock only refers to an object
653 // managed by the service, which does outlive the thread.
654 Compiles.add(Compile: [Lock = std::move(Lock), ModCI1 = std::move(ModCI1),
655 ModCI2 = std::move(ModCI2), DC = std::move(DC),
656 ModController = std::move(ModController), Service = &Service,
657 Compiles = &Compiles] {
658 llvm::CrashRecoveryContext CRC;
659 (void)CRC.RunSafely(Fn: [&] {
660 // Quickly discovers and compiles modules for the real scan below.
661 SingleModuleWithAsyncModuleCompiles Action1(*Service, *ModController,
662 *Compiles);
663 (void)ModCI1->ExecuteAction(Act&: Action1);
664 // The real scan below.
665 ModCI2->getPreprocessorOpts().SingleModuleParseMode = false;
666 GenerateModuleFromModuleMapAction Action2;
667 (void)ModCI2->ExecuteAction(Act&: Action2);
668 });
669 });
670 }
671};
672
673/// Runs the preprocessor on a TU with single-module-parse-mode and compiles
674/// modules asynchronously without blocking or importing them.
675struct SingleTUWithAsyncModuleCompiles : PreprocessOnlyAction {
676 DependencyScanningService &Service;
677 DependencyActionController &Controller;
678 AsyncModuleCompiles &Compiles;
679
680 SingleTUWithAsyncModuleCompiles(DependencyScanningService &Service,
681 DependencyActionController &Controller,
682 AsyncModuleCompiles &Compiles)
683 : Service(Service), Controller(Controller), Compiles(Compiles) {}
684
685 bool BeginSourceFileAction(CompilerInstance &CI) override {
686 CI.getInvocation().getPreprocessorOpts().SingleModuleParseMode = true;
687 CI.getPreprocessor().addPPCallbacks(C: std::make_unique<AsyncModuleCompile>(
688 args&: CI, args&: Service, args&: Controller, args&: Compiles));
689 return true;
690 }
691};
692
693bool SingleModuleWithAsyncModuleCompiles::BeginSourceFileAction(
694 CompilerInstance &CI) {
695 CI.getInvocation().getPreprocessorOpts().SingleModuleParseMode = true;
696 CI.getPreprocessor().addPPCallbacks(
697 C: std::make_unique<AsyncModuleCompile>(args&: CI, args&: Service, args&: Controller, args&: Compiles));
698 return true;
699}
700
701bool DependencyScanningAction::runInvocation(
702 std::string Executable,
703 std::unique_ptr<CompilerInvocation> OriginalInvocation,
704 IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
705 std::shared_ptr<PCHContainerOperations> PCHContainerOps,
706 DiagnosticConsumer *DiagConsumer) {
707 // Making sure that we canonicalize the defines early to avoid unnecessary
708 // variants in both the scanner and in the resulting explicit command lines.
709 if (any(Val: Service.getOpts().OptimizeArgs & ScanningOptimizations::Macros))
710 canonicalizeDefines(PPOpts&: OriginalInvocation->getPreprocessorOpts());
711
712 if (Scanned) {
713 CompilerInstance &ScanInstance = *ScanInstanceStorage;
714
715 // Scanning runs once for the first -cc1 invocation in a chain of driver
716 // jobs. For any dependent jobs, reuse the scanning result and just
717 // update the new invocation.
718 // FIXME: to support multi-arch builds, each arch requires a separate scan
719 if (MDC)
720 MDC->applyDiscoveredDependencies(CI&: *OriginalInvocation);
721
722 bool Success = OriginalInvocation->withCowRef<bool>(
723 Fn: [&](CowCompilerInvocation &CowOriginalInvocation) {
724 return Controller.finalize(ScanInstance, NewInvocation&: CowOriginalInvocation);
725 });
726 if (!Success)
727 return false;
728
729 Consumer.handleBuildCommand(
730 Cmd: {.Executable: Executable, .Arguments: OriginalInvocation->getCC1CommandLine()});
731 return true;
732 }
733
734 Scanned = true;
735
736 // Create a compiler instance to handle the actual work.
737 auto ScanInvocation =
738 createScanCompilerInvocation(Invocation: *OriginalInvocation, Service, Controller);
739
740 // Quickly discovers and compiles modules for the real scan below.
741 std::optional<AsyncModuleCompiles> AsyncCompiles;
742 if (Service.getOpts().AsyncScanModules) {
743 auto ModCache = makeInProcessModuleCache(Entries&: Service.getModuleCacheEntries(),
744 Logger&: Service.getLogger());
745 auto ScanInstanceStorage = std::make_unique<CompilerInstance>(
746 args: std::make_shared<CompilerInvocation>(args&: *ScanInvocation), args&: PCHContainerOps,
747 args: std::move(ModCache));
748 CompilerInstance &ScanInstance = *ScanInstanceStorage;
749
750 DiagnosticConsumer DiagConsumer;
751 initializeScanCompilerInstance(ScanInstance, FS, DiagConsumer: &DiagConsumer, Service,
752 DepFS);
753
754 // FIXME: Do this only once.
755 SmallVector<StringRef> StableDirs = getInitialStableDirs(ScanInstance);
756 auto MaybePrebuiltModulesASTMap =
757 computePrebuiltModulesASTMap(ScanInstance, StableDirs);
758 if (!MaybePrebuiltModulesASTMap)
759 return false;
760
761 // Normally this would be handled by GeneratePCHAction
762 if (ScanInstance.getFrontendOpts().ProgramAction == frontend::GeneratePCH)
763 ScanInstance.getLangOpts().CompilingPCH = true;
764
765 AsyncCompiles.emplace();
766 SingleTUWithAsyncModuleCompiles Action(Service, Controller, *AsyncCompiles);
767 (void)ScanInstance.ExecuteAction(Act&: Action);
768 }
769
770 auto ModCache = makeInProcessModuleCache(Entries&: Service.getModuleCacheEntries(),
771 Logger&: Service.getLogger());
772 ScanInstanceStorage.emplace(args: std::move(ScanInvocation),
773 args: std::move(PCHContainerOps), args: std::move(ModCache));
774 CompilerInstance &ScanInstance = *ScanInstanceStorage;
775
776 initializeScanCompilerInstance(ScanInstance, FS, DiagConsumer, Service,
777 DepFS);
778
779 llvm::SmallVector<StringRef> StableDirs = getInitialStableDirs(ScanInstance);
780 auto MaybePrebuiltModulesASTMap =
781 computePrebuiltModulesASTMap(ScanInstance, StableDirs);
782 if (!MaybePrebuiltModulesASTMap)
783 return false;
784
785 auto DepOutputOpts = createDependencyOutputOptions(Invocation: *OriginalInvocation);
786
787 MDC = initializeScanInstanceDependencyCollector(
788 ScanInstance, DepOutputOpts: std::move(DepOutputOpts), Service, Inv&: *OriginalInvocation,
789 Controller, PrebuiltModulesASTMap: *MaybePrebuiltModulesASTMap, StableDirs);
790
791 if (ScanInstance.getDiagnostics().hasErrorOccurred())
792 return false;
793
794 if (!Controller.initialize(ScanInstance, NewInvocation&: *OriginalInvocation))
795 return false;
796
797 ReadPCHAndPreprocessAction Action;
798 const bool Result = ScanInstance.ExecuteAction(Act&: Action);
799
800 if (Result) {
801 if (MDC) {
802 MDC->run(Consumer);
803 MDC->applyDiscoveredDependencies(CI&: *OriginalInvocation);
804 }
805
806 bool Success = OriginalInvocation->withCowRef<bool>(
807 Fn: [&](CowCompilerInvocation &CowOriginalInvocation) {
808 return Controller.finalize(ScanInstance, NewInvocation&: CowOriginalInvocation);
809 });
810 if (!Success)
811 return false;
812
813 Consumer.handleBuildCommand(
814 Cmd: {.Executable: Executable, .Arguments: OriginalInvocation->getCC1CommandLine()});
815 }
816
817 return Result;
818}
819