1//===- DependencyScanningWorker.cpp - Thread-Safe Scanning Worker ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "clang/DependencyScanning/DependencyScanningWorker.h"
10#include "clang/Basic/Diagnostic.h"
11#include "clang/Basic/DiagnosticFrontend.h"
12#include "clang/Basic/DiagnosticSerialization.h"
13#include "clang/DependencyScanning/DependencyActionController.h"
14#include "clang/DependencyScanning/DependencyConsumer.h"
15#include "clang/DependencyScanning/DependencyScanningFilesystem.h"
16#include "clang/Frontend/FrontendActions.h"
17#include "clang/Lex/Preprocessor.h"
18#include "clang/Sema/SemaOpenACC.h"
19#include "clang/Serialization/ObjectFilePCHContainerReader.h"
20#include "llvm/ADT/IntrusiveRefCntPtr.h"
21#include "llvm/ADT/ScopeExit.h"
22#include "llvm/Option/Option.h"
23#include "llvm/Support/AdvisoryLock.h"
24#include "llvm/Support/CrashRecoveryContext.h"
25#include "llvm/Support/VirtualFileSystem.h"
26#include "llvm/TargetParser/Host.h"
27#include <mutex>
28#include <thread>
29
30using namespace clang;
31using namespace dependencies;
32
33static bool checkHeaderSearchPaths(const HeaderSearchOptions &HSOpts,
34 const HeaderSearchOptions &ExistingHSOpts,
35 DiagnosticsEngine *Diags,
36 const LangOptions &LangOpts) {
37 if (LangOpts.Modules) {
38 if (HSOpts.VFSOverlayFiles != ExistingHSOpts.VFSOverlayFiles) {
39 if (Diags) {
40 Diags->Report(DiagID: diag::warn_pch_vfsoverlay_mismatch);
41 auto VFSNote = [&](int Type, ArrayRef<std::string> VFSOverlays) {
42 if (VFSOverlays.empty()) {
43 Diags->Report(DiagID: diag::note_pch_vfsoverlay_empty) << Type;
44 } else {
45 std::string Files = llvm::join(R&: VFSOverlays, Separator: "\n");
46 Diags->Report(DiagID: diag::note_pch_vfsoverlay_files) << Type << Files;
47 }
48 };
49 VFSNote(0, HSOpts.VFSOverlayFiles);
50 VFSNote(1, ExistingHSOpts.VFSOverlayFiles);
51 }
52 }
53 }
54 return false;
55}
56namespace {
57using PrebuiltModuleFilesT = decltype(HeaderSearchOptions::PrebuiltModuleFiles);
58
59/// A listener that collects the imported modules and the input
60/// files. While visiting, collect vfsoverlays and file inputs that determine
61/// whether prebuilt modules fully resolve in stable directories.
62class PrebuiltModuleListener : public ASTReaderListener {
63public:
64 PrebuiltModuleListener(PrebuiltModuleFilesT &PrebuiltModuleFiles,
65 llvm::SmallVector<std::string> &NewModuleFiles,
66 PrebuiltModulesAttrsMap &PrebuiltModulesASTMap,
67 const HeaderSearchOptions &HSOpts,
68 const LangOptions &LangOpts, DiagnosticsEngine &Diags,
69 const ArrayRef<StringRef> StableDirs)
70 : PrebuiltModuleFiles(PrebuiltModuleFiles),
71 NewModuleFiles(NewModuleFiles),
72 PrebuiltModulesASTMap(PrebuiltModulesASTMap), ExistingHSOpts(HSOpts),
73 ExistingLangOpts(LangOpts), Diags(Diags), StableDirs(StableDirs) {}
74
75 bool needsImportVisitation() const override { return true; }
76 bool needsInputFileVisitation() override { return true; }
77 bool needsSystemInputFileVisitation() override { return true; }
78
79 /// Accumulate the modules are transitively depended on by the initial
80 /// prebuilt module.
81 void visitImport(StringRef ModuleName, StringRef Filename) override {
82 if (PrebuiltModuleFiles.insert(x: {ModuleName.str(), Filename.str()}).second)
83 NewModuleFiles.push_back(Elt: Filename.str());
84
85 auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(Key: Filename);
86 PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second;
87 if (PrebuiltMapEntry.second)
88 PrebuiltModule.setInStableDir(!StableDirs.empty());
89
90 if (auto It = PrebuiltModulesASTMap.find(Key: CurrentFile);
91 It != PrebuiltModulesASTMap.end() && CurrentFile != Filename)
92 PrebuiltModule.addDependent(ModuleFile: It->getKey());
93 }
94
95 /// For each input file discovered, check whether it's external path is in a
96 /// stable directory. Traversal is stopped if the current module is not
97 /// considered stable.
98 bool visitInputFileAsRequested(StringRef FilenameAsRequested,
99 StringRef Filename, bool isSystem,
100 bool isOverridden, time_t StoredTime,
101 bool isExplicitModule) override {
102 if (StableDirs.empty())
103 return false;
104 auto PrebuiltEntryIt = PrebuiltModulesASTMap.find(Key: CurrentFile);
105 if ((PrebuiltEntryIt == PrebuiltModulesASTMap.end()) ||
106 (!PrebuiltEntryIt->second.isInStableDir()))
107 return false;
108
109 PrebuiltEntryIt->second.setInStableDir(
110 isPathInStableDir(Directories: StableDirs, Input: Filename));
111 return PrebuiltEntryIt->second.isInStableDir();
112 }
113
114 /// Update which module that is being actively traversed.
115 void visitModuleFile(ModuleFileName Filename, serialization::ModuleKind Kind,
116 bool DirectlyImported) override {
117 // If the CurrentFile is not
118 // considered stable, update any of it's transitive dependents.
119 auto PrebuiltEntryIt = PrebuiltModulesASTMap.find(Key: CurrentFile);
120 if ((PrebuiltEntryIt != PrebuiltModulesASTMap.end()) &&
121 !PrebuiltEntryIt->second.isInStableDir())
122 PrebuiltEntryIt->second.updateDependentsNotInStableDirs(
123 PrebuiltModulesMap&: PrebuiltModulesASTMap);
124 CurrentFile = Filename.str();
125 }
126
127 /// Check the header search options for a given module when considering
128 /// if the module comes from stable directories.
129 bool ReadHeaderSearchOptions(const HeaderSearchOptions &HSOpts,
130 StringRef ModuleFilename, StringRef ContextHash,
131 bool Complain) override {
132
133 auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(Key: CurrentFile);
134 PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second;
135 if (PrebuiltMapEntry.second)
136 PrebuiltModule.setInStableDir(!StableDirs.empty());
137
138 if (PrebuiltModule.isInStableDir())
139 PrebuiltModule.setInStableDir(areOptionsInStableDir(Directories: StableDirs, HSOpts));
140
141 return false;
142 }
143
144 /// Accumulate vfsoverlays used to build these prebuilt modules.
145 bool ReadHeaderSearchPaths(const HeaderSearchOptions &HSOpts,
146 bool Complain) override {
147
148 auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(Key: CurrentFile);
149 PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second;
150 if (PrebuiltMapEntry.second)
151 PrebuiltModule.setInStableDir(!StableDirs.empty());
152
153 PrebuiltModule.setVFS(
154 llvm::StringSet<>(llvm::from_range, HSOpts.VFSOverlayFiles));
155
156 return checkHeaderSearchPaths(
157 HSOpts, ExistingHSOpts, Diags: Complain ? &Diags : nullptr, LangOpts: ExistingLangOpts);
158 }
159
160private:
161 PrebuiltModuleFilesT &PrebuiltModuleFiles;
162 llvm::SmallVector<std::string> &NewModuleFiles;
163 PrebuiltModulesAttrsMap &PrebuiltModulesASTMap;
164 const HeaderSearchOptions &ExistingHSOpts;
165 const LangOptions &ExistingLangOpts;
166 DiagnosticsEngine &Diags;
167 std::string CurrentFile;
168 const ArrayRef<StringRef> StableDirs;
169};
170} // namespace
171
172/// Visit the given prebuilt module and collect all of the modules it
173/// transitively imports and contributing input files.
174static bool visitPrebuiltModule(StringRef PrebuiltModuleFilename,
175 CompilerInstance &CI,
176 PrebuiltModuleFilesT &ModuleFiles,
177 PrebuiltModulesAttrsMap &PrebuiltModulesASTMap,
178 DiagnosticsEngine &Diags,
179 const ArrayRef<StringRef> StableDirs) {
180 // List of module files to be processed.
181 llvm::SmallVector<std::string> Worklist;
182
183 PrebuiltModuleListener Listener(ModuleFiles, Worklist, PrebuiltModulesASTMap,
184 CI.getHeaderSearchOpts(), CI.getLangOpts(),
185 Diags, StableDirs);
186
187 Listener.visitModuleFile(Filename: ModuleFileName::makeExplicit(Name: PrebuiltModuleFilename),
188 Kind: serialization::MK_ExplicitModule,
189 /*DirectlyImported=*/true);
190 if (ASTReader::readASTFileControlBlock(
191 Filename: PrebuiltModuleFilename, FileMgr&: CI.getFileManager(), ModCache: CI.getModuleCache(),
192 PCHContainerRdr: CI.getPCHContainerReader(),
193 /*FindModuleFileExtensions=*/false, Listener,
194 /*ValidateDiagnosticOptions=*/false, ClientLoadCapabilities: ASTReader::ARR_OutOfDate))
195 return true;
196
197 while (!Worklist.empty()) {
198 // FIXME: This is assuming the PCH only refers to explicitly-built modules,
199 // which technically is not guaranteed. To remove the assumption, we'd need
200 // to also rework how the module files are handled to the scan, specifically
201 // change the values of HeaderSearchOptions::PrebuiltModuleFiles from plain
202 // paths to ModuleFileName.
203 Listener.visitModuleFile(Filename: ModuleFileName::makeExplicit(Name: Worklist.back()),
204 Kind: serialization::MK_ExplicitModule,
205 /*DirectlyImported=*/false);
206 if (ASTReader::readASTFileControlBlock(
207 Filename: Worklist.pop_back_val(), FileMgr&: CI.getFileManager(), ModCache: CI.getModuleCache(),
208 PCHContainerRdr: CI.getPCHContainerReader(),
209 /*FindModuleFileExtensions=*/false, Listener,
210 /*ValidateDiagnosticOptions=*/false))
211 return true;
212 }
213 return false;
214}
215
216/// Transform arbitrary file name into an object-like file name.
217static std::string makeObjFileName(StringRef FileName) {
218 SmallString<128> ObjFileName(FileName);
219 llvm::sys::path::replace_extension(path&: ObjFileName, extension: "o");
220 return std::string(ObjFileName);
221}
222
223/// Deduce the dependency target based on the output file and input files.
224static std::string
225deduceDepTarget(const std::string &OutputFile,
226 const SmallVectorImpl<FrontendInputFile> &InputFiles) {
227 if (OutputFile != "-")
228 return OutputFile;
229
230 if (InputFiles.empty() || !InputFiles.front().isFile())
231 return "clang-scan-deps\\ dependency";
232
233 return makeObjFileName(FileName: InputFiles.front().getFile());
234}
235
236// Clang implements -D and -U by splatting text into a predefines buffer. This
237// allows constructs such as `-DFඞ=3 "-D F\u{0D9E} 4 3 2”` to be accepted and
238// define the same macro, or adding C++ style comments before the macro name.
239//
240// This function checks that the first non-space characters in the macro
241// obviously form an identifier that can be uniqued on without lexing. Failing
242// to do this could lead to changing the final definition of a macro.
243//
244// We could set up a preprocessor and actually lex the name, but that's very
245// heavyweight for a situation that will almost never happen in practice.
246static std::optional<StringRef> getSimpleMacroName(StringRef Macro) {
247 StringRef Name = Macro.split(Separator: "=").first.ltrim(Chars: " \t");
248 std::size_t I = 0;
249
250 auto FinishName = [&]() -> std::optional<StringRef> {
251 StringRef SimpleName = Name.slice(Start: 0, End: I);
252 if (SimpleName.empty())
253 return std::nullopt;
254 return SimpleName;
255 };
256
257 for (; I != Name.size(); ++I) {
258 switch (Name[I]) {
259 case '(': // Start of macro parameter list
260 case ' ': // End of macro name
261 case '\t':
262 return FinishName();
263 case '_':
264 continue;
265 default:
266 if (llvm::isAlnum(C: Name[I]))
267 continue;
268 return std::nullopt;
269 }
270 }
271 return FinishName();
272}
273
274static void canonicalizeDefines(PreprocessorOptions &PPOpts) {
275 using MacroOpt = std::pair<StringRef, std::size_t>;
276 std::vector<MacroOpt> SimpleNames;
277 SimpleNames.reserve(n: PPOpts.Macros.size());
278 std::size_t Index = 0;
279 for (const auto &M : PPOpts.Macros) {
280 auto SName = getSimpleMacroName(Macro: M.first);
281 // Skip optimizing if we can't guarantee we can preserve relative order.
282 if (!SName)
283 return;
284 SimpleNames.emplace_back(args&: *SName, args&: Index);
285 ++Index;
286 }
287
288 llvm::stable_sort(Range&: SimpleNames, C: llvm::less_first());
289 // Keep the last instance of each macro name by going in reverse
290 auto NewEnd = std::unique(
291 first: SimpleNames.rbegin(), last: SimpleNames.rend(),
292 binary_pred: [](const MacroOpt &A, const MacroOpt &B) { return A.first == B.first; });
293 SimpleNames.erase(first: SimpleNames.begin(), last: NewEnd.base());
294
295 // Apply permutation.
296 decltype(PPOpts.Macros) NewMacros;
297 NewMacros.reserve(n: SimpleNames.size());
298 for (std::size_t I = 0, E = SimpleNames.size(); I != E; ++I) {
299 std::size_t OriginalIndex = SimpleNames[I].second;
300 // We still emit undefines here as they may be undefining a predefined macro
301 NewMacros.push_back(x: std::move(PPOpts.Macros[OriginalIndex]));
302 }
303 std::swap(x&: PPOpts.Macros, y&: NewMacros);
304}
305
306namespace {
307class ScanningDependencyDirectivesGetter : public DependencyDirectivesGetter {
308 DependencyScanningWorkerFilesystem *DepFS;
309
310public:
311 ScanningDependencyDirectivesGetter(FileManager &FileMgr) : DepFS(nullptr) {
312 FileMgr.getVirtualFileSystem().visit(Callback: [&](llvm::vfs::FileSystem &FS) {
313 auto *DFS = llvm::dyn_cast<DependencyScanningWorkerFilesystem>(Val: &FS);
314 if (DFS) {
315 assert(!DepFS && "Found multiple scanning VFSs");
316 DepFS = DFS;
317 }
318 });
319 assert(DepFS && "Did not find scanning VFS");
320 }
321
322 std::unique_ptr<DependencyDirectivesGetter>
323 cloneFor(FileManager &FileMgr) override {
324 return std::make_unique<ScanningDependencyDirectivesGetter>(args&: FileMgr);
325 }
326
327 std::optional<ArrayRef<dependency_directives_scan::Directive>>
328 operator()(FileEntryRef File) override {
329 return DepFS->getDirectiveTokens(Path: File.getName());
330 }
331};
332} // namespace
333
334/// Sanitize diagnostic options for dependency scan.
335static void sanitizeDiagOpts(DiagnosticOptions &DiagOpts) {
336 // Don't print 'X warnings and Y errors generated'.
337 DiagOpts.ShowCarets = false;
338 // Don't write out diagnostic file.
339 DiagOpts.DiagnosticSerializationFile.clear();
340 // Don't emit warnings except for scanning specific warnings.
341 // TODO: It would be useful to add a more principled way to ignore all
342 // warnings that come from source code. The issue is that we need to
343 // ignore warnings that could be surpressed by
344 // `#pragma clang diagnostic`, while still allowing some scanning
345 // warnings for things we're not ready to turn into errors yet.
346 // See `test/ClangScanDeps/diagnostic-pragmas.c` for an example.
347 llvm::erase_if(C&: DiagOpts.Warnings, P: [](StringRef Warning) {
348 return llvm::StringSwitch<bool>(Warning)
349 .Cases(CaseStrings: {"pch-vfs-diff", "error=pch-vfs-diff"}, Value: false)
350 .StartsWith(S: "no-error=", Value: false)
351 .Default(Value: true);
352 });
353}
354
355static std::unique_ptr<CompilerInvocation>
356createCompilerInvocation(ArrayRef<std::string> CommandLine,
357 DiagnosticsEngine &Diags) {
358 llvm::opt::ArgStringList Argv;
359 for (const std::string &Str : ArrayRef(CommandLine).drop_front())
360 Argv.push_back(Elt: Str.c_str());
361
362 auto Invocation = std::make_unique<CompilerInvocation>();
363 if (!CompilerInvocation::CreateFromArgs(Res&: *Invocation, CommandLineArgs: Argv, Diags)) {
364 // FIXME: Should we just go on like cc1_main does?
365 return nullptr;
366 }
367 return Invocation;
368}
369
370static void initializeScanCompilerInstance(
371 CompilerInstance &ScanInstance,
372 IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
373 DiagnosticConsumer *DiagConsumer, DependencyScanningService &Service,
374 IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS) {
375 ScanInstance.setBuildingModule(false);
376 ScanInstance.createVirtualFileSystem(BaseFS: FS, DC: DiagConsumer);
377 ScanInstance.createDiagnostics(Client: DiagConsumer, /*ShouldOwnClient=*/false);
378 if (!Service.getOpts().EmitWarnings)
379 ScanInstance.getDiagnostics().setIgnoreAllWarnings(true);
380 ScanInstance.createFileManager();
381 ScanInstance.createSourceManager();
382
383 // Use DepFS for getting the dependency directives if requested to do so.
384 if (Service.getOpts().Mode == ScanningMode::DependencyDirectivesScan)
385 ScanInstance.setDependencyDirectivesGetter(
386 std::make_unique<ScanningDependencyDirectivesGetter>(
387 args&: ScanInstance.getFileManager()));
388}
389
390static std::shared_ptr<CompilerInvocation>
391createScanCompilerInvocation(const CompilerInvocation &Invocation,
392 const DependencyScanningService &Service,
393 DependencyActionController &Controller) {
394 auto ScanInvocation = std::make_shared<CompilerInvocation>(args: Invocation);
395
396 sanitizeDiagOpts(DiagOpts&: ScanInvocation->getDiagnosticOpts());
397
398 ScanInvocation->getPreprocessorOpts().AllowPCHWithDifferentModulesCachePath =
399 true;
400
401 if (ScanInvocation->getHeaderSearchOpts()
402 .ModulesValidateOncePerBuildSession ||
403 Service.getOpts().ValidateAgainstInvalidatedPaths)
404 ScanInvocation->getHeaderSearchOpts().BuildSessionTimestamp =
405 Service.getOpts().BuildSessionTimestamp;
406 if (Service.getOpts().ValidateAgainstInvalidatedPaths)
407 ScanInvocation->getHeaderSearchOpts().ModulesValidateDirectoryDependencies =
408 true;
409
410 ScanInvocation->getFrontendOpts().DisableFree = false;
411 ScanInvocation->getFrontendOpts().GenerateGlobalModuleIndex = false;
412 ScanInvocation->getFrontendOpts().UseGlobalModuleIndex = false;
413 ScanInvocation->getFrontendOpts().GenReducedBMI = false;
414 ScanInvocation->getFrontendOpts().ModuleOutputPath.clear();
415 // This will prevent us compiling individual modules asynchronously since
416 // FileManager is not thread-safe, but it does improve performance for now.
417 ScanInvocation->getFrontendOpts().ModulesShareFileManager = true;
418 ScanInvocation->getHeaderSearchOpts().ModuleFormat = "raw";
419 ScanInvocation->getHeaderSearchOpts().ModulesIncludeVFSUsage =
420 any(Val: Service.getOpts().OptimizeArgs & ScanningOptimizations::VFS);
421
422 // Consider different header search and diagnostic options to create
423 // different modules. This avoids the unsound aliasing of module PCMs.
424 //
425 // TODO: Implement diagnostic bucketing to reduce the impact of strict
426 // context hashing.
427 ScanInvocation->getHeaderSearchOpts().ModulesStrictContextHash = true;
428 ScanInvocation->getHeaderSearchOpts().ModulesSerializeOnlyPreprocessor = true;
429 ScanInvocation->getHeaderSearchOpts().ModulesSkipDiagnosticOptions = true;
430 ScanInvocation->getHeaderSearchOpts().ModulesSkipHeaderSearchPaths = true;
431 ScanInvocation->getHeaderSearchOpts().ModulesSkipPragmaDiagnosticMappings =
432 true;
433 ScanInvocation->getHeaderSearchOpts().ModulesForceValidateUserHeaders = false;
434
435 // Avoid some checks and module map parsing when loading PCM files.
436 ScanInvocation->getPreprocessorOpts().ModulesCheckRelocated = false;
437
438 // FIXME: Do this even with PCHs by marking the option as something like
439 // "preprocessor benign" in LangOptions.def so that it passes the
440 // compatibility checks in ASTReader.
441 if (ScanInvocation->getPreprocessorOpts().ImplicitPCHInclude.empty()) {
442 // Application extension only affects the handling of availability
443 // attributes, which cannot change the dependencies.
444 ScanInvocation->getLangOpts().AppExt = false;
445 }
446
447 // Ensure that the scanner does not create new dependency collectors,
448 // and thus won't write out the extra '.d' files to disk.
449 ScanInvocation->getDependencyOutputOpts() = {};
450
451 Controller.initializeScanInvocation(ScanInvocation&: *ScanInvocation);
452
453 return ScanInvocation;
454}
455
456static llvm::SmallVector<StringRef>
457getInitialStableDirs(const CompilerInstance &ScanInstance) {
458 // Create a collection of stable directories derived from the ScanInstance
459 // for determining whether module dependencies would fully resolve from
460 // those directories.
461 llvm::SmallVector<StringRef> StableDirs;
462 const StringRef Sysroot = ScanInstance.getHeaderSearchOpts().Sysroot;
463 if (!Sysroot.empty() && (llvm::sys::path::root_directory(path: Sysroot) != Sysroot))
464 StableDirs = {Sysroot, ScanInstance.getHeaderSearchOpts().ResourceDir};
465 return StableDirs;
466}
467
468static std::optional<PrebuiltModulesAttrsMap>
469computePrebuiltModulesASTMap(CompilerInstance &ScanInstance,
470 llvm::SmallVector<StringRef> &StableDirs) {
471 // Store a mapping of prebuilt module files and their properties like header
472 // search options. This will prevent the implicit build to create duplicate
473 // modules and will force reuse of the existing prebuilt module files
474 // instead.
475 PrebuiltModulesAttrsMap PrebuiltModulesASTMap;
476
477 if (!ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty())
478 if (visitPrebuiltModule(
479 PrebuiltModuleFilename: ScanInstance.getPreprocessorOpts().ImplicitPCHInclude, CI&: ScanInstance,
480 ModuleFiles&: ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles,
481 PrebuiltModulesASTMap, Diags&: ScanInstance.getDiagnostics(), StableDirs))
482 return {};
483
484 return PrebuiltModulesASTMap;
485}
486
487static std::unique_ptr<DependencyOutputOptions>
488createDependencyOutputOptions(const CompilerInvocation &Invocation) {
489 auto Opts = std::make_unique<DependencyOutputOptions>(
490 args: Invocation.getDependencyOutputOpts());
491 // We need at least one -MT equivalent for the generator of make dependency
492 // files to work.
493 if (Opts->Targets.empty())
494 Opts->Targets = {deduceDepTarget(OutputFile: Invocation.getFrontendOpts().OutputFile,
495 InputFiles: Invocation.getFrontendOpts().Inputs)};
496 Opts->IncludeSystemHeaders = true;
497
498 return Opts;
499}
500
501static std::shared_ptr<ModuleDepCollector>
502initializeScanInstanceDependencyCollector(
503 CompilerInstance &ScanInstance,
504 std::unique_ptr<DependencyOutputOptions> DepOutputOpts,
505 DependencyScanningService &Service, CompilerInvocation &Inv,
506 DependencyActionController &Controller,
507 PrebuiltModulesAttrsMap PrebuiltModulesASTMap,
508 SmallVector<StringRef> &StableDirs) {
509 auto MDC = std::make_shared<ModuleDepCollector>(
510 args&: Service, args: std::move(DepOutputOpts), args&: ScanInstance, args&: Controller, args&: Inv,
511 args: std::move(PrebuiltModulesASTMap), args&: StableDirs);
512 ScanInstance.addDependencyCollector(Listener: MDC);
513 return MDC;
514}
515
516namespace {
517/// Manages (and terminates) the asynchronous compilation of modules.
518class AsyncModuleCompiles {
519 std::mutex Mutex;
520 bool Stop = false;
521 // FIXME: Have the service own a thread pool and use that instead.
522 std::vector<std::thread> Compiles;
523
524public:
525 /// Registers the module compilation, unless this instance is about to be
526 /// destroyed.
527 void add(llvm::unique_function<void()> Compile) {
528 std::lock_guard<std::mutex> Lock(Mutex);
529 if (!Stop)
530 Compiles.emplace_back(args: std::move(Compile));
531 }
532
533 ~AsyncModuleCompiles() {
534 {
535 std::lock_guard<std::mutex> Lock(Mutex);
536 Stop = true;
537 }
538 for (std::thread &Compile : Compiles)
539 Compile.join();
540 }
541};
542
543struct SingleModuleWithAsyncModuleCompiles : PreprocessOnlyAction {
544 DependencyScanningService &Service;
545 DependencyActionController &Controller;
546 AsyncModuleCompiles &Compiles;
547
548 SingleModuleWithAsyncModuleCompiles(DependencyScanningService &Service,
549 DependencyActionController &Controller,
550 AsyncModuleCompiles &Compiles)
551 : Service(Service), Controller(Controller), Compiles(Compiles) {}
552
553 bool BeginSourceFileAction(CompilerInstance &CI) override;
554};
555
556/// Runs the preprocessor on a TU with single-module-parse-mode and compiles
557/// modules asynchronously without blocking or importing them.
558struct SingleTUWithAsyncModuleCompiles : PreprocessOnlyAction {
559 DependencyScanningService &Service;
560 DependencyActionController &Controller;
561 AsyncModuleCompiles &Compiles;
562
563 SingleTUWithAsyncModuleCompiles(DependencyScanningService &Service,
564 DependencyActionController &Controller,
565 AsyncModuleCompiles &Compiles)
566 : Service(Service), Controller(Controller), Compiles(Compiles) {}
567
568 bool BeginSourceFileAction(CompilerInstance &CI) override;
569};
570
571/// The preprocessor callback that takes care of initiating an asynchronous
572/// module compilation if needed.
573struct AsyncModuleCompile : PPCallbacks {
574 CompilerInstance &CI;
575 DependencyScanningService &Service;
576 DependencyActionController &Controller;
577 AsyncModuleCompiles &Compiles;
578
579 AsyncModuleCompile(CompilerInstance &CI, DependencyScanningService &Service,
580 DependencyActionController &Controller,
581 AsyncModuleCompiles &Compiles)
582 : CI(CI), Service(Service), Controller(Controller), Compiles(Compiles) {}
583
584 void moduleLoadSkipped(Module *M) override {
585 M = M->getTopLevelModule();
586
587 HeaderSearch &HS = CI.getPreprocessor().getHeaderSearchInfo();
588 ModuleCache &ModCache = CI.getModuleCache();
589 ModuleFileName ModuleFileName = HS.getCachedModuleFileName(Module: M);
590
591 uint64_t Timestamp = ModCache.getModuleTimestamp(ModuleFilename: ModuleFileName);
592 // Someone else already built/validated the PCM.
593 if (Timestamp > CI.getHeaderSearchOpts().BuildSessionTimestamp)
594 return;
595
596 if (!CI.getASTReader())
597 CI.createASTReader();
598 SmallVector<ASTReader::ImportedModule, 0> Imported;
599 // Only calling ReadASTCore() to avoid the expensive eager deserialization
600 // of the clang::Module objects in ReadAST().
601 // FIXME: Consider doing this in the new thread depending on how expensive
602 // the read turns out to be.
603 switch (CI.getASTReader()->ReadASTCore(
604 FileName: ModuleFileName, Type: serialization::MK_ImplicitModule, ImportLoc: SourceLocation(),
605 ImportedBy: nullptr, Loaded&: Imported, ExpectedSize: {}, ExpectedModTime: {}, ExpectedSignature: {},
606 ClientLoadCapabilities: ASTReader::ARR_OutOfDate | ASTReader::ARR_Missing |
607 ASTReader::ARR_TreatModuleWithErrorsAsOutOfDate)) {
608 case ASTReader::Success:
609 // We successfully read a valid, up-to-date PCM.
610 // FIXME: This could update the timestamp. Regular calls to
611 // ASTReader::ReadAST() would do so unless they encountered corrupted
612 // AST block, corrupted extension block, or did not read the expected
613 // top-level module.
614 return;
615 case ASTReader::OutOfDate:
616 case ASTReader::Missing:
617 // The most interesting case.
618 break;
619 default:
620 // Let the regular scan diagnose this.
621 return;
622 }
623
624 auto Lock = ModCache.getLock(ModuleFilename: ModuleFileName);
625 bool Owned;
626 llvm::Error LockErr = Lock->tryLock().moveInto(Value&: Owned);
627 // Someone else is building the PCM right now.
628 if (!LockErr && !Owned)
629 return;
630 // We should build the PCM.
631 IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS =
632 llvm::makeIntrusiveRefCnt<DependencyScanningWorkerFilesystem>(
633 A&: Service, A: Service.getOpts().MakeVFS());
634 VFS = createVFSFromCompilerInvocation(CI: CI.getInvocation(),
635 Diags&: CI.getDiagnostics(), BaseFS: std::move(VFS));
636 auto DC = std::make_unique<DiagnosticConsumer>();
637 auto MC = makeInProcessModuleCache(Entries&: Service.getModuleCacheEntries(),
638 Logger&: Service.getLogger());
639 CompilerInstance::ThreadSafeCloneConfig CloneConfig(std::move(VFS), *DC,
640 std::move(MC));
641 auto ModCI1 = CI.cloneForModuleCompile(ImportLoc: SourceLocation(), Module: M, ModuleFileName,
642 ThreadSafeConfig: CloneConfig);
643 auto ModCI2 = CI.cloneForModuleCompile(ImportLoc: SourceLocation(), Module: M, ModuleFileName,
644 ThreadSafeConfig: CloneConfig);
645
646 auto ModController = Controller.clone();
647
648 // Note: This lock belongs to a module cache that might not outlive the
649 // thread. This works, because the in-process lock only refers to an object
650 // managed by the service, which does outlive the thread.
651 Compiles.add(Compile: [Lock = std::move(Lock), ModCI1 = std::move(ModCI1),
652 ModCI2 = std::move(ModCI2), DC = std::move(DC),
653 ModController = std::move(ModController), Service = &Service,
654 Compiles = &Compiles] {
655 llvm::CrashRecoveryContext CRC;
656 (void)CRC.RunSafely(Fn: [&] {
657 // Quickly discovers and compiles modules for the real scan below.
658 SingleModuleWithAsyncModuleCompiles Action1(*Service, *ModController,
659 *Compiles);
660 (void)ModCI1->ExecuteAction(Act&: Action1);
661 // The real scan below.
662 ModCI2->getPreprocessorOpts().SingleModuleParseMode = false;
663 GenerateModuleFromModuleMapAction Action2;
664 (void)ModCI2->ExecuteAction(Act&: Action2);
665 });
666 });
667 }
668};
669
670bool SingleModuleWithAsyncModuleCompiles::BeginSourceFileAction(
671 CompilerInstance &CI) {
672 CI.getInvocation().getPreprocessorOpts().SingleModuleParseMode = true;
673 CI.getPreprocessor().addPPCallbacks(
674 C: std::make_unique<AsyncModuleCompile>(args&: CI, args&: Service, args&: Controller, args&: Compiles));
675 return true;
676}
677
678bool SingleTUWithAsyncModuleCompiles::BeginSourceFileAction(
679 CompilerInstance &CI) {
680 CI.getInvocation().getPreprocessorOpts().SingleModuleParseMode = true;
681 CI.getPreprocessor().addPPCallbacks(
682 C: std::make_unique<AsyncModuleCompile>(args&: CI, args&: Service, args&: Controller, args&: Compiles));
683 return true;
684}
685} // namespace
686
687static void runTUModulePrescan(CompilerInstance &PrescanCI,
688 DependencyScanningService &Service,
689 DependencyActionController &Controller,
690 AsyncModuleCompiles &Compiles) {
691 SingleTUWithAsyncModuleCompiles Action(Service, Controller, Compiles);
692 (void)PrescanCI.ExecuteAction(Act&: Action);
693}
694
695namespace clang {
696namespace dependencies {
697
698std::unique_ptr<DiagnosticOptions>
699createScanningDiagOptions(ArrayRef<std::string> CommandLine) {
700 std::vector<const char *> CCommandLine(CommandLine.size(), nullptr);
701 llvm::transform(Range&: CommandLine, d_first: CCommandLine.begin(),
702 F: [](const std::string &Str) { return Str.c_str(); });
703 auto DiagOpts = CreateAndPopulateDiagOpts(Argv: CCommandLine);
704 sanitizeDiagOpts(DiagOpts&: *DiagOpts);
705 return DiagOpts;
706}
707
708class CompilerInstanceWithContext {
709 // Context
710 DependencyScanningWorker &Worker;
711 llvm::StringRef CWD;
712 std::vector<std::string> CommandLine;
713
714 // Context - compiler invocation
715 std::unique_ptr<CompilerInvocation> OriginalInvocation;
716
717 // Context - output options
718 std::unique_ptr<DependencyOutputOptions> OutputOpts;
719
720 // Context - stable directory handling
721 llvm::SmallVector<StringRef> StableDirs;
722 PrebuiltModulesAttrsMap PrebuiltModuleASTMap;
723
724 // Context - used by AsyncScan's prescan pass
725 IntrusiveRefCntPtr<llvm::vfs::FileSystem> ScanFS;
726
727 // Compiler Instance
728 std::unique_ptr<CompilerInstance> CIPtr;
729
730 // Source location offset.
731 int32_t SrcLocOffset = 0;
732
733 CompilerInstanceWithContext(DependencyScanningWorker &Worker, StringRef CWD,
734 ArrayRef<std::string> CMD)
735 : Worker(Worker), CWD(CWD), CommandLine(CMD.begin(), CMD.end()) {}
736
737 bool initialize(DependencyActionController &Controller,
738 DiagnosticsEngine &DiagEngine,
739 IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS) {
740 {
741 auto LogLine = Worker.Service.getLogger().log();
742 LogLine.logArray(Prefix: "init_compiler_instance_with_context:", Sep: " ",
743 Arr: CommandLine);
744 }
745
746 ScanFS = Worker.makeEffectiveVFS(WorkingDirectory: CWD, OverlayFS: std::move(OverlayFS));
747 OriginalInvocation = createCompilerInvocation(CommandLine, Diags&: DiagEngine);
748 if (!OriginalInvocation) {
749 DiagEngine.Report(DiagID: diag::err_fe_expected_compiler_job)
750 << llvm::join(R&: CommandLine, Separator: " ");
751 return false;
752 }
753
754 return initializeScanInstance(Controller, DiagConsumer: DiagEngine.getClient());
755 }
756
757 bool initializeScanInstance(DependencyActionController &Controller,
758 DiagnosticConsumer *DiagConsumer) {
759 assert(OriginalInvocation && ScanFS &&
760 "OriginalInvocation and ScanFS must be set before this call");
761
762 if (any(Val: Worker.Service.getOpts().OptimizeArgs &
763 ScanningOptimizations::Macros))
764 canonicalizeDefines(PPOpts&: OriginalInvocation->getPreprocessorOpts());
765
766 // Create the CompilerInstance.
767 std::shared_ptr<ModuleCache> ModCache = makeInProcessModuleCache(
768 Entries&: Worker.Service.getModuleCacheEntries(), Logger&: Worker.Service.getLogger());
769 CIPtr = std::make_unique<CompilerInstance>(
770 args: createScanCompilerInvocation(Invocation: *OriginalInvocation, Service: Worker.Service,
771 Controller),
772 args&: Worker.PCHContainerOps, args: std::move(ModCache));
773 auto &CI = *CIPtr;
774
775 initializeScanCompilerInstance(ScanInstance&: CI, FS: ScanFS, DiagConsumer, Service&: Worker.Service,
776 DepFS: Worker.DepFS);
777
778 StableDirs = getInitialStableDirs(ScanInstance: CI);
779 auto MaybePrebuiltModulesASTMap =
780 computePrebuiltModulesASTMap(ScanInstance&: CI, StableDirs);
781 if (!MaybePrebuiltModulesASTMap)
782 return false;
783
784 PrebuiltModuleASTMap = std::move(*MaybePrebuiltModulesASTMap);
785 OutputOpts = createDependencyOutputOptions(Invocation: *OriginalInvocation);
786
787 // We do not create the target in initializeScanCompilerInstance because
788 // setting it here is unique for by-name lookups. We create the target only
789 // once here, and the information is reused for all computeDependencies
790 // calls. We do not need to call createTarget explicitly if we go through
791 // CompilerInstance::ExecuteAction to perform scanning.
792 return CI.createTarget();
793 }
794
795 bool prescanModulesAsync(AsyncModuleCompiles &Compiles,
796 DependencyActionController &Controller) {
797 auto ModCache = makeInProcessModuleCache(
798 Entries&: Worker.Service.getModuleCacheEntries(), Logger&: Worker.Service.getLogger());
799 CompilerInstance PrescanCI(
800 std::make_shared<CompilerInvocation>(args&: CIPtr->getInvocation()),
801 Worker.PCHContainerOps, std::move(ModCache));
802
803 DiagnosticConsumer DiagConsumer;
804 initializeScanCompilerInstance(ScanInstance&: PrescanCI, FS: ScanFS, DiagConsumer: &DiagConsumer,
805 Service&: Worker.Service, DepFS: Worker.DepFS);
806
807 // FIXME: reuse the StableDirs/PrebuiltModuleASTMap computed in
808 // initialize().
809 SmallVector<StringRef> PrescanStableDirs = getInitialStableDirs(ScanInstance: PrescanCI);
810 if (!computePrebuiltModulesASTMap(ScanInstance&: PrescanCI, StableDirs&: PrescanStableDirs))
811 return false;
812
813 if (PrescanCI.getFrontendOpts().ProgramAction == frontend::GeneratePCH)
814 PrescanCI.getLangOpts().CompilingPCH = true;
815
816 runTUModulePrescan(PrescanCI, Service&: Worker.Service, Controller, Compiles);
817 return true;
818 }
819
820public:
821 static std::optional<CompilerInstanceWithContext>
822 initializeFromCC1Commandline(
823 DependencyScanningWorker &Worker, StringRef CWD,
824 ArrayRef<std::string> CC1CommandLine, DiagnosticsEngine &DiagEngine,
825 IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS,
826 DependencyActionController &Controller) {
827 CompilerInstanceWithContext CIWC(Worker, CWD, CC1CommandLine);
828 if (!CIWC.initialize(Controller, DiagEngine, OverlayFS: std::move(OverlayFS)))
829 return std::nullopt;
830 return std::move(CIWC);
831 }
832
833 bool computeDependencies(StringRef ModuleName, DependencyConsumer &Consumer,
834 DependencyActionController &Controller) {
835 Worker.Service.getLogger().log() << "start scan_by_name: " << ModuleName;
836 llvm::scope_exit ExitLogging([&] {
837 Worker.Service.getLogger().log() << "finish scan_by_name: " << ModuleName;
838 });
839 if (SrcLocOffset >= DependencyScanningWorker::MaxNumOfByNameQueries)
840 llvm::report_fatal_error(reason: "exceeded maximum by-name scans for worker");
841
842 assert(CIPtr && "CIPtr must be initialized before calling this method");
843 auto &CI = *CIPtr;
844
845 // We need to reset the diagnostics, so that the diagnostics issued
846 // during a previous computeDependencies call do not affect the current
847 // call. If we do not reset, we may inherit fatal errors from a previous
848 // call.
849 CI.getDiagnostics().Reset();
850
851 // We create this cleanup object because computeDependencies may exit
852 // early with errors.
853 llvm::scope_exit CleanUp([&]() {
854 CI.clearDependencyCollectors();
855 // The preprocessor may not be created at the entry of this method,
856 // but it must have been created when this method returns, whether
857 // there are errors during scanning or not.
858 CI.getPreprocessor().removePPCallbacks();
859 });
860
861 auto MDC = initializeScanInstanceDependencyCollector(
862 ScanInstance&: CI, DepOutputOpts: std::make_unique<DependencyOutputOptions>(args&: *OutputOpts),
863 Service&: Worker.Service,
864 /* The MDC's constructor makes a copy of the OriginalInvocation, so
865 we can pass it in without worrying that it might be changed across
866 invocations of computeDependencies. */
867 Inv&: *OriginalInvocation, Controller, PrebuiltModulesASTMap: PrebuiltModuleASTMap, StableDirs);
868
869 CompilerInvocation ModuleInvocation(*OriginalInvocation);
870 if (!Controller.initialize(ScanInstance&: CI, NewInvocation&: ModuleInvocation))
871 return false;
872
873 if (!SrcLocOffset) {
874 // When SrcLocOffset is zero, we are at the beginning of the fake source
875 // file. In this case, we call BeginSourceFile to initialize.
876 std::unique_ptr<FrontendAction> Action =
877 std::make_unique<PreprocessOnlyAction>();
878 auto *InputFile = CI.getFrontendOpts().Inputs.begin();
879 bool ActionBeginSucceeded = Action->BeginSourceFile(CI, Input: *InputFile);
880 assert(ActionBeginSucceeded && "Action BeginSourceFile must succeed");
881 (void)ActionBeginSucceeded;
882 }
883
884 Preprocessor &PP = CI.getPreprocessor();
885 SourceManager &SM = PP.getSourceManager();
886 FileID MainFileID = SM.getMainFileID();
887 SourceLocation FileStart = SM.getLocForStartOfFile(FID: MainFileID);
888 SourceLocation IDLocation = FileStart.getLocWithOffset(Offset: SrcLocOffset);
889 PPCallbacks *CB = nullptr;
890 if (!SrcLocOffset) {
891 // We need to call EnterSourceFile when SrcLocOffset is zero to initialize
892 // the preprocessor.
893 bool PPFailed = PP.EnterSourceFile(FID: MainFileID, Dir: nullptr, Loc: SourceLocation());
894 assert(!PPFailed && "Preprocess must be able to enter the main file.");
895 (void)PPFailed;
896 CB = MDC->getPPCallbacks();
897 } else {
898 // When SrcLocOffset is non-zero, the preprocessor has already been
899 // initialized through a previous call of computeDependencies. We want to
900 // preserve the PP's state, hence we do not call EnterSourceFile again.
901 MDC->attachToPreprocessor(PP);
902 CB = MDC->getPPCallbacks();
903
904 FileID PrevFID;
905 SrcMgr::CharacteristicKind FileType =
906 SM.getFileCharacteristic(Loc: IDLocation);
907 CB->LexedFileChanged(FID: MainFileID,
908 Reason: PPChainedCallbacks::LexedFileChangeReason::EnterFile,
909 FileType, PrevFID, Loc: IDLocation);
910 }
911
912 // FIXME: Scan modules asynchronously here as well.
913
914 SrcLocOffset++;
915 SmallVector<IdentifierLoc, 2> Path;
916 IdentifierInfo *ModuleID = PP.getIdentifierInfo(Name: ModuleName);
917 Path.emplace_back(Args&: IDLocation, Args&: ModuleID);
918 auto ModResult = CI.loadModule(ImportLoc: IDLocation, Path, Visibility: Module::Hidden, IsInclusionDirective: false);
919
920 assert(CB && "Must have PPCallbacks after module loading");
921 CB->moduleImport(ImportLoc: SourceLocation(), Path, Imported: ModResult);
922
923 if (!ModResult)
924 return false;
925
926 if (CI.getDiagnostics().hasErrorOccurred())
927 return false;
928
929 MDC->run(Consumer);
930 MDC->applyDiscoveredDependencies(CI&: ModuleInvocation);
931
932 bool Success = ModuleInvocation.withCowRef<bool>(
933 Fn: [&](CowCompilerInvocation &CowModuleInvocation) {
934 return Controller.finalize(ScanInstance&: CI, NewInvocation&: CowModuleInvocation);
935 });
936 if (!Success)
937 return false;
938
939 Consumer.handleBuildCommand(
940 Cmd: {.Executable: CommandLine[0], .Arguments: ModuleInvocation.getCC1CommandLine()});
941
942 return true;
943 }
944
945 std::shared_ptr<ModuleDepCollector>
946 scanTranslationUnit(DependencyConsumer &Consumer,
947 DependencyActionController &Controller) {
948 assert(CIPtr && "CIPtr must be initialized before calling this method");
949 auto &CI = *CIPtr;
950
951 std::optional<AsyncModuleCompiles> AsyncCompiles;
952 if (Worker.Service.getOpts().AsyncScanModules) {
953 AsyncCompiles.emplace();
954 if (!prescanModulesAsync(Compiles&: *AsyncCompiles, Controller))
955 return nullptr;
956 }
957
958 auto MDC = initializeScanInstanceDependencyCollector(
959 ScanInstance&: CI, DepOutputOpts: std::make_unique<DependencyOutputOptions>(args&: *OutputOpts),
960 Service&: Worker.Service, Inv&: *OriginalInvocation, Controller, PrebuiltModulesASTMap: PrebuiltModuleASTMap,
961 StableDirs);
962
963 if (CI.getDiagnostics().hasErrorOccurred())
964 return nullptr;
965
966 if (!Controller.initialize(ScanInstance&: CI, NewInvocation&: *OriginalInvocation))
967 return nullptr;
968
969 ReadPCHAndPreprocessAction Action;
970 if (!CI.ExecuteAction(Act&: Action))
971 return nullptr;
972
973 MDC->run(Consumer);
974 if (!applyAndReport(MDC&: *MDC, ModuleInvocation&: *OriginalInvocation, Consumer, Controller,
975 Executable: CommandLine[0]))
976 return nullptr;
977 return MDC;
978 }
979
980 bool applyAndReport(ModuleDepCollector &MDC,
981 CompilerInvocation &ModuleInvocation,
982 DependencyConsumer &Consumer,
983 DependencyActionController &Controller,
984 StringRef Executable) {
985 MDC.applyDiscoveredDependencies(CI&: ModuleInvocation);
986 bool Success = ModuleInvocation.withCowRef<bool>(
987 Fn: [&](CowCompilerInvocation &CowModuleInvocation) {
988 return Controller.finalize(ScanInstance&: *CIPtr, NewInvocation&: CowModuleInvocation);
989 });
990 if (!Success)
991 return false;
992 Consumer.handleBuildCommand(
993 Cmd: {.Executable: Executable.str(), .Arguments: ModuleInvocation.getCC1CommandLine()});
994 return true;
995 }
996};
997} // namespace dependencies
998} // namespace clang
999
1000DependencyScanningWorker::DependencyScanningWorker(
1001 DependencyScanningService &Service)
1002 : Service(Service) {
1003 PCHContainerOps = std::make_shared<PCHContainerOperations>();
1004 // We need to read object files from PCH built outside the scanner.
1005 PCHContainerOps->registerReader(
1006 Reader: std::make_unique<ObjectFilePCHContainerReader>());
1007 // The scanner itself writes only raw ast files.
1008 PCHContainerOps->registerWriter(Writer: std::make_unique<RawPCHContainerWriter>());
1009
1010 auto BaseFS = Service.getOpts().MakeVFS();
1011
1012 if (Service.getOpts().TraceVFS) {
1013 TracingFS = llvm::makeIntrusiveRefCnt<llvm::vfs::TracingFileSystem>(
1014 A: std::move(BaseFS));
1015 BaseFS = TracingFS;
1016 }
1017
1018 DepFS = llvm::makeIntrusiveRefCnt<DependencyScanningWorkerFilesystem>(
1019 A&: Service, A: std::move(BaseFS));
1020}
1021
1022DependencyScanningWorker::~DependencyScanningWorker() = default;
1023
1024IntrusiveRefCntPtr<llvm::vfs::FileSystem>
1025DependencyScanningWorker::makeEffectiveVFS(
1026 StringRef WorkingDirectory,
1027 IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS) const {
1028 IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS = DepFS;
1029 if (OverlayFS) {
1030 auto NewFS =
1031 llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(A: std::move(FS));
1032 NewFS->pushOverlay(FS: std::move(OverlayFS));
1033 FS = std::move(NewFS);
1034 }
1035 FS->setCurrentWorkingDirectory(WorkingDirectory);
1036 return FS;
1037}
1038
1039bool DependencyScanningWorker::computeDependencies(
1040 StringRef WorkingDirectory, ArrayRef<ArrayRef<std::string>> CommandLines,
1041 DependencyConsumer &DepConsumer, DependencyActionController &Controller,
1042 DiagnosticConsumer &DiagConsumer,
1043 IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS) {
1044 auto FS = makeEffectiveVFS(WorkingDirectory, OverlayFS);
1045
1046 bool Scanned = false;
1047 std::shared_ptr<ModuleDepCollector> MDC;
1048 std::optional<CompilerInstanceWithContext> CIWC;
1049
1050 const bool Success = llvm::all_of(Range&: CommandLines, P: [&](const auto &Cmd) {
1051 if (StringRef(Cmd[1]) != "-cc1") {
1052 // Non-clang command. Just pass through to the dependency consumer.
1053 DepConsumer.handleBuildCommand(
1054 Cmd: {Cmd.front(), {Cmd.begin() + 1, Cmd.end()}});
1055 return true;
1056 }
1057
1058 Service.getLogger().log().logArray("starting scanning command:", " ", Cmd);
1059 llvm::scope_exit ExitLogging([&] {
1060 Service.getLogger().log().logArray("finished scanning command:", " ",
1061 Cmd);
1062 });
1063
1064 auto DiagOpts = createScanningDiagOptions(Cmd);
1065 auto DiagEngine =
1066 CompilerInstance::createDiagnostics(*FS, *DiagOpts, &DiagConsumer,
1067 /*ShouldOwnClient=*/false);
1068 if (!Scanned) {
1069 // Scanning runs once for the first -cc1 invocation in a chain of driver
1070 // jobs.
1071 // For any dependent jobs, reuse the scanning result and just update the
1072 // new invocation.
1073 // FIXME: to support multi-arch builds, each arch requires a separate
1074 // scan.
1075 Scanned = true;
1076 auto Result = CompilerInstanceWithContext::initializeFromCC1Commandline(
1077 Worker&: *this, CWD: WorkingDirectory, CC1CommandLine: Cmd, DiagEngine&: *DiagEngine, OverlayFS, Controller);
1078 if (!Result)
1079 return false;
1080 CIWC.emplace(std::move(*Result));
1081 MDC = CIWC->scanTranslationUnit(Consumer&: DepConsumer, Controller);
1082 return MDC != nullptr;
1083 }
1084
1085 auto Invocation = createCompilerInvocation(Cmd, *DiagEngine);
1086 if (!Invocation)
1087 return false;
1088
1089 // The first cc1 is canonicalized in initializeScanInstance; each sibling
1090 // invocation must likewise be canonicalized before its cc1 command line is
1091 // emitted. This is mostly relevant for multi-arch jobs where we currently
1092 // do not do re-scans.
1093 if (any(Val: Service.getOpts().OptimizeArgs & ScanningOptimizations::Macros))
1094 canonicalizeDefines(Invocation->getPreprocessorOpts());
1095
1096 assert(CIWC && "Must have an initialized CIWC");
1097 return CIWC->applyAndReport(MDC&: *MDC, ModuleInvocation&: *Invocation, Consumer&: DepConsumer, Controller,
1098 Executable: Cmd.front());
1099 });
1100
1101 return Success && Scanned;
1102}
1103
1104bool DependencyScanningWorker::computeDependenciesByName(
1105 StringRef CWD, ArrayRef<std::string> CC1CommandLine,
1106 IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS,
1107 DiagnosticConsumer &DiagConsumer, DependencyActionController &Controller,
1108 llvm::function_ref<std::optional<std::string>()> getNextName,
1109 DependencyConsumer &DepConsumer) {
1110 auto FS = makeEffectiveVFS(WorkingDirectory: CWD, OverlayFS);
1111 auto DiagOpts = createScanningDiagOptions(CommandLine: CC1CommandLine);
1112 auto DiagEngine =
1113 CompilerInstance::createDiagnostics(VFS&: *FS, Opts&: *DiagOpts, Client: &DiagConsumer,
1114 /*ShouldOwnClient=*/false);
1115 std::optional<CompilerInstanceWithContext> CIWC =
1116 CompilerInstanceWithContext::initializeFromCC1Commandline(
1117 Worker&: *this, CWD, CC1CommandLine, DiagEngine&: *DiagEngine, OverlayFS: std::move(OverlayFS),
1118 Controller);
1119 if (!CIWC)
1120 return false;
1121
1122 bool AllScansSucceeded = true;
1123 while (std::optional<std::string> NextName = getNextName()) {
1124 bool Success =
1125 CIWC->computeDependencies(ModuleName: *NextName, Consumer&: DepConsumer, Controller);
1126 DepConsumer.finishQuery(ModuleName: *NextName, Success);
1127 AllScansSucceeded = AllScansSucceeded && Success;
1128 }
1129 return AllScansSucceeded;
1130}
1131