1//===- DependencyScanningWorker.cpp - Thread-Safe Scanning Worker ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "clang/DependencyScanning/DependencyScanningWorker.h"
10#include "clang/Basic/Diagnostic.h"
11#include "clang/Basic/DiagnosticFrontend.h"
12#include "clang/Basic/DiagnosticSerialization.h"
13#include "clang/DependencyScanning/DependencyActionController.h"
14#include "clang/DependencyScanning/DependencyConsumer.h"
15#include "clang/DependencyScanning/DependencyScanningFilesystem.h"
16#include "clang/Frontend/FrontendActions.h"
17#include "clang/Lex/HeaderSearch.h"
18#include "clang/Lex/Preprocessor.h"
19#include "clang/Sema/SemaOpenACC.h"
20#include "clang/Serialization/ObjectFilePCHContainerReader.h"
21#include "llvm/ADT/IntrusiveRefCntPtr.h"
22#include "llvm/ADT/ScopeExit.h"
23#include "llvm/Option/Option.h"
24#include "llvm/Support/AdvisoryLock.h"
25#include "llvm/Support/CrashRecoveryContext.h"
26#include "llvm/Support/VirtualFileSystem.h"
27#include "llvm/TargetParser/Host.h"
28#include <mutex>
29#include <thread>
30
31using namespace clang;
32using namespace dependencies;
33
34static bool checkHeaderSearchPaths(const HeaderSearchOptions &HSOpts,
35 const HeaderSearchOptions &ExistingHSOpts,
36 DiagnosticsEngine *Diags,
37 const LangOptions &LangOpts) {
38 if (LangOpts.Modules) {
39 if (HSOpts.VFSOverlayFiles != ExistingHSOpts.VFSOverlayFiles) {
40 if (Diags) {
41 Diags->Report(DiagID: diag::warn_pch_vfsoverlay_mismatch);
42 auto VFSNote = [&](int Type, ArrayRef<std::string> VFSOverlays) {
43 if (VFSOverlays.empty()) {
44 Diags->Report(DiagID: diag::note_pch_vfsoverlay_empty) << Type;
45 } else {
46 std::string Files = llvm::join(R&: VFSOverlays, Separator: "\n");
47 Diags->Report(DiagID: diag::note_pch_vfsoverlay_files) << Type << Files;
48 }
49 };
50 VFSNote(0, HSOpts.VFSOverlayFiles);
51 VFSNote(1, ExistingHSOpts.VFSOverlayFiles);
52 }
53 }
54 }
55 return false;
56}
57namespace {
58using PrebuiltModuleFilesT = decltype(HeaderSearchOptions::PrebuiltModuleFiles);
59
60/// A listener that collects the imported modules and the input
61/// files. While visiting, collect vfsoverlays and file inputs that determine
62/// whether prebuilt modules fully resolve in stable directories.
63class PrebuiltModuleListener : public ASTReaderListener {
64public:
65 PrebuiltModuleListener(PrebuiltModuleFilesT &PrebuiltModuleFiles,
66 llvm::SmallVector<std::string> &NewModuleFiles,
67 PrebuiltModulesAttrsMap &PrebuiltModulesASTMap,
68 const HeaderSearchOptions &HSOpts,
69 const LangOptions &LangOpts, DiagnosticsEngine &Diags,
70 const ArrayRef<StringRef> StableDirs)
71 : PrebuiltModuleFiles(PrebuiltModuleFiles),
72 NewModuleFiles(NewModuleFiles),
73 PrebuiltModulesASTMap(PrebuiltModulesASTMap), ExistingHSOpts(HSOpts),
74 ExistingLangOpts(LangOpts), Diags(Diags), StableDirs(StableDirs) {}
75
76 bool needsImportVisitation() const override { return true; }
77 bool needsInputFileVisitation() override { return true; }
78 bool needsSystemInputFileVisitation() override { return true; }
79
80 /// Accumulate the modules are transitively depended on by the initial
81 /// prebuilt module.
82 void visitImport(StringRef ModuleName, StringRef Filename) override {
83 if (PrebuiltModuleFiles.insert(x: {ModuleName.str(), Filename.str()}).second)
84 NewModuleFiles.push_back(Elt: Filename.str());
85
86 auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(Key: Filename);
87 PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second;
88 if (PrebuiltMapEntry.second)
89 PrebuiltModule.setInStableDir(!StableDirs.empty());
90
91 if (auto It = PrebuiltModulesASTMap.find(Key: CurrentFile);
92 It != PrebuiltModulesASTMap.end() && CurrentFile != Filename)
93 PrebuiltModule.addDependent(ModuleFile: It->getKey());
94 }
95
96 /// For each input file discovered, check whether it's external path is in a
97 /// stable directory. Traversal is stopped if the current module is not
98 /// considered stable.
99 bool visitInputFileAsRequested(StringRef FilenameAsRequested,
100 StringRef Filename, bool isSystem,
101 bool isOverridden, time_t StoredTime,
102 bool isExplicitModule) override {
103 if (StableDirs.empty())
104 return false;
105 auto PrebuiltEntryIt = PrebuiltModulesASTMap.find(Key: CurrentFile);
106 if ((PrebuiltEntryIt == PrebuiltModulesASTMap.end()) ||
107 (!PrebuiltEntryIt->second.isInStableDir()))
108 return false;
109
110 PrebuiltEntryIt->second.setInStableDir(
111 isPathInStableDir(Directories: StableDirs, Input: Filename));
112 return PrebuiltEntryIt->second.isInStableDir();
113 }
114
115 /// Update which module that is being actively traversed.
116 void visitModuleFile(ModuleFileName Filename, serialization::ModuleKind Kind,
117 bool DirectlyImported) override {
118 // If the CurrentFile is not
119 // considered stable, update any of it's transitive dependents.
120 auto PrebuiltEntryIt = PrebuiltModulesASTMap.find(Key: CurrentFile);
121 if ((PrebuiltEntryIt != PrebuiltModulesASTMap.end()) &&
122 !PrebuiltEntryIt->second.isInStableDir())
123 PrebuiltEntryIt->second.updateDependentsNotInStableDirs(
124 PrebuiltModulesMap&: PrebuiltModulesASTMap);
125 CurrentFile = Filename.str();
126 }
127
128 /// Check the header search options for a given module when considering
129 /// if the module comes from stable directories.
130 bool ReadHeaderSearchOptions(const HeaderSearchOptions &HSOpts,
131 StringRef ModuleFilename, StringRef ContextHash,
132 bool Complain) override {
133
134 auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(Key: CurrentFile);
135 PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second;
136 if (PrebuiltMapEntry.second)
137 PrebuiltModule.setInStableDir(!StableDirs.empty());
138
139 if (PrebuiltModule.isInStableDir())
140 PrebuiltModule.setInStableDir(areOptionsInStableDir(Directories: StableDirs, HSOpts));
141
142 return false;
143 }
144
145 /// Accumulate vfsoverlays used to build these prebuilt modules.
146 bool ReadHeaderSearchPaths(const HeaderSearchOptions &HSOpts,
147 bool Complain) override {
148
149 auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(Key: CurrentFile);
150 PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second;
151 if (PrebuiltMapEntry.second)
152 PrebuiltModule.setInStableDir(!StableDirs.empty());
153
154 PrebuiltModule.setVFS(
155 llvm::StringSet<>(llvm::from_range, HSOpts.VFSOverlayFiles));
156
157 return checkHeaderSearchPaths(
158 HSOpts, ExistingHSOpts, Diags: Complain ? &Diags : nullptr, LangOpts: ExistingLangOpts);
159 }
160
161private:
162 PrebuiltModuleFilesT &PrebuiltModuleFiles;
163 llvm::SmallVector<std::string> &NewModuleFiles;
164 PrebuiltModulesAttrsMap &PrebuiltModulesASTMap;
165 const HeaderSearchOptions &ExistingHSOpts;
166 const LangOptions &ExistingLangOpts;
167 DiagnosticsEngine &Diags;
168 std::string CurrentFile;
169 const ArrayRef<StringRef> StableDirs;
170};
171} // namespace
172
173/// Visit the given prebuilt module and collect all of the modules it
174/// transitively imports and contributing input files.
175static bool visitPrebuiltModule(StringRef PrebuiltModuleFilename,
176 CompilerInstance &CI,
177 PrebuiltModuleFilesT &ModuleFiles,
178 PrebuiltModulesAttrsMap &PrebuiltModulesASTMap,
179 DiagnosticsEngine &Diags,
180 const ArrayRef<StringRef> StableDirs) {
181 // List of module files to be processed.
182 llvm::SmallVector<std::string> Worklist;
183
184 PrebuiltModuleListener Listener(ModuleFiles, Worklist, PrebuiltModulesASTMap,
185 CI.getHeaderSearchOpts(), CI.getLangOpts(),
186 Diags, StableDirs);
187
188 Listener.visitModuleFile(Filename: ModuleFileName::makeExplicit(Name: PrebuiltModuleFilename),
189 Kind: serialization::MK_ExplicitModule,
190 /*DirectlyImported=*/true);
191 if (ASTReader::readASTFileControlBlock(
192 Filename: PrebuiltModuleFilename, FileMgr&: CI.getFileManager(), ModCache: CI.getModuleCache(),
193 PCHContainerRdr: CI.getPCHContainerReader(),
194 /*FindModuleFileExtensions=*/false, Listener,
195 /*ValidateDiagnosticOptions=*/false, ClientLoadCapabilities: ASTReader::ARR_OutOfDate))
196 return true;
197
198 while (!Worklist.empty()) {
199 // FIXME: This is assuming the PCH only refers to explicitly-built modules,
200 // which technically is not guaranteed. To remove the assumption, we'd need
201 // to also rework how the module files are handled to the scan, specifically
202 // change the values of HeaderSearchOptions::PrebuiltModuleFiles from plain
203 // paths to ModuleFileName.
204 Listener.visitModuleFile(Filename: ModuleFileName::makeExplicit(Name: Worklist.back()),
205 Kind: serialization::MK_ExplicitModule,
206 /*DirectlyImported=*/false);
207 if (ASTReader::readASTFileControlBlock(
208 Filename: Worklist.pop_back_val(), FileMgr&: CI.getFileManager(), ModCache: CI.getModuleCache(),
209 PCHContainerRdr: CI.getPCHContainerReader(),
210 /*FindModuleFileExtensions=*/false, Listener,
211 /*ValidateDiagnosticOptions=*/false))
212 return true;
213 }
214 return false;
215}
216
217/// Transform arbitrary file name into an object-like file name.
218static std::string makeObjFileName(StringRef FileName) {
219 SmallString<128> ObjFileName(FileName);
220 llvm::sys::path::replace_extension(path&: ObjFileName, extension: "o");
221 return std::string(ObjFileName);
222}
223
224/// Deduce the dependency target based on the output file and input files.
225static std::string
226deduceDepTarget(const std::string &OutputFile,
227 const SmallVectorImpl<FrontendInputFile> &InputFiles) {
228 if (OutputFile != "-")
229 return OutputFile;
230
231 if (InputFiles.empty() || !InputFiles.front().isFile())
232 return "clang-scan-deps\\ dependency";
233
234 return makeObjFileName(FileName: InputFiles.front().getFile());
235}
236
237// Clang implements -D and -U by splatting text into a predefines buffer. This
238// allows constructs such as `-DFඞ=3 "-D F\u{0D9E} 4 3 2”` to be accepted and
239// define the same macro, or adding C++ style comments before the macro name.
240//
241// This function checks that the first non-space characters in the macro
242// obviously form an identifier that can be uniqued on without lexing. Failing
243// to do this could lead to changing the final definition of a macro.
244//
245// We could set up a preprocessor and actually lex the name, but that's very
246// heavyweight for a situation that will almost never happen in practice.
247static std::optional<StringRef> getSimpleMacroName(StringRef Macro) {
248 StringRef Name = Macro.split(Separator: "=").first.ltrim(Chars: " \t");
249 std::size_t I = 0;
250
251 auto FinishName = [&]() -> std::optional<StringRef> {
252 StringRef SimpleName = Name.slice(Start: 0, End: I);
253 if (SimpleName.empty())
254 return std::nullopt;
255 return SimpleName;
256 };
257
258 for (; I != Name.size(); ++I) {
259 switch (Name[I]) {
260 case '(': // Start of macro parameter list
261 case ' ': // End of macro name
262 case '\t':
263 return FinishName();
264 case '_':
265 continue;
266 default:
267 if (llvm::isAlnum(C: Name[I]))
268 continue;
269 return std::nullopt;
270 }
271 }
272 return FinishName();
273}
274
275static void canonicalizeDefines(PreprocessorOptions &PPOpts) {
276 using MacroOpt = std::pair<StringRef, std::size_t>;
277 std::vector<MacroOpt> SimpleNames;
278 SimpleNames.reserve(n: PPOpts.Macros.size());
279 std::size_t Index = 0;
280 for (const auto &M : PPOpts.Macros) {
281 auto SName = getSimpleMacroName(Macro: M.first);
282 // Skip optimizing if we can't guarantee we can preserve relative order.
283 if (!SName)
284 return;
285 SimpleNames.emplace_back(args&: *SName, args&: Index);
286 ++Index;
287 }
288
289 llvm::stable_sort(Range&: SimpleNames, C: llvm::less_first());
290 // Keep the last instance of each macro name by going in reverse
291 auto NewEnd = std::unique(
292 first: SimpleNames.rbegin(), last: SimpleNames.rend(),
293 binary_pred: [](const MacroOpt &A, const MacroOpt &B) { return A.first == B.first; });
294 SimpleNames.erase(first: SimpleNames.begin(), last: NewEnd.base());
295
296 // Apply permutation.
297 decltype(PPOpts.Macros) NewMacros;
298 NewMacros.reserve(n: SimpleNames.size());
299 for (std::size_t I = 0, E = SimpleNames.size(); I != E; ++I) {
300 std::size_t OriginalIndex = SimpleNames[I].second;
301 // We still emit undefines here as they may be undefining a predefined macro
302 NewMacros.push_back(x: std::move(PPOpts.Macros[OriginalIndex]));
303 }
304 std::swap(x&: PPOpts.Macros, y&: NewMacros);
305}
306
307namespace {
308class ScanningDependencyDirectivesGetter : public DependencyDirectivesGetter {
309 DependencyScanningWorkerFilesystem *DepFS;
310
311public:
312 ScanningDependencyDirectivesGetter(FileManager &FileMgr) : DepFS(nullptr) {
313 FileMgr.getVirtualFileSystem().visit(Callback: [&](llvm::vfs::FileSystem &FS) {
314 auto *DFS = llvm::dyn_cast<DependencyScanningWorkerFilesystem>(Val: &FS);
315 if (DFS) {
316 assert(!DepFS && "Found multiple scanning VFSs");
317 DepFS = DFS;
318 }
319 });
320 assert(DepFS && "Did not find scanning VFS");
321 }
322
323 std::unique_ptr<DependencyDirectivesGetter>
324 cloneFor(FileManager &FileMgr) override {
325 return std::make_unique<ScanningDependencyDirectivesGetter>(args&: FileMgr);
326 }
327
328 std::optional<ArrayRef<dependency_directives_scan::Directive>>
329 operator()(FileEntryRef File) override {
330 return DepFS->getDirectiveTokens(Path: File.getName());
331 }
332};
333} // namespace
334
335/// Sanitize diagnostic options for dependency scan.
336static void sanitizeDiagOpts(DiagnosticOptions &DiagOpts) {
337 // Don't print 'X warnings and Y errors generated'.
338 DiagOpts.ShowCarets = false;
339 // Don't write out diagnostic file.
340 DiagOpts.DiagnosticSerializationFile.clear();
341 // Don't emit warnings except for scanning specific warnings.
342 // TODO: It would be useful to add a more principled way to ignore all
343 // warnings that come from source code. The issue is that we need to
344 // ignore warnings that could be surpressed by
345 // `#pragma clang diagnostic`, while still allowing some scanning
346 // warnings for things we're not ready to turn into errors yet.
347 // See `test/ClangScanDeps/diagnostic-pragmas.c` for an example.
348 llvm::erase_if(C&: DiagOpts.Warnings, P: [](StringRef Warning) {
349 return llvm::StringSwitch<bool>(Warning)
350 .Cases(CaseStrings: {"pch-vfs-diff", "error=pch-vfs-diff"}, Value: false)
351 .StartsWith(S: "no-error=", Value: false)
352 .Default(Value: true);
353 });
354}
355
356static std::unique_ptr<CompilerInvocation>
357createCompilerInvocation(ArrayRef<std::string> CommandLine,
358 DiagnosticsEngine &Diags) {
359 llvm::opt::ArgStringList Argv;
360 for (const std::string &Str : ArrayRef(CommandLine).drop_front())
361 Argv.push_back(Elt: Str.c_str());
362
363 auto Invocation = std::make_unique<CompilerInvocation>();
364 if (!CompilerInvocation::CreateFromArgs(Res&: *Invocation, CommandLineArgs: Argv, Diags)) {
365 // FIXME: Should we just go on like cc1_main does?
366 return nullptr;
367 }
368 return Invocation;
369}
370
371static void initializeScanCompilerInstance(
372 CompilerInstance &ScanInstance,
373 IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
374 DiagnosticConsumer *DiagConsumer, DependencyScanningService &Service,
375 IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS) {
376 ScanInstance.setBuildingModule(false);
377 ScanInstance.createVirtualFileSystem(BaseFS: FS, DC: DiagConsumer);
378 ScanInstance.createDiagnostics(Client: DiagConsumer, /*ShouldOwnClient=*/false);
379 if (!Service.getOpts().EmitWarnings)
380 ScanInstance.getDiagnostics().setIgnoreAllWarnings(true);
381 ScanInstance.createFileManager();
382 ScanInstance.createSourceManager();
383
384 // Use DepFS for getting the dependency directives if requested to do so.
385 if (Service.getOpts().Mode == ScanningMode::DependencyDirectivesScan)
386 ScanInstance.setDependencyDirectivesGetter(
387 std::make_unique<ScanningDependencyDirectivesGetter>(
388 args&: ScanInstance.getFileManager()));
389}
390
391static std::shared_ptr<CompilerInvocation>
392createScanCompilerInvocation(const CompilerInvocation &Invocation,
393 const DependencyScanningService &Service,
394 DependencyActionController &Controller) {
395 auto ScanInvocation = std::make_shared<CompilerInvocation>(args: Invocation);
396
397 sanitizeDiagOpts(DiagOpts&: ScanInvocation->getDiagnosticOpts());
398
399 ScanInvocation->getPreprocessorOpts().AllowPCHWithDifferentModulesCachePath =
400 true;
401
402 if (ScanInvocation->getHeaderSearchOpts()
403 .ModulesValidateOncePerBuildSession ||
404 Service.getOpts().ValidateAgainstInvalidatedPaths)
405 ScanInvocation->getHeaderSearchOpts().BuildSessionTimestamp =
406 Service.getOpts().BuildSessionTimestamp;
407 if (Service.getOpts().ValidateAgainstInvalidatedPaths)
408 ScanInvocation->getHeaderSearchOpts().ModulesValidateDirectoryDependencies =
409 true;
410
411 ScanInvocation->getFrontendOpts().DisableFree = false;
412 ScanInvocation->getFrontendOpts().GenerateGlobalModuleIndex = false;
413 ScanInvocation->getFrontendOpts().UseGlobalModuleIndex = false;
414 ScanInvocation->getFrontendOpts().GenReducedBMI = false;
415 ScanInvocation->getFrontendOpts().ModuleOutputPath.clear();
416 // This will prevent us compiling individual modules asynchronously since
417 // FileManager is not thread-safe, but it does improve performance for now.
418 ScanInvocation->getFrontendOpts().ModulesShareFileManager = true;
419 ScanInvocation->getHeaderSearchOpts().ModuleFormat = "raw";
420 ScanInvocation->getHeaderSearchOpts().ModulesIncludeVFSUsage =
421 any(Val: Service.getOpts().OptimizeArgs & ScanningOptimizations::VFS);
422
423 // Consider different header search and diagnostic options to create
424 // different modules. This avoids the unsound aliasing of module PCMs.
425 //
426 // TODO: Implement diagnostic bucketing to reduce the impact of strict
427 // context hashing.
428 ScanInvocation->getHeaderSearchOpts().ModulesStrictContextHash = true;
429 ScanInvocation->getHeaderSearchOpts().ModulesSerializeOnlyPreprocessor = true;
430 ScanInvocation->getHeaderSearchOpts().ModulesSkipDiagnosticOptions = true;
431 ScanInvocation->getHeaderSearchOpts().ModulesSkipHeaderSearchPaths = true;
432 ScanInvocation->getHeaderSearchOpts().ModulesSkipPragmaDiagnosticMappings =
433 true;
434 ScanInvocation->getHeaderSearchOpts().ModulesForceValidateUserHeaders = false;
435
436 // FIXME: Do this even with PCHs by marking the option as something like
437 // "preprocessor benign" in LangOptions.def so that it passes the
438 // compatibility checks in ASTReader.
439 if (ScanInvocation->getPreprocessorOpts().ImplicitPCHInclude.empty()) {
440 // Application extension only affects the handling of availability
441 // attributes, which cannot change the dependencies.
442 ScanInvocation->getLangOpts().AppExt = false;
443 }
444
445 // Ensure that the scanner does not create new dependency collectors,
446 // and thus won't write out the extra '.d' files to disk.
447 ScanInvocation->getDependencyOutputOpts() = {};
448
449 Controller.initializeScanInvocation(ScanInvocation&: *ScanInvocation);
450
451 return ScanInvocation;
452}
453
454static llvm::SmallVector<StringRef>
455getInitialStableDirs(const CompilerInstance &ScanInstance) {
456 // Create a collection of stable directories derived from the ScanInstance
457 // for determining whether module dependencies would fully resolve from
458 // those directories.
459 llvm::SmallVector<StringRef> StableDirs;
460 const StringRef Sysroot = ScanInstance.getHeaderSearchOpts().Sysroot;
461 if (!Sysroot.empty() && (llvm::sys::path::root_directory(path: Sysroot) != Sysroot))
462 StableDirs = {Sysroot, ScanInstance.getHeaderSearchOpts().ResourceDir};
463 return StableDirs;
464}
465
466static std::optional<PrebuiltModulesAttrsMap>
467computePrebuiltModulesASTMap(CompilerInstance &ScanInstance,
468 llvm::SmallVector<StringRef> &StableDirs) {
469 // Store a mapping of prebuilt module files and their properties like header
470 // search options. This will prevent the implicit build to create duplicate
471 // modules and will force reuse of the existing prebuilt module files
472 // instead.
473 PrebuiltModulesAttrsMap PrebuiltModulesASTMap;
474
475 if (!ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty())
476 if (visitPrebuiltModule(
477 PrebuiltModuleFilename: ScanInstance.getPreprocessorOpts().ImplicitPCHInclude, CI&: ScanInstance,
478 ModuleFiles&: ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles,
479 PrebuiltModulesASTMap, Diags&: ScanInstance.getDiagnostics(), StableDirs))
480 return {};
481
482 return PrebuiltModulesASTMap;
483}
484
485static std::unique_ptr<DependencyOutputOptions>
486createDependencyOutputOptions(const CompilerInvocation &Invocation) {
487 auto Opts = std::make_unique<DependencyOutputOptions>(
488 args: Invocation.getDependencyOutputOpts());
489 // We need at least one -MT equivalent for the generator of make dependency
490 // files to work.
491 if (Opts->Targets.empty())
492 Opts->Targets = {deduceDepTarget(OutputFile: Invocation.getFrontendOpts().OutputFile,
493 InputFiles: Invocation.getFrontendOpts().Inputs)};
494 Opts->IncludeSystemHeaders = true;
495
496 return Opts;
497}
498
499static std::shared_ptr<ModuleDepCollector>
500initializeScanInstanceDependencyCollector(
501 CompilerInstance &ScanInstance,
502 std::unique_ptr<DependencyOutputOptions> DepOutputOpts,
503 DependencyScanningService &Service, CompilerInvocation &Inv,
504 DependencyActionController &Controller,
505 PrebuiltModulesAttrsMap PrebuiltModulesASTMap,
506 SmallVector<StringRef> &StableDirs) {
507 auto MDC = std::make_shared<ModuleDepCollector>(
508 args&: Service, args: std::move(DepOutputOpts), args&: ScanInstance, args&: Controller, args&: Inv,
509 args: std::move(PrebuiltModulesASTMap), args&: StableDirs);
510 ScanInstance.addDependencyCollector(Listener: MDC);
511 return MDC;
512}
513
514namespace {
515/// Manages (and terminates) the asynchronous compilation of modules.
516class AsyncModuleCompiles {
517 std::mutex Mutex;
518 bool Stop = false;
519 // FIXME: Have the service own a thread pool and use that instead.
520 std::vector<std::thread> Compiles;
521
522public:
523 /// Registers the module compilation, unless this instance is about to be
524 /// destroyed.
525 void add(llvm::unique_function<void()> Compile) {
526 std::lock_guard<std::mutex> Lock(Mutex);
527 if (!Stop)
528 Compiles.emplace_back(args: std::move(Compile));
529 }
530
531 ~AsyncModuleCompiles() {
532 {
533 std::lock_guard<std::mutex> Lock(Mutex);
534 Stop = true;
535 }
536 for (std::thread &Compile : Compiles)
537 Compile.join();
538 }
539};
540
541struct SingleModuleWithAsyncModuleCompiles : PreprocessOnlyAction {
542 DependencyScanningService &Service;
543 DependencyActionController &Controller;
544 AsyncModuleCompiles &Compiles;
545
546 SingleModuleWithAsyncModuleCompiles(DependencyScanningService &Service,
547 DependencyActionController &Controller,
548 AsyncModuleCompiles &Compiles)
549 : Service(Service), Controller(Controller), Compiles(Compiles) {}
550
551 bool BeginSourceFileAction(CompilerInstance &CI) override;
552};
553
554/// Runs the preprocessor on a TU with single-module-parse-mode and compiles
555/// modules asynchronously without blocking or importing them.
556struct SingleTUWithAsyncModuleCompiles : PreprocessOnlyAction {
557 DependencyScanningService &Service;
558 DependencyActionController &Controller;
559 AsyncModuleCompiles &Compiles;
560
561 SingleTUWithAsyncModuleCompiles(DependencyScanningService &Service,
562 DependencyActionController &Controller,
563 AsyncModuleCompiles &Compiles)
564 : Service(Service), Controller(Controller), Compiles(Compiles) {}
565
566 bool BeginSourceFileAction(CompilerInstance &CI) override;
567};
568
569/// The preprocessor callback that takes care of initiating an asynchronous
570/// module compilation if needed.
571struct AsyncModuleCompile : PPCallbacks {
572 CompilerInstance &CI;
573 DependencyScanningService &Service;
574 DependencyActionController &Controller;
575 AsyncModuleCompiles &Compiles;
576
577 AsyncModuleCompile(CompilerInstance &CI, DependencyScanningService &Service,
578 DependencyActionController &Controller,
579 AsyncModuleCompiles &Compiles)
580 : CI(CI), Service(Service), Controller(Controller), Compiles(Compiles) {}
581
582 void moduleLoadSkipped(Module *M) override {
583 M = M->getTopLevelModule();
584
585 HeaderSearch &HS = CI.getPreprocessor().getHeaderSearchInfo();
586 ModuleCache &ModCache = CI.getModuleCache();
587 ModuleFileName ModuleFileName = HS.getCachedModuleFileName(Module: M);
588
589 uint64_t Timestamp = ModCache.getModuleTimestamp(ModuleFilename: ModuleFileName);
590 // Someone else already built/validated the PCM.
591 if (Timestamp > CI.getHeaderSearchOpts().BuildSessionTimestamp)
592 return;
593
594 if (!CI.getASTReader())
595 CI.createASTReader();
596 SmallVector<ASTReader::ImportedModule, 0> Imported;
597 // Only calling ReadASTCore() to avoid the expensive eager deserialization
598 // of the clang::Module objects in ReadAST().
599 // FIXME: Consider doing this in the new thread depending on how expensive
600 // the read turns out to be.
601 switch (CI.getASTReader()->ReadASTCore(
602 FileName: ModuleFileName, Type: serialization::MK_ImplicitModule, ImportLoc: SourceLocation(),
603 ImportedBy: nullptr, Loaded&: Imported, ExpectedSize: {}, ExpectedModTime: {}, ExpectedSignature: {},
604 ClientLoadCapabilities: ASTReader::ARR_OutOfDate | ASTReader::ARR_Missing |
605 ASTReader::ARR_TreatModuleWithErrorsAsOutOfDate)) {
606 case ASTReader::Success:
607 // We successfully read a valid, up-to-date PCM.
608 // FIXME: This could update the timestamp. Regular calls to
609 // ASTReader::ReadAST() would do so unless they encountered corrupted
610 // AST block, corrupted extension block, or did not read the expected
611 // top-level module.
612 return;
613 case ASTReader::OutOfDate:
614 case ASTReader::Missing:
615 // The most interesting case.
616 break;
617 default:
618 // Let the regular scan diagnose this.
619 return;
620 }
621
622 auto Lock = ModCache.getLock(ModuleFilename: ModuleFileName);
623 bool Owned;
624 llvm::Error LockErr = Lock->tryLock().moveInto(Value&: Owned);
625 // Someone else is building the PCM right now.
626 if (!LockErr && !Owned)
627 return;
628 // We should build the PCM.
629 IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS =
630 llvm::makeIntrusiveRefCnt<DependencyScanningWorkerFilesystem>(
631 A&: Service, A: Service.getOpts().MakeVFS());
632 VFS = createVFSFromCompilerInvocation(CI: CI.getInvocation(),
633 Diags&: CI.getDiagnostics(), BaseFS: std::move(VFS));
634 auto DC = std::make_unique<DiagnosticConsumer>();
635 auto MC = makeInProcessModuleCache(Entries&: Service.getModuleCacheEntries(),
636 Logger&: Service.getLogger());
637 CompilerInstance::ThreadSafeCloneConfig CloneConfig(std::move(VFS), *DC,
638 std::move(MC));
639 auto ModCI1 = CI.cloneForModuleCompile(ImportLoc: SourceLocation(), Module: M, ModuleFileName,
640 ThreadSafeConfig: CloneConfig);
641 auto ModCI2 = CI.cloneForModuleCompile(ImportLoc: SourceLocation(), Module: M, ModuleFileName,
642 ThreadSafeConfig: CloneConfig);
643
644 auto ModController = Controller.clone();
645
646 // Note: This lock belongs to a module cache that might not outlive the
647 // thread. This works, because the in-process lock only refers to an object
648 // managed by the service, which does outlive the thread.
649 Compiles.add(Compile: [Lock = std::move(Lock), ModCI1 = std::move(ModCI1),
650 ModCI2 = std::move(ModCI2), DC = std::move(DC),
651 ModController = std::move(ModController), Service = &Service,
652 Compiles = &Compiles] {
653 llvm::CrashRecoveryContext CRC;
654 (void)CRC.RunSafely(Fn: [&] {
655 // Quickly discovers and compiles modules for the real scan below.
656 SingleModuleWithAsyncModuleCompiles Action1(*Service, *ModController,
657 *Compiles);
658 (void)ModCI1->ExecuteAction(Act&: Action1);
659 // The real scan below.
660 ModCI2->getPreprocessorOpts().SingleModuleParseMode = false;
661 GenerateModuleFromModuleMapAction Action2;
662 (void)ModCI2->ExecuteAction(Act&: Action2);
663 });
664 });
665 }
666};
667
668bool SingleModuleWithAsyncModuleCompiles::BeginSourceFileAction(
669 CompilerInstance &CI) {
670 CI.getInvocation().getPreprocessorOpts().SingleModuleParseMode = true;
671 CI.getPreprocessor().addPPCallbacks(
672 C: std::make_unique<AsyncModuleCompile>(args&: CI, args&: Service, args&: Controller, args&: Compiles));
673 return true;
674}
675
676bool SingleTUWithAsyncModuleCompiles::BeginSourceFileAction(
677 CompilerInstance &CI) {
678 CI.getInvocation().getPreprocessorOpts().SingleModuleParseMode = true;
679 CI.getPreprocessor().addPPCallbacks(
680 C: std::make_unique<AsyncModuleCompile>(args&: CI, args&: Service, args&: Controller, args&: Compiles));
681 return true;
682}
683} // namespace
684
685static void runTUModulePrescan(CompilerInstance &PrescanCI,
686 DependencyScanningService &Service,
687 DependencyActionController &Controller,
688 AsyncModuleCompiles &Compiles) {
689 SingleTUWithAsyncModuleCompiles Action(Service, Controller, Compiles);
690 (void)PrescanCI.ExecuteAction(Act&: Action);
691}
692
693namespace clang {
694namespace dependencies {
695
696std::unique_ptr<DiagnosticOptions>
697createScanningDiagOptions(ArrayRef<std::string> CommandLine) {
698 std::vector<const char *> CCommandLine(CommandLine.size(), nullptr);
699 llvm::transform(Range&: CommandLine, d_first: CCommandLine.begin(),
700 F: [](const std::string &Str) { return Str.c_str(); });
701 auto DiagOpts = CreateAndPopulateDiagOpts(Argv: CCommandLine);
702 sanitizeDiagOpts(DiagOpts&: *DiagOpts);
703 return DiagOpts;
704}
705
706namespace {
707/// Find the module maps that define modules without loading them.
708class ModuleMapLookup {
709public:
710 explicit ModuleMapLookup(CompilerInstance &CI)
711 : CI(CI), HSOpts(CI.getHeaderSearchOpts()),
712 Diags(DiagnosticIDs::create(), DiagOpts, new IgnoringDiagConsumer()),
713 SourceMgr(Diags, CI.getFileManager()),
714 HS(HSOpts, SourceMgr, Diags, CI.getLangOpts(), &CI.getTarget()) {
715 HSOpts.LazyLoadModuleMaps = true;
716 // Ignore verbose mode as it's already respected by the real scan.
717 HSOpts.Verbose = false;
718 ApplyHeaderSearchOptions(HS, HSOpts, Lang: CI.getLangOpts(),
719 triple: CI.getTarget().getTriple());
720 }
721
722 llvm::StringMap<std::string>
723 computeModuleMapsForModules(ArrayRef<StringRef> ModuleNames) {
724 // Resolve any modulemaps passed directly first.
725 for (const std::string &Filename : CI.getFrontendOpts().ModuleMapFiles)
726 if (OptionalFileEntryRef File =
727 CI.getFileManager().getOptionalFileRef(Filename))
728 HS.parseModuleMapFile(File: *File, /*IsSystem=*/false,
729 /*ImplicitlyDiscovered=*/false);
730
731 const bool HasPrebuiltModules = !HSOpts.PrebuiltModuleFiles.empty() ||
732 !HSOpts.PrebuiltModulePaths.empty();
733
734 llvm::StringMap<std::string> Result;
735 // Then resolve modulemaps through header search.
736 for (StringRef Name : ModuleNames) {
737 // Skip over prebuilt modules.
738 if (HasPrebuiltModules && !HS.getPrebuiltModuleFileName(ModuleName: Name).empty())
739 continue;
740 OptionalFileEntryRef File = HS.lookupModuleMapFileForModule(
741 ModuleName: Name, /*AllowExtraModuleMapSearch=*/true);
742 // Skip over modules that didn't find a modulemap.
743 if (!File)
744 continue;
745 // Canonicalize the path the same way as for the real scan.
746 SmallString<128> Path = File->getNameAsRequested();
747 HS.getModuleMap().canonicalizeModuleMapPath(Path);
748 Result[Name] = std::string(Path);
749 }
750 return Result;
751 }
752
753private:
754 CompilerInstance &CI;
755 HeaderSearchOptions HSOpts;
756 DiagnosticOptions DiagOpts;
757 DiagnosticsEngine Diags;
758 SourceManager SourceMgr;
759 HeaderSearch HS;
760};
761} // namespace
762
763class CompilerInstanceWithContext {
764 // Context
765 DependencyScanningWorker &Worker;
766 llvm::StringRef CWD;
767 std::vector<std::string> CommandLine;
768
769 // Context - compiler invocation
770 std::unique_ptr<CompilerInvocation> OriginalInvocation;
771
772 // Context - output options
773 std::unique_ptr<DependencyOutputOptions> OutputOpts;
774
775 // Context - stable directory handling
776 llvm::SmallVector<StringRef> StableDirs;
777 PrebuiltModulesAttrsMap PrebuiltModuleASTMap;
778
779 // Context - used by AsyncScan's prescan pass
780 IntrusiveRefCntPtr<llvm::vfs::FileSystem> ScanFS;
781
782 // Compiler Instance
783 std::unique_ptr<CompilerInstance> CIPtr;
784
785 // Source location offset.
786 int32_t SrcLocOffset = 0;
787
788 CompilerInstanceWithContext(DependencyScanningWorker &Worker, StringRef CWD,
789 ArrayRef<std::string> CMD)
790 : Worker(Worker), CWD(CWD), CommandLine(CMD.begin(), CMD.end()) {}
791
792 bool initialize(DependencyActionController &Controller,
793 DiagnosticsEngine &DiagEngine,
794 IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS) {
795 {
796 auto LogLine = Worker.Service.getLogger().log();
797 LogLine.logArray(Prefix: "init_compiler_instance_with_context:", Sep: " ",
798 Arr: CommandLine);
799 }
800
801 ScanFS = Worker.makeEffectiveVFS(WorkingDirectory: CWD, OverlayFS: std::move(OverlayFS));
802 OriginalInvocation = createCompilerInvocation(CommandLine, Diags&: DiagEngine);
803 if (!OriginalInvocation) {
804 DiagEngine.Report(DiagID: diag::err_fe_expected_compiler_job)
805 << llvm::join(R&: CommandLine, Separator: " ");
806 return false;
807 }
808
809 return initializeScanInstance(Controller, DiagConsumer: DiagEngine.getClient());
810 }
811
812 bool initializeScanInstance(DependencyActionController &Controller,
813 DiagnosticConsumer *DiagConsumer) {
814 assert(OriginalInvocation && ScanFS &&
815 "OriginalInvocation and ScanFS must be set before this call");
816
817 if (any(Val: Worker.Service.getOpts().OptimizeArgs &
818 ScanningOptimizations::Macros))
819 canonicalizeDefines(PPOpts&: OriginalInvocation->getPreprocessorOpts());
820
821 // Create the CompilerInstance.
822 std::shared_ptr<ModuleCache> ModCache = makeInProcessModuleCache(
823 Entries&: Worker.Service.getModuleCacheEntries(), Logger&: Worker.Service.getLogger());
824 CIPtr = std::make_unique<CompilerInstance>(
825 args: createScanCompilerInvocation(Invocation: *OriginalInvocation, Service: Worker.Service,
826 Controller),
827 args&: Worker.PCHContainerOps, args: std::move(ModCache));
828 auto &CI = *CIPtr;
829
830 initializeScanCompilerInstance(ScanInstance&: CI, FS: ScanFS, DiagConsumer, Service&: Worker.Service,
831 DepFS: Worker.DepFS);
832
833 StableDirs = getInitialStableDirs(ScanInstance: CI);
834 auto MaybePrebuiltModulesASTMap =
835 computePrebuiltModulesASTMap(ScanInstance&: CI, StableDirs);
836 if (!MaybePrebuiltModulesASTMap)
837 return false;
838
839 PrebuiltModuleASTMap = std::move(*MaybePrebuiltModulesASTMap);
840 OutputOpts = createDependencyOutputOptions(Invocation: *OriginalInvocation);
841
842 // We do not create the target in initializeScanCompilerInstance because
843 // setting it here is unique for by-name lookups. We create the target only
844 // once here, and the information is reused for all computeDependencies
845 // calls. We do not need to call createTarget explicitly if we go through
846 // CompilerInstance::ExecuteAction to perform scanning.
847 return CI.createTarget();
848 }
849
850 bool prescanModulesAsync(AsyncModuleCompiles &Compiles,
851 DependencyActionController &Controller) {
852 auto ModCache = makeInProcessModuleCache(
853 Entries&: Worker.Service.getModuleCacheEntries(), Logger&: Worker.Service.getLogger());
854 CompilerInstance PrescanCI(
855 std::make_shared<CompilerInvocation>(args&: CIPtr->getInvocation()),
856 Worker.PCHContainerOps, std::move(ModCache));
857
858 DiagnosticConsumer DiagConsumer;
859 initializeScanCompilerInstance(ScanInstance&: PrescanCI, FS: ScanFS, DiagConsumer: &DiagConsumer,
860 Service&: Worker.Service, DepFS: Worker.DepFS);
861
862 // FIXME: reuse the StableDirs/PrebuiltModuleASTMap computed in
863 // initialize().
864 SmallVector<StringRef> PrescanStableDirs = getInitialStableDirs(ScanInstance: PrescanCI);
865 if (!computePrebuiltModulesASTMap(ScanInstance&: PrescanCI, StableDirs&: PrescanStableDirs))
866 return false;
867
868 if (PrescanCI.getFrontendOpts().ProgramAction == frontend::GeneratePCH)
869 PrescanCI.getLangOpts().CompilingPCH = true;
870
871 runTUModulePrescan(PrescanCI, Service&: Worker.Service, Controller, Compiles);
872 return true;
873 }
874
875public:
876 static std::optional<CompilerInstanceWithContext>
877 initializeFromCC1Commandline(
878 DependencyScanningWorker &Worker, StringRef CWD,
879 ArrayRef<std::string> CC1CommandLine, DiagnosticsEngine &DiagEngine,
880 IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS,
881 DependencyActionController &Controller) {
882 CompilerInstanceWithContext CIWC(Worker, CWD, CC1CommandLine);
883 if (!CIWC.initialize(Controller, DiagEngine, OverlayFS: std::move(OverlayFS)))
884 return std::nullopt;
885 return std::move(CIWC);
886 }
887
888 /// Initialize from a cc1 command line, reporting diagnostics to
889 /// \p DiagConsumer.
890 static std::optional<CompilerInstanceWithContext>
891 initializeFromCC1Commandline(
892 DependencyScanningWorker &Worker, StringRef CWD,
893 ArrayRef<std::string> CC1CommandLine, DiagnosticConsumer &DiagConsumer,
894 IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS,
895 DependencyActionController &Controller) {
896 auto FS = Worker.makeEffectiveVFS(WorkingDirectory: CWD, OverlayFS);
897 auto DiagOpts = createScanningDiagOptions(CommandLine: CC1CommandLine);
898 auto DiagEngine =
899 CompilerInstance::createDiagnostics(VFS&: *FS, Opts&: *DiagOpts, Client: &DiagConsumer,
900 /*ShouldOwnClient=*/false);
901 return initializeFromCC1Commandline(Worker, CWD, CC1CommandLine,
902 DiagEngine&: *DiagEngine, OverlayFS: std::move(OverlayFS),
903 Controller);
904 }
905
906 bool computeDependencies(StringRef ModuleName, DependencyConsumer &Consumer,
907 DependencyActionController &Controller) {
908 Worker.Service.getLogger().log() << "start scan_by_name: " << ModuleName;
909 llvm::scope_exit ExitLogging([&] {
910 Worker.Service.getLogger().log() << "finish scan_by_name: " << ModuleName;
911 });
912 if (SrcLocOffset >= DependencyScanningWorker::MaxNumOfByNameQueries)
913 llvm::report_fatal_error(reason: "exceeded maximum by-name scans for worker");
914
915 assert(CIPtr && "CIPtr must be initialized before calling this method");
916 auto &CI = *CIPtr;
917
918 // We need to reset the diagnostics, so that the diagnostics issued
919 // during a previous computeDependencies call do not affect the current
920 // call. If we do not reset, we may inherit fatal errors from a previous
921 // call.
922 CI.getDiagnostics().Reset();
923
924 // We create this cleanup object because computeDependencies may exit
925 // early with errors.
926 llvm::scope_exit CleanUp([&]() {
927 CI.clearDependencyCollectors();
928 // The preprocessor may not be created at the entry of this method,
929 // but it must have been created when this method returns, whether
930 // there are errors during scanning or not.
931 CI.getPreprocessor().removePPCallbacks();
932 });
933
934 auto MDC = initializeScanInstanceDependencyCollector(
935 ScanInstance&: CI, DepOutputOpts: std::make_unique<DependencyOutputOptions>(args&: *OutputOpts),
936 Service&: Worker.Service,
937 /* The MDC's constructor makes a copy of the OriginalInvocation, so
938 we can pass it in without worrying that it might be changed across
939 invocations of computeDependencies. */
940 Inv&: *OriginalInvocation, Controller, PrebuiltModulesASTMap: PrebuiltModuleASTMap, StableDirs);
941
942 CompilerInvocation ModuleInvocation(*OriginalInvocation);
943 if (!Controller.initialize(ScanInstance&: CI, NewInvocation&: ModuleInvocation))
944 return false;
945
946 if (!SrcLocOffset) {
947 // When SrcLocOffset is zero, we are at the beginning of the fake source
948 // file. In this case, we call BeginSourceFile to initialize.
949 std::unique_ptr<FrontendAction> Action =
950 std::make_unique<PreprocessOnlyAction>();
951 auto *InputFile = CI.getFrontendOpts().Inputs.begin();
952 bool ActionBeginSucceeded = Action->BeginSourceFile(CI, Input: *InputFile);
953 assert(ActionBeginSucceeded && "Action BeginSourceFile must succeed");
954 (void)ActionBeginSucceeded;
955 }
956
957 Preprocessor &PP = CI.getPreprocessor();
958 SourceManager &SM = PP.getSourceManager();
959 FileID MainFileID = SM.getMainFileID();
960 SourceLocation FileStart = SM.getLocForStartOfFile(FID: MainFileID);
961 SourceLocation IDLocation = FileStart.getLocWithOffset(Offset: SrcLocOffset);
962 PPCallbacks *CB = nullptr;
963 if (!SrcLocOffset) {
964 // We need to call EnterSourceFile when SrcLocOffset is zero to initialize
965 // the preprocessor.
966 bool PPFailed = PP.EnterSourceFile(FID: MainFileID, Dir: nullptr, Loc: SourceLocation());
967 assert(!PPFailed && "Preprocess must be able to enter the main file.");
968 (void)PPFailed;
969 CB = MDC->getPPCallbacks();
970 } else {
971 // When SrcLocOffset is non-zero, the preprocessor has already been
972 // initialized through a previous call of computeDependencies. We want to
973 // preserve the PP's state, hence we do not call EnterSourceFile again.
974 MDC->attachToPreprocessor(PP);
975 CB = MDC->getPPCallbacks();
976
977 FileID PrevFID;
978 SrcMgr::CharacteristicKind FileType =
979 SM.getFileCharacteristic(Loc: IDLocation);
980 CB->LexedFileChanged(FID: MainFileID,
981 Reason: PPChainedCallbacks::LexedFileChangeReason::EnterFile,
982 FileType, PrevFID, Loc: IDLocation);
983 }
984
985 // FIXME: Scan modules asynchronously here as well.
986
987 SrcLocOffset++;
988 SmallVector<IdentifierLoc, 2> Path;
989 IdentifierInfo *ModuleID = PP.getIdentifierInfo(Name: ModuleName);
990 Path.emplace_back(Args&: IDLocation, Args&: ModuleID);
991 auto ModResult = CI.loadModule(ImportLoc: IDLocation, Path, Visibility: Module::Hidden,
992 /*IsInclusionDirective=*/false);
993
994 assert(CB && "Must have PPCallbacks after module loading");
995 CB->moduleImport(ImportLoc: SourceLocation(), Path, Imported: ModResult);
996
997 if (!ModResult)
998 return false;
999
1000 if (CI.getDiagnostics().hasErrorOccurred())
1001 return false;
1002
1003 MDC->run(Consumer);
1004 MDC->applyDiscoveredDependencies(CI&: ModuleInvocation);
1005
1006 bool Success = ModuleInvocation.withCowRef<bool>(
1007 Fn: [&](CowCompilerInvocation &CowModuleInvocation) {
1008 return Controller.finalize(ScanInstance&: CI, NewInvocation&: CowModuleInvocation);
1009 });
1010 if (!Success)
1011 return false;
1012
1013 Consumer.handleBuildCommand(
1014 Cmd: {.Executable: CommandLine[0], .Arguments: ModuleInvocation.getCC1CommandLine()});
1015
1016 return true;
1017 }
1018
1019 llvm::StringMap<std::string>
1020 computeModuleMapsForModules(ArrayRef<StringRef> ModuleNames) {
1021 assert(CIPtr && "CIPtr must be initialized before calling this method");
1022 return ModuleMapLookup(*CIPtr).computeModuleMapsForModules(ModuleNames);
1023 }
1024
1025 std::shared_ptr<ModuleDepCollector>
1026 scanTranslationUnit(DependencyConsumer &Consumer,
1027 DependencyActionController &Controller) {
1028 assert(CIPtr && "CIPtr must be initialized before calling this method");
1029 auto &CI = *CIPtr;
1030
1031 std::optional<AsyncModuleCompiles> AsyncCompiles;
1032 if (Worker.Service.getOpts().AsyncScanModules) {
1033 AsyncCompiles.emplace();
1034 if (!prescanModulesAsync(Compiles&: *AsyncCompiles, Controller))
1035 return nullptr;
1036 }
1037
1038 auto MDC = initializeScanInstanceDependencyCollector(
1039 ScanInstance&: CI, DepOutputOpts: std::make_unique<DependencyOutputOptions>(args&: *OutputOpts),
1040 Service&: Worker.Service, Inv&: *OriginalInvocation, Controller, PrebuiltModulesASTMap: PrebuiltModuleASTMap,
1041 StableDirs);
1042
1043 if (CI.getDiagnostics().hasErrorOccurred())
1044 return nullptr;
1045
1046 if (!Controller.initialize(ScanInstance&: CI, NewInvocation&: *OriginalInvocation))
1047 return nullptr;
1048
1049 ReadPCHAndPreprocessAction Action;
1050 if (!CI.ExecuteAction(Act&: Action))
1051 return nullptr;
1052
1053 MDC->run(Consumer);
1054 if (!applyAndReport(MDC&: *MDC, ModuleInvocation&: *OriginalInvocation, Consumer, Controller,
1055 Executable: CommandLine[0]))
1056 return nullptr;
1057 return MDC;
1058 }
1059
1060 bool applyAndReport(ModuleDepCollector &MDC,
1061 CompilerInvocation &ModuleInvocation,
1062 DependencyConsumer &Consumer,
1063 DependencyActionController &Controller,
1064 StringRef Executable) {
1065 MDC.applyDiscoveredDependencies(CI&: ModuleInvocation);
1066 bool Success = ModuleInvocation.withCowRef<bool>(
1067 Fn: [&](CowCompilerInvocation &CowModuleInvocation) {
1068 return Controller.finalize(ScanInstance&: *CIPtr, NewInvocation&: CowModuleInvocation);
1069 });
1070 if (!Success)
1071 return false;
1072 Consumer.handleBuildCommand(
1073 Cmd: {.Executable: Executable.str(), .Arguments: ModuleInvocation.getCC1CommandLine()});
1074 return true;
1075 }
1076};
1077} // namespace dependencies
1078} // namespace clang
1079
1080DependencyScanningWorker::DependencyScanningWorker(
1081 DependencyScanningService &Service,
1082 IntrusiveRefCntPtr<llvm::vfs::FileSystem> LocalFS)
1083 : Service(Service) {
1084 PCHContainerOps = std::make_shared<PCHContainerOperations>();
1085 // We need to read object files from PCH built outside the scanner.
1086 PCHContainerOps->registerReader(
1087 Reader: std::make_unique<ObjectFilePCHContainerReader>());
1088 // The scanner itself writes only raw ast files.
1089 PCHContainerOps->registerWriter(Writer: std::make_unique<RawPCHContainerWriter>());
1090
1091 auto BaseFS = Service.getOpts().MakeVFS();
1092
1093 if (Service.getOpts().TraceVFS) {
1094 TracingFS = llvm::makeIntrusiveRefCnt<llvm::vfs::TracingFileSystem>(
1095 A: std::move(BaseFS));
1096 BaseFS = TracingFS;
1097 }
1098
1099 DepFS = llvm::makeIntrusiveRefCnt<DependencyScanningWorkerFilesystem>(
1100 A&: Service, A: std::move(BaseFS), A: std::move(LocalFS));
1101}
1102
1103DependencyScanningWorker::~DependencyScanningWorker() = default;
1104
1105IntrusiveRefCntPtr<llvm::vfs::FileSystem>
1106DependencyScanningWorker::makeEffectiveVFS(
1107 StringRef WorkingDirectory,
1108 IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS) const {
1109 IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS = DepFS;
1110 if (OverlayFS) {
1111 auto NewFS =
1112 llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(A: std::move(FS));
1113 NewFS->pushOverlay(FS: std::move(OverlayFS));
1114 FS = std::move(NewFS);
1115 }
1116 FS->setCurrentWorkingDirectory(WorkingDirectory);
1117 return FS;
1118}
1119
1120bool DependencyScanningWorker::computeDependencies(
1121 StringRef WorkingDirectory, ArrayRef<ArrayRef<std::string>> CommandLines,
1122 DependencyConsumer &DepConsumer, DependencyActionController &Controller,
1123 DiagnosticConsumer &DiagConsumer,
1124 IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS) {
1125 auto FS = makeEffectiveVFS(WorkingDirectory, OverlayFS);
1126
1127 bool Scanned = false;
1128 std::shared_ptr<ModuleDepCollector> MDC;
1129 std::optional<CompilerInstanceWithContext> CIWC;
1130
1131 const bool Success = llvm::all_of(Range&: CommandLines, P: [&](const auto &Cmd) {
1132 if (StringRef(Cmd[1]) != "-cc1") {
1133 // Non-clang command. Just pass through to the dependency consumer.
1134 DepConsumer.handleBuildCommand(
1135 Cmd: {Cmd.front(), {Cmd.begin() + 1, Cmd.end()}});
1136 return true;
1137 }
1138
1139 Service.getLogger().log().logArray("starting scanning command:", " ", Cmd);
1140 llvm::scope_exit ExitLogging([&] {
1141 Service.getLogger().log().logArray("finished scanning command:", " ",
1142 Cmd);
1143 });
1144
1145 auto DiagOpts = createScanningDiagOptions(Cmd);
1146 auto DiagEngine =
1147 CompilerInstance::createDiagnostics(*FS, *DiagOpts, &DiagConsumer,
1148 /*ShouldOwnClient=*/false);
1149 if (!Scanned) {
1150 // Scanning runs once for the first -cc1 invocation in a chain of driver
1151 // jobs.
1152 // For any dependent jobs, reuse the scanning result and just update the
1153 // new invocation.
1154 // FIXME: to support multi-arch builds, each arch requires a separate
1155 // scan.
1156 Scanned = true;
1157 auto Result = CompilerInstanceWithContext::initializeFromCC1Commandline(
1158 *this, WorkingDirectory, Cmd, *DiagEngine, OverlayFS, Controller);
1159 if (!Result)
1160 return false;
1161 CIWC.emplace(std::move(*Result));
1162 MDC = CIWC->scanTranslationUnit(Consumer&: DepConsumer, Controller);
1163 return MDC != nullptr;
1164 }
1165
1166 auto Invocation = createCompilerInvocation(Cmd, *DiagEngine);
1167 if (!Invocation)
1168 return false;
1169
1170 // The first cc1 is canonicalized in initializeScanInstance; each sibling
1171 // invocation must likewise be canonicalized before its cc1 command line is
1172 // emitted. This is mostly relevant for multi-arch jobs where we currently
1173 // do not do re-scans.
1174 if (any(Val: Service.getOpts().OptimizeArgs & ScanningOptimizations::Macros))
1175 canonicalizeDefines(Invocation->getPreprocessorOpts());
1176
1177 assert(CIWC && "Must have an initialized CIWC");
1178 return CIWC->applyAndReport(MDC&: *MDC, ModuleInvocation&: *Invocation, Consumer&: DepConsumer, Controller,
1179 Executable: Cmd.front());
1180 });
1181
1182 return Success && Scanned;
1183}
1184
1185bool DependencyScanningWorker::computeDependenciesByName(
1186 StringRef CWD, ArrayRef<std::string> CC1CommandLine,
1187 IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS,
1188 DiagnosticConsumer &DiagConsumer, DependencyActionController &Controller,
1189 llvm::function_ref<std::optional<std::string>()> getNextName,
1190 DependencyConsumer &DepConsumer) {
1191 std::optional<CompilerInstanceWithContext> CIWC =
1192 CompilerInstanceWithContext::initializeFromCC1Commandline(
1193 Worker&: *this, CWD, CC1CommandLine, DiagConsumer, OverlayFS: std::move(OverlayFS),
1194 Controller);
1195 if (!CIWC)
1196 return false;
1197
1198 bool AllScansSucceeded = true;
1199 while (std::optional<std::string> NextName = getNextName()) {
1200 bool Success =
1201 CIWC->computeDependencies(ModuleName: *NextName, Consumer&: DepConsumer, Controller);
1202 DepConsumer.finishQuery(ModuleName: *NextName, Success);
1203 AllScansSucceeded = AllScansSucceeded && Success;
1204 }
1205 return AllScansSucceeded;
1206}
1207
1208std::optional<llvm::StringMap<std::string>>
1209DependencyScanningWorker::computeModuleMapsForModules(
1210 StringRef CWD, ArrayRef<std::string> CC1CommandLine,
1211 IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS,
1212 DiagnosticConsumer &DiagConsumer, DependencyActionController &Controller,
1213 ArrayRef<StringRef> ModuleNames) {
1214 std::optional<CompilerInstanceWithContext> CIWC =
1215 CompilerInstanceWithContext::initializeFromCC1Commandline(
1216 Worker&: *this, CWD, CC1CommandLine, DiagConsumer, OverlayFS: std::move(OverlayFS),
1217 Controller);
1218 if (!CIWC)
1219 return std::nullopt;
1220 return CIWC->computeModuleMapsForModules(ModuleNames);
1221}
1222