1//===- DependencyScannerImpl.cpp - Implements module dependency scanning --===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "clang/DependencyScanning/DependencyScannerImpl.h"
10#include "clang/Basic/DiagnosticFrontend.h"
11#include "clang/Basic/DiagnosticSerialization.h"
12#include "clang/DependencyScanning/DependencyScanningFilesystem.h"
13#include "clang/DependencyScanning/DependencyScanningService.h"
14#include "clang/DependencyScanning/DependencyScanningWorker.h"
15#include "clang/Driver/Driver.h"
16#include "clang/Frontend/FrontendActions.h"
17#include "llvm/ADT/IntrusiveRefCntPtr.h"
18#include "llvm/ADT/ScopeExit.h"
19#include "llvm/Support/AdvisoryLock.h"
20#include "llvm/Support/CrashRecoveryContext.h"
21#include "llvm/Support/VirtualFileSystem.h"
22#include "llvm/TargetParser/Host.h"
23
24#include <mutex>
25#include <thread>
26
27using namespace clang;
28using namespace dependencies;
29
30namespace {
31/// Forwards the gatherered dependencies to the consumer.
32class DependencyConsumerForwarder : public DependencyFileGenerator {
33public:
34 DependencyConsumerForwarder(std::unique_ptr<DependencyOutputOptions> Opts,
35 StringRef WorkingDirectory, DependencyConsumer &C)
36 : DependencyFileGenerator(*Opts), WorkingDirectory(WorkingDirectory),
37 Opts(std::move(Opts)), C(C) {}
38
39 void finishedMainFile(DiagnosticsEngine &Diags) override {
40 C.handleDependencyOutputOpts(Opts: *Opts);
41 llvm::SmallString<256> CanonPath;
42 for (const auto &File : getDependencies()) {
43 CanonPath = File;
44 llvm::sys::path::remove_dots(path&: CanonPath, /*remove_dot_dot=*/true);
45 llvm::sys::path::make_absolute(current_directory: WorkingDirectory, path&: CanonPath);
46 C.handleFileDependency(Filename: CanonPath);
47 }
48 }
49
50private:
51 StringRef WorkingDirectory;
52 std::unique_ptr<DependencyOutputOptions> Opts;
53 DependencyConsumer &C;
54};
55
56static bool checkHeaderSearchPaths(const HeaderSearchOptions &HSOpts,
57 const HeaderSearchOptions &ExistingHSOpts,
58 DiagnosticsEngine *Diags,
59 const LangOptions &LangOpts) {
60 if (LangOpts.Modules) {
61 if (HSOpts.VFSOverlayFiles != ExistingHSOpts.VFSOverlayFiles) {
62 if (Diags) {
63 Diags->Report(DiagID: diag::warn_pch_vfsoverlay_mismatch);
64 auto VFSNote = [&](int Type, ArrayRef<std::string> VFSOverlays) {
65 if (VFSOverlays.empty()) {
66 Diags->Report(DiagID: diag::note_pch_vfsoverlay_empty) << Type;
67 } else {
68 std::string Files = llvm::join(R&: VFSOverlays, Separator: "\n");
69 Diags->Report(DiagID: diag::note_pch_vfsoverlay_files) << Type << Files;
70 }
71 };
72 VFSNote(0, HSOpts.VFSOverlayFiles);
73 VFSNote(1, ExistingHSOpts.VFSOverlayFiles);
74 }
75 }
76 }
77 return false;
78}
79
80using PrebuiltModuleFilesT = decltype(HeaderSearchOptions::PrebuiltModuleFiles);
81
82/// A listener that collects the imported modules and the input
83/// files. While visiting, collect vfsoverlays and file inputs that determine
84/// whether prebuilt modules fully resolve in stable directories.
85class PrebuiltModuleListener : public ASTReaderListener {
86public:
87 PrebuiltModuleListener(PrebuiltModuleFilesT &PrebuiltModuleFiles,
88 llvm::SmallVector<std::string> &NewModuleFiles,
89 PrebuiltModulesAttrsMap &PrebuiltModulesASTMap,
90 const HeaderSearchOptions &HSOpts,
91 const LangOptions &LangOpts, DiagnosticsEngine &Diags,
92 const ArrayRef<StringRef> StableDirs)
93 : PrebuiltModuleFiles(PrebuiltModuleFiles),
94 NewModuleFiles(NewModuleFiles),
95 PrebuiltModulesASTMap(PrebuiltModulesASTMap), ExistingHSOpts(HSOpts),
96 ExistingLangOpts(LangOpts), Diags(Diags), StableDirs(StableDirs) {}
97
98 bool needsImportVisitation() const override { return true; }
99 bool needsInputFileVisitation() override { return true; }
100 bool needsSystemInputFileVisitation() override { return true; }
101
102 /// Accumulate the modules are transitively depended on by the initial
103 /// prebuilt module.
104 void visitImport(StringRef ModuleName, StringRef Filename) override {
105 if (PrebuiltModuleFiles.insert(x: {ModuleName.str(), Filename.str()}).second)
106 NewModuleFiles.push_back(Elt: Filename.str());
107
108 auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(Key: Filename);
109 PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second;
110 if (PrebuiltMapEntry.second)
111 PrebuiltModule.setInStableDir(!StableDirs.empty());
112
113 if (auto It = PrebuiltModulesASTMap.find(Key: CurrentFile);
114 It != PrebuiltModulesASTMap.end() && CurrentFile != Filename)
115 PrebuiltModule.addDependent(ModuleFile: It->getKey());
116 }
117
118 /// For each input file discovered, check whether it's external path is in a
119 /// stable directory. Traversal is stopped if the current module is not
120 /// considered stable.
121 bool visitInputFileAsRequested(StringRef FilenameAsRequested,
122 StringRef Filename, bool isSystem,
123 bool isOverridden, time_t StoredTime,
124 bool isExplicitModule) override {
125 if (StableDirs.empty())
126 return false;
127 auto PrebuiltEntryIt = PrebuiltModulesASTMap.find(Key: CurrentFile);
128 if ((PrebuiltEntryIt == PrebuiltModulesASTMap.end()) ||
129 (!PrebuiltEntryIt->second.isInStableDir()))
130 return false;
131
132 PrebuiltEntryIt->second.setInStableDir(
133 isPathInStableDir(Directories: StableDirs, Input: Filename));
134 return PrebuiltEntryIt->second.isInStableDir();
135 }
136
137 /// Update which module that is being actively traversed.
138 void visitModuleFile(StringRef Filename,
139 serialization::ModuleKind Kind) override {
140 // If the CurrentFile is not
141 // considered stable, update any of it's transitive dependents.
142 auto PrebuiltEntryIt = PrebuiltModulesASTMap.find(Key: CurrentFile);
143 if ((PrebuiltEntryIt != PrebuiltModulesASTMap.end()) &&
144 !PrebuiltEntryIt->second.isInStableDir())
145 PrebuiltEntryIt->second.updateDependentsNotInStableDirs(
146 PrebuiltModulesMap&: PrebuiltModulesASTMap);
147 CurrentFile = Filename;
148 }
149
150 /// Check the header search options for a given module when considering
151 /// if the module comes from stable directories.
152 bool ReadHeaderSearchOptions(const HeaderSearchOptions &HSOpts,
153 StringRef ModuleFilename,
154 StringRef SpecificModuleCachePath,
155 bool Complain) override {
156
157 auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(Key: CurrentFile);
158 PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second;
159 if (PrebuiltMapEntry.second)
160 PrebuiltModule.setInStableDir(!StableDirs.empty());
161
162 if (PrebuiltModule.isInStableDir())
163 PrebuiltModule.setInStableDir(areOptionsInStableDir(Directories: StableDirs, HSOpts));
164
165 return false;
166 }
167
168 /// Accumulate vfsoverlays used to build these prebuilt modules.
169 bool ReadHeaderSearchPaths(const HeaderSearchOptions &HSOpts,
170 bool Complain) override {
171
172 auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(Key: CurrentFile);
173 PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second;
174 if (PrebuiltMapEntry.second)
175 PrebuiltModule.setInStableDir(!StableDirs.empty());
176
177 PrebuiltModule.setVFS(
178 llvm::StringSet<>(llvm::from_range, HSOpts.VFSOverlayFiles));
179
180 return checkHeaderSearchPaths(
181 HSOpts, ExistingHSOpts, Diags: Complain ? &Diags : nullptr, LangOpts: ExistingLangOpts);
182 }
183
184private:
185 PrebuiltModuleFilesT &PrebuiltModuleFiles;
186 llvm::SmallVector<std::string> &NewModuleFiles;
187 PrebuiltModulesAttrsMap &PrebuiltModulesASTMap;
188 const HeaderSearchOptions &ExistingHSOpts;
189 const LangOptions &ExistingLangOpts;
190 DiagnosticsEngine &Diags;
191 std::string CurrentFile;
192 const ArrayRef<StringRef> StableDirs;
193};
194
195/// Visit the given prebuilt module and collect all of the modules it
196/// transitively imports and contributing input files.
197static bool visitPrebuiltModule(StringRef PrebuiltModuleFilename,
198 CompilerInstance &CI,
199 PrebuiltModuleFilesT &ModuleFiles,
200 PrebuiltModulesAttrsMap &PrebuiltModulesASTMap,
201 DiagnosticsEngine &Diags,
202 const ArrayRef<StringRef> StableDirs) {
203 // List of module files to be processed.
204 llvm::SmallVector<std::string> Worklist;
205
206 PrebuiltModuleListener Listener(ModuleFiles, Worklist, PrebuiltModulesASTMap,
207 CI.getHeaderSearchOpts(), CI.getLangOpts(),
208 Diags, StableDirs);
209
210 Listener.visitModuleFile(Filename: PrebuiltModuleFilename,
211 Kind: serialization::MK_ExplicitModule);
212 if (ASTReader::readASTFileControlBlock(
213 Filename: PrebuiltModuleFilename, FileMgr&: CI.getFileManager(), ModCache: CI.getModuleCache(),
214 PCHContainerRdr: CI.getPCHContainerReader(),
215 /*FindModuleFileExtensions=*/false, Listener,
216 /*ValidateDiagnosticOptions=*/false, ClientLoadCapabilities: ASTReader::ARR_OutOfDate))
217 return true;
218
219 while (!Worklist.empty()) {
220 Listener.visitModuleFile(Filename: Worklist.back(), Kind: serialization::MK_ExplicitModule);
221 if (ASTReader::readASTFileControlBlock(
222 Filename: Worklist.pop_back_val(), FileMgr&: CI.getFileManager(), ModCache: CI.getModuleCache(),
223 PCHContainerRdr: CI.getPCHContainerReader(),
224 /*FindModuleFileExtensions=*/false, Listener,
225 /*ValidateDiagnosticOptions=*/false))
226 return true;
227 }
228 return false;
229}
230
231/// Transform arbitrary file name into an object-like file name.
232static std::string makeObjFileName(StringRef FileName) {
233 SmallString<128> ObjFileName(FileName);
234 llvm::sys::path::replace_extension(path&: ObjFileName, extension: "o");
235 return std::string(ObjFileName);
236}
237
238/// Deduce the dependency target based on the output file and input files.
239static std::string
240deduceDepTarget(const std::string &OutputFile,
241 const SmallVectorImpl<FrontendInputFile> &InputFiles) {
242 if (OutputFile != "-")
243 return OutputFile;
244
245 if (InputFiles.empty() || !InputFiles.front().isFile())
246 return "clang-scan-deps\\ dependency";
247
248 return makeObjFileName(FileName: InputFiles.front().getFile());
249}
250
251// Clang implements -D and -U by splatting text into a predefines buffer. This
252// allows constructs such as `-DFඞ=3 "-D F\u{0D9E} 4 3 2”` to be accepted and
253// define the same macro, or adding C++ style comments before the macro name.
254//
255// This function checks that the first non-space characters in the macro
256// obviously form an identifier that can be uniqued on without lexing. Failing
257// to do this could lead to changing the final definition of a macro.
258//
259// We could set up a preprocessor and actually lex the name, but that's very
260// heavyweight for a situation that will almost never happen in practice.
261static std::optional<StringRef> getSimpleMacroName(StringRef Macro) {
262 StringRef Name = Macro.split(Separator: "=").first.ltrim(Chars: " \t");
263 std::size_t I = 0;
264
265 auto FinishName = [&]() -> std::optional<StringRef> {
266 StringRef SimpleName = Name.slice(Start: 0, End: I);
267 if (SimpleName.empty())
268 return std::nullopt;
269 return SimpleName;
270 };
271
272 for (; I != Name.size(); ++I) {
273 switch (Name[I]) {
274 case '(': // Start of macro parameter list
275 case ' ': // End of macro name
276 case '\t':
277 return FinishName();
278 case '_':
279 continue;
280 default:
281 if (llvm::isAlnum(C: Name[I]))
282 continue;
283 return std::nullopt;
284 }
285 }
286 return FinishName();
287}
288
289static void canonicalizeDefines(PreprocessorOptions &PPOpts) {
290 using MacroOpt = std::pair<StringRef, std::size_t>;
291 std::vector<MacroOpt> SimpleNames;
292 SimpleNames.reserve(n: PPOpts.Macros.size());
293 std::size_t Index = 0;
294 for (const auto &M : PPOpts.Macros) {
295 auto SName = getSimpleMacroName(Macro: M.first);
296 // Skip optimizing if we can't guarantee we can preserve relative order.
297 if (!SName)
298 return;
299 SimpleNames.emplace_back(args&: *SName, args&: Index);
300 ++Index;
301 }
302
303 llvm::stable_sort(Range&: SimpleNames, C: llvm::less_first());
304 // Keep the last instance of each macro name by going in reverse
305 auto NewEnd = std::unique(
306 first: SimpleNames.rbegin(), last: SimpleNames.rend(),
307 binary_pred: [](const MacroOpt &A, const MacroOpt &B) { return A.first == B.first; });
308 SimpleNames.erase(first: SimpleNames.begin(), last: NewEnd.base());
309
310 // Apply permutation.
311 decltype(PPOpts.Macros) NewMacros;
312 NewMacros.reserve(n: SimpleNames.size());
313 for (std::size_t I = 0, E = SimpleNames.size(); I != E; ++I) {
314 std::size_t OriginalIndex = SimpleNames[I].second;
315 // We still emit undefines here as they may be undefining a predefined macro
316 NewMacros.push_back(x: std::move(PPOpts.Macros[OriginalIndex]));
317 }
318 std::swap(x&: PPOpts.Macros, y&: NewMacros);
319}
320
321class ScanningDependencyDirectivesGetter : public DependencyDirectivesGetter {
322 DependencyScanningWorkerFilesystem *DepFS;
323
324public:
325 ScanningDependencyDirectivesGetter(FileManager &FileMgr) : DepFS(nullptr) {
326 FileMgr.getVirtualFileSystem().visit(Callback: [&](llvm::vfs::FileSystem &FS) {
327 auto *DFS = llvm::dyn_cast<DependencyScanningWorkerFilesystem>(Val: &FS);
328 if (DFS) {
329 assert(!DepFS && "Found multiple scanning VFSs");
330 DepFS = DFS;
331 }
332 });
333 assert(DepFS && "Did not find scanning VFS");
334 }
335
336 std::unique_ptr<DependencyDirectivesGetter>
337 cloneFor(FileManager &FileMgr) override {
338 return std::make_unique<ScanningDependencyDirectivesGetter>(args&: FileMgr);
339 }
340
341 std::optional<ArrayRef<dependency_directives_scan::Directive>>
342 operator()(FileEntryRef File) override {
343 return DepFS->getDirectiveTokens(Path: File.getName());
344 }
345};
346
347/// Sanitize diagnostic options for dependency scan.
348void sanitizeDiagOpts(DiagnosticOptions &DiagOpts) {
349 // Don't print 'X warnings and Y errors generated'.
350 DiagOpts.ShowCarets = false;
351 // Don't write out diagnostic file.
352 DiagOpts.DiagnosticSerializationFile.clear();
353 // Don't emit warnings except for scanning specific warnings.
354 // TODO: It would be useful to add a more principled way to ignore all
355 // warnings that come from source code. The issue is that we need to
356 // ignore warnings that could be surpressed by
357 // `#pragma clang diagnostic`, while still allowing some scanning
358 // warnings for things we're not ready to turn into errors yet.
359 // See `test/ClangScanDeps/diagnostic-pragmas.c` for an example.
360 llvm::erase_if(C&: DiagOpts.Warnings, P: [](StringRef Warning) {
361 return llvm::StringSwitch<bool>(Warning)
362 .Cases(CaseStrings: {"pch-vfs-diff", "error=pch-vfs-diff"}, Value: false)
363 .StartsWith(S: "no-error=", Value: false)
364 .Default(Value: true);
365 });
366}
367} // namespace
368
369std::unique_ptr<DiagnosticOptions>
370dependencies::createDiagOptions(ArrayRef<std::string> CommandLine) {
371 std::vector<const char *> CLI;
372 for (const std::string &Arg : CommandLine)
373 CLI.push_back(x: Arg.c_str());
374 auto DiagOpts = CreateAndPopulateDiagOpts(Argv: CLI);
375 sanitizeDiagOpts(DiagOpts&: *DiagOpts);
376 return DiagOpts;
377}
378
379DiagnosticsEngineWithDiagOpts::DiagnosticsEngineWithDiagOpts(
380 ArrayRef<std::string> CommandLine,
381 IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS, DiagnosticConsumer &DC) {
382 std::vector<const char *> CCommandLine(CommandLine.size(), nullptr);
383 llvm::transform(Range&: CommandLine, d_first: CCommandLine.begin(),
384 F: [](const std::string &Str) { return Str.c_str(); });
385 DiagOpts = CreateAndPopulateDiagOpts(Argv: CCommandLine);
386 sanitizeDiagOpts(DiagOpts&: *DiagOpts);
387 DiagEngine = CompilerInstance::createDiagnostics(VFS&: *FS, Opts&: *DiagOpts, Client: &DC,
388 /*ShouldOwnClient=*/false);
389}
390
391std::unique_ptr<CompilerInvocation>
392dependencies::createCompilerInvocation(ArrayRef<std::string> CommandLine,
393 DiagnosticsEngine &Diags) {
394 llvm::opt::ArgStringList Argv;
395 for (const std::string &Str : ArrayRef(CommandLine).drop_front())
396 Argv.push_back(Elt: Str.c_str());
397
398 auto Invocation = std::make_unique<CompilerInvocation>();
399 if (!CompilerInvocation::CreateFromArgs(Res&: *Invocation, CommandLineArgs: Argv, Diags)) {
400 // FIXME: Should we just go on like cc1_main does?
401 return nullptr;
402 }
403 return Invocation;
404}
405
406void dependencies::initializeScanCompilerInstance(
407 CompilerInstance &ScanInstance,
408 IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
409 DiagnosticConsumer *DiagConsumer, DependencyScanningService &Service,
410 IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS) {
411 ScanInstance.setBuildingModule(false);
412 ScanInstance.createVirtualFileSystem(BaseFS: FS, DC: DiagConsumer);
413 ScanInstance.createDiagnostics(Client: DiagConsumer, /*ShouldOwnClient=*/false);
414 ScanInstance.createFileManager();
415 ScanInstance.createSourceManager();
416
417 // Use DepFS for getting the dependency directives if requested to do so.
418 if (Service.getOpts().Mode == ScanningMode::DependencyDirectivesScan) {
419 DepFS->resetBypassedPathPrefix();
420 SmallString<256> ModulesCachePath;
421 normalizeModuleCachePath(FileMgr&: ScanInstance.getFileManager(),
422 Path: ScanInstance.getHeaderSearchOpts().ModuleCachePath,
423 NormalizedPath&: ModulesCachePath);
424 if (!ModulesCachePath.empty())
425 DepFS->setBypassedPathPrefix(ModulesCachePath);
426
427 ScanInstance.setDependencyDirectivesGetter(
428 std::make_unique<ScanningDependencyDirectivesGetter>(
429 args&: ScanInstance.getFileManager()));
430 }
431}
432
433/// Creates a CompilerInvocation suitable for the dependency scanner.
434static std::shared_ptr<CompilerInvocation>
435createScanCompilerInvocation(const CompilerInvocation &Invocation,
436 const DependencyScanningService &Service) {
437 auto ScanInvocation = std::make_shared<CompilerInvocation>(args: Invocation);
438
439 sanitizeDiagOpts(DiagOpts&: ScanInvocation->getDiagnosticOpts());
440
441 ScanInvocation->getPreprocessorOpts().AllowPCHWithDifferentModulesCachePath =
442 true;
443
444 if (ScanInvocation->getHeaderSearchOpts().ModulesValidateOncePerBuildSession)
445 ScanInvocation->getHeaderSearchOpts().BuildSessionTimestamp =
446 Service.getOpts().BuildSessionTimestamp;
447
448 ScanInvocation->getFrontendOpts().DisableFree = false;
449 ScanInvocation->getFrontendOpts().GenerateGlobalModuleIndex = false;
450 ScanInvocation->getFrontendOpts().UseGlobalModuleIndex = false;
451 ScanInvocation->getFrontendOpts().GenReducedBMI = false;
452 ScanInvocation->getFrontendOpts().ModuleOutputPath.clear();
453 // This will prevent us compiling individual modules asynchronously since
454 // FileManager is not thread-safe, but it does improve performance for now.
455 ScanInvocation->getFrontendOpts().ModulesShareFileManager = true;
456 ScanInvocation->getHeaderSearchOpts().ModuleFormat = "raw";
457 ScanInvocation->getHeaderSearchOpts().ModulesIncludeVFSUsage =
458 any(Val: Service.getOpts().OptimizeArgs & ScanningOptimizations::VFS);
459
460 // Consider different header search and diagnostic options to create
461 // different modules. This avoids the unsound aliasing of module PCMs.
462 //
463 // TODO: Implement diagnostic bucketing to reduce the impact of strict
464 // context hashing.
465 ScanInvocation->getHeaderSearchOpts().ModulesStrictContextHash = true;
466 ScanInvocation->getHeaderSearchOpts().ModulesSerializeOnlyPreprocessor = true;
467 ScanInvocation->getHeaderSearchOpts().ModulesSkipDiagnosticOptions = true;
468 ScanInvocation->getHeaderSearchOpts().ModulesSkipHeaderSearchPaths = true;
469 ScanInvocation->getHeaderSearchOpts().ModulesSkipPragmaDiagnosticMappings =
470 true;
471 ScanInvocation->getHeaderSearchOpts().ModulesForceValidateUserHeaders = false;
472
473 // Avoid some checks and module map parsing when loading PCM files.
474 ScanInvocation->getPreprocessorOpts().ModulesCheckRelocated = false;
475
476 // Ensure that the scanner does not create new dependency collectors,
477 // and thus won't write out the extra '.d' files to disk.
478 ScanInvocation->getDependencyOutputOpts() = {};
479
480 return ScanInvocation;
481}
482
483llvm::SmallVector<StringRef>
484dependencies::getInitialStableDirs(const CompilerInstance &ScanInstance) {
485 // Create a collection of stable directories derived from the ScanInstance
486 // for determining whether module dependencies would fully resolve from
487 // those directories.
488 llvm::SmallVector<StringRef> StableDirs;
489 const StringRef Sysroot = ScanInstance.getHeaderSearchOpts().Sysroot;
490 if (!Sysroot.empty() && (llvm::sys::path::root_directory(path: Sysroot) != Sysroot))
491 StableDirs = {Sysroot, ScanInstance.getHeaderSearchOpts().ResourceDir};
492 return StableDirs;
493}
494
495std::optional<PrebuiltModulesAttrsMap>
496dependencies::computePrebuiltModulesASTMap(
497 CompilerInstance &ScanInstance, llvm::SmallVector<StringRef> &StableDirs) {
498 // Store a mapping of prebuilt module files and their properties like header
499 // search options. This will prevent the implicit build to create duplicate
500 // modules and will force reuse of the existing prebuilt module files
501 // instead.
502 PrebuiltModulesAttrsMap PrebuiltModulesASTMap;
503
504 if (!ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty())
505 if (visitPrebuiltModule(
506 PrebuiltModuleFilename: ScanInstance.getPreprocessorOpts().ImplicitPCHInclude, CI&: ScanInstance,
507 ModuleFiles&: ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles,
508 PrebuiltModulesASTMap, Diags&: ScanInstance.getDiagnostics(), StableDirs))
509 return {};
510
511 return PrebuiltModulesASTMap;
512}
513
514/// Creates dependency output options to be reported to the dependency consumer,
515/// deducing missing information if necessary.
516static std::unique_ptr<DependencyOutputOptions>
517createDependencyOutputOptions(const CompilerInvocation &Invocation) {
518 auto Opts = std::make_unique<DependencyOutputOptions>(
519 args: Invocation.getDependencyOutputOpts());
520 // We need at least one -MT equivalent for the generator of make dependency
521 // files to work.
522 if (Opts->Targets.empty())
523 Opts->Targets = {deduceDepTarget(OutputFile: Invocation.getFrontendOpts().OutputFile,
524 InputFiles: Invocation.getFrontendOpts().Inputs)};
525 Opts->IncludeSystemHeaders = true;
526
527 return Opts;
528}
529
530std::shared_ptr<ModuleDepCollector>
531dependencies::initializeScanInstanceDependencyCollector(
532 CompilerInstance &ScanInstance,
533 std::unique_ptr<DependencyOutputOptions> DepOutputOpts,
534 StringRef WorkingDirectory, DependencyConsumer &Consumer,
535 DependencyScanningService &Service, CompilerInvocation &Inv,
536 DependencyActionController &Controller,
537 PrebuiltModulesAttrsMap PrebuiltModulesASTMap,
538 llvm::SmallVector<StringRef> &StableDirs) {
539 std::shared_ptr<ModuleDepCollector> MDC;
540 switch (Service.getOpts().Format) {
541 case ScanningOutputFormat::Make:
542 ScanInstance.addDependencyCollector(
543 Listener: std::make_shared<DependencyConsumerForwarder>(
544 args: std::move(DepOutputOpts), args&: WorkingDirectory, args&: Consumer));
545 break;
546 case ScanningOutputFormat::P1689:
547 case ScanningOutputFormat::Full:
548 MDC = std::make_shared<ModuleDepCollector>(
549 args&: Service, args: std::move(DepOutputOpts), args&: ScanInstance, args&: Consumer, args&: Controller,
550 args&: Inv, args: std::move(PrebuiltModulesASTMap), args&: StableDirs);
551 ScanInstance.addDependencyCollector(Listener: MDC);
552 break;
553 }
554
555 return MDC;
556}
557
558/// Manages (and terminates) the asynchronous compilation of modules.
559class AsyncModuleCompiles {
560 std::mutex Mutex;
561 bool Stop = false;
562 // FIXME: Have the service own a thread pool and use that instead.
563 std::vector<std::thread> Compiles;
564
565public:
566 /// Registers the module compilation, unless this instance is about to be
567 /// destroyed.
568 void add(llvm::unique_function<void()> Compile) {
569 std::lock_guard<std::mutex> Lock(Mutex);
570 if (!Stop)
571 Compiles.emplace_back(args: std::move(Compile));
572 }
573
574 ~AsyncModuleCompiles() {
575 {
576 // Prevent registration of further module compiles.
577 std::lock_guard<std::mutex> Lock(Mutex);
578 Stop = true;
579 }
580
581 // Wait for outstanding module compiles to finish.
582 for (std::thread &Compile : Compiles)
583 Compile.join();
584 }
585};
586
587struct SingleModuleWithAsyncModuleCompiles : PreprocessOnlyAction {
588 DependencyScanningService &Service;
589 AsyncModuleCompiles &Compiles;
590
591 SingleModuleWithAsyncModuleCompiles(DependencyScanningService &Service,
592 AsyncModuleCompiles &Compiles)
593 : Service(Service), Compiles(Compiles) {}
594
595 bool BeginSourceFileAction(CompilerInstance &CI) override;
596};
597
598/// The preprocessor callback that takes care of initiating an asynchronous
599/// module compilation if needed.
600struct AsyncModuleCompile : PPCallbacks {
601 CompilerInstance &CI;
602 DependencyScanningService &Service;
603 AsyncModuleCompiles &Compiles;
604
605 AsyncModuleCompile(CompilerInstance &CI, DependencyScanningService &Service,
606 AsyncModuleCompiles &Compiles)
607 : CI(CI), Service(Service), Compiles(Compiles) {}
608
609 void moduleLoadSkipped(Module *M) override {
610 M = M->getTopLevelModule();
611
612 HeaderSearch &HS = CI.getPreprocessor().getHeaderSearchInfo();
613 ModuleCache &ModCache = CI.getModuleCache();
614 std::string ModuleFileName = HS.getCachedModuleFileName(Module: M);
615
616 uint64_t Timestamp = ModCache.getModuleTimestamp(ModuleFilename: ModuleFileName);
617 // Someone else already built/validated the PCM.
618 if (Timestamp > CI.getHeaderSearchOpts().BuildSessionTimestamp)
619 return;
620
621 if (!CI.getASTReader())
622 CI.createASTReader();
623 SmallVector<ASTReader::ImportedModule, 0> Imported;
624 // Only calling ReadASTCore() to avoid the expensive eager deserialization
625 // of the clang::Module objects in ReadAST().
626 // FIXME: Consider doing this in the new thread depending on how expensive
627 // the read turns out to be.
628 switch (CI.getASTReader()->ReadASTCore(
629 FileName: ModuleFileName, Type: serialization::MK_ImplicitModule, ImportLoc: SourceLocation(),
630 ImportedBy: nullptr, Loaded&: Imported, ExpectedSize: {}, ExpectedModTime: {}, ExpectedSignature: {},
631 ClientLoadCapabilities: ASTReader::ARR_OutOfDate | ASTReader::ARR_Missing |
632 ASTReader::ARR_TreatModuleWithErrorsAsOutOfDate)) {
633 case ASTReader::Success:
634 // We successfully read a valid, up-to-date PCM.
635 // FIXME: This could update the timestamp. Regular calls to
636 // ASTReader::ReadAST() would do so unless they encountered corrupted
637 // AST block, corrupted extension block, or did not read the expected
638 // top-level module.
639 return;
640 case ASTReader::OutOfDate:
641 case ASTReader::Missing:
642 // The most interesting case.
643 break;
644 default:
645 // Let the regular scan diagnose this.
646 return;
647 }
648
649 ModCache.prepareForGetLock(ModuleFilename: ModuleFileName);
650 auto Lock = ModCache.getLock(ModuleFilename: ModuleFileName);
651 bool Owned;
652 llvm::Error LockErr = Lock->tryLock().moveInto(Value&: Owned);
653 // Someone else is building the PCM right now.
654 if (!LockErr && !Owned)
655 return;
656 // We should build the PCM.
657 IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS =
658 llvm::makeIntrusiveRefCnt<DependencyScanningWorkerFilesystem>(
659 A&: Service.getSharedCache(), A: Service.getOpts().MakeVFS());
660 VFS = createVFSFromCompilerInvocation(CI: CI.getInvocation(),
661 Diags&: CI.getDiagnostics(), BaseFS: std::move(VFS));
662 auto DC = std::make_unique<DiagnosticConsumer>();
663 auto MC = makeInProcessModuleCache(Entries&: Service.getModuleCacheEntries());
664 CompilerInstance::ThreadSafeCloneConfig CloneConfig(std::move(VFS), *DC,
665 std::move(MC));
666 auto ModCI1 = CI.cloneForModuleCompile(ImportLoc: SourceLocation(), Module: M, ModuleFileName,
667 ThreadSafeConfig: CloneConfig);
668 auto ModCI2 = CI.cloneForModuleCompile(ImportLoc: SourceLocation(), Module: M, ModuleFileName,
669 ThreadSafeConfig: CloneConfig);
670
671 // Note: This lock belongs to a module cache that might not outlive the
672 // thread. This works, because the in-process lock only refers to an object
673 // managed by the service, which does outlive the thread.
674 Compiles.add(Compile: [Lock = std::move(Lock), ModCI1 = std::move(ModCI1),
675 ModCI2 = std::move(ModCI2), DC = std::move(DC),
676 Service = &Service, Compiles = &Compiles] {
677 llvm::CrashRecoveryContext CRC;
678 (void)CRC.RunSafely(Fn: [&] {
679 // Quickly discovers and compiles modules for the real scan below.
680 SingleModuleWithAsyncModuleCompiles Action1(*Service, *Compiles);
681 (void)ModCI1->ExecuteAction(Act&: Action1);
682 // The real scan below.
683 ModCI2->getPreprocessorOpts().SingleModuleParseMode = false;
684 GenerateModuleFromModuleMapAction Action2;
685 (void)ModCI2->ExecuteAction(Act&: Action2);
686 });
687 });
688 }
689};
690
691/// Runs the preprocessor on a TU with single-module-parse-mode and compiles
692/// modules asynchronously without blocking or importing them.
693struct SingleTUWithAsyncModuleCompiles : PreprocessOnlyAction {
694 DependencyScanningService &Service;
695 AsyncModuleCompiles &Compiles;
696
697 SingleTUWithAsyncModuleCompiles(DependencyScanningService &Service,
698 AsyncModuleCompiles &Compiles)
699 : Service(Service), Compiles(Compiles) {}
700
701 bool BeginSourceFileAction(CompilerInstance &CI) override {
702 CI.getInvocation().getPreprocessorOpts().SingleModuleParseMode = true;
703 CI.getPreprocessor().addPPCallbacks(
704 C: std::make_unique<AsyncModuleCompile>(args&: CI, args&: Service, args&: Compiles));
705 return true;
706 }
707};
708
709bool SingleModuleWithAsyncModuleCompiles::BeginSourceFileAction(
710 CompilerInstance &CI) {
711 CI.getInvocation().getPreprocessorOpts().SingleModuleParseMode = true;
712 CI.getPreprocessor().addPPCallbacks(
713 C: std::make_unique<AsyncModuleCompile>(args&: CI, args&: Service, args&: Compiles));
714 return true;
715}
716
717bool DependencyScanningAction::runInvocation(
718 std::string Executable,
719 std::unique_ptr<CompilerInvocation> OriginalInvocation,
720 IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
721 std::shared_ptr<PCHContainerOperations> PCHContainerOps,
722 DiagnosticConsumer *DiagConsumer) {
723 // Making sure that we canonicalize the defines early to avoid unnecessary
724 // variants in both the scanner and in the resulting explicit command lines.
725 if (any(Val: Service.getOpts().OptimizeArgs & ScanningOptimizations::Macros))
726 canonicalizeDefines(PPOpts&: OriginalInvocation->getPreprocessorOpts());
727
728 if (Scanned) {
729 // Scanning runs once for the first -cc1 invocation in a chain of driver
730 // jobs. For any dependent jobs, reuse the scanning result and just
731 // update the new invocation.
732 // FIXME: to support multi-arch builds, each arch requires a separate scan
733 if (MDC)
734 MDC->applyDiscoveredDependencies(CI&: *OriginalInvocation);
735 Consumer.handleBuildCommand(
736 Cmd: {.Executable: Executable, .Arguments: OriginalInvocation->getCC1CommandLine()});
737 return true;
738 }
739
740 Scanned = true;
741
742 // Create a compiler instance to handle the actual work.
743 auto ScanInvocation =
744 createScanCompilerInvocation(Invocation: *OriginalInvocation, Service);
745
746 // Quickly discovers and compiles modules for the real scan below.
747 std::optional<AsyncModuleCompiles> AsyncCompiles;
748 if (Service.getOpts().AsyncScanModules) {
749 auto ModCache = makeInProcessModuleCache(Entries&: Service.getModuleCacheEntries());
750 auto ScanInstanceStorage = std::make_unique<CompilerInstance>(
751 args: std::make_shared<CompilerInvocation>(args&: *ScanInvocation), args&: PCHContainerOps,
752 args: std::move(ModCache));
753 CompilerInstance &ScanInstance = *ScanInstanceStorage;
754
755 DiagnosticConsumer DiagConsumer;
756 initializeScanCompilerInstance(ScanInstance, FS, DiagConsumer: &DiagConsumer, Service,
757 DepFS);
758
759 // FIXME: Do this only once.
760 SmallVector<StringRef> StableDirs = getInitialStableDirs(ScanInstance);
761 auto MaybePrebuiltModulesASTMap =
762 computePrebuiltModulesASTMap(ScanInstance, StableDirs);
763 if (!MaybePrebuiltModulesASTMap)
764 return false;
765
766 // Normally this would be handled by GeneratePCHAction
767 if (ScanInstance.getFrontendOpts().ProgramAction == frontend::GeneratePCH)
768 ScanInstance.getLangOpts().CompilingPCH = true;
769
770 AsyncCompiles.emplace();
771 SingleTUWithAsyncModuleCompiles Action(Service, *AsyncCompiles);
772 (void)ScanInstance.ExecuteAction(Act&: Action);
773 }
774
775 auto ModCache = makeInProcessModuleCache(Entries&: Service.getModuleCacheEntries());
776 ScanInstanceStorage.emplace(args: std::move(ScanInvocation),
777 args: std::move(PCHContainerOps), args: std::move(ModCache));
778 CompilerInstance &ScanInstance = *ScanInstanceStorage;
779
780 assert(!DiagConsumerFinished && "attempt to reuse finished consumer");
781 initializeScanCompilerInstance(ScanInstance, FS, DiagConsumer, Service,
782 DepFS);
783
784 llvm::SmallVector<StringRef> StableDirs = getInitialStableDirs(ScanInstance);
785 auto MaybePrebuiltModulesASTMap =
786 computePrebuiltModulesASTMap(ScanInstance, StableDirs);
787 if (!MaybePrebuiltModulesASTMap)
788 return false;
789
790 auto DepOutputOpts = createDependencyOutputOptions(Invocation: *OriginalInvocation);
791
792 MDC = initializeScanInstanceDependencyCollector(
793 ScanInstance, DepOutputOpts: std::move(DepOutputOpts), WorkingDirectory, Consumer,
794 Service, Inv&: *OriginalInvocation, Controller, PrebuiltModulesASTMap: *MaybePrebuiltModulesASTMap,
795 StableDirs);
796
797 if (ScanInstance.getDiagnostics().hasErrorOccurred())
798 return false;
799
800 ReadPCHAndPreprocessAction Action;
801 const bool Result = ScanInstance.ExecuteAction(Act&: Action);
802
803 // ExecuteAction is responsible for calling finish.
804 DiagConsumerFinished = true;
805
806 if (Result) {
807 if (MDC)
808 MDC->applyDiscoveredDependencies(CI&: *OriginalInvocation);
809 Consumer.handleBuildCommand(
810 Cmd: {.Executable: Executable, .Arguments: OriginalInvocation->getCC1CommandLine()});
811 }
812
813 return Result;
814}
815
816bool CompilerInstanceWithContext::initialize(
817 std::unique_ptr<DiagnosticsEngineWithDiagOpts> DiagEngineWithDiagOpts,
818 IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> OverlayFS) {
819 assert(DiagEngineWithDiagOpts && "Valid diagnostics engine required!");
820 DiagEngineWithCmdAndOpts = std::move(DiagEngineWithDiagOpts);
821 DiagConsumer = DiagEngineWithCmdAndOpts->DiagEngine->getClient();
822
823#ifndef NDEBUG
824 assert(OverlayFS && "OverlayFS required!");
825 bool SawDepFS = false;
826 OverlayFS->visit([&](llvm::vfs::FileSystem &VFS) {
827 SawDepFS |= &VFS == Worker.DepFS.get();
828 });
829 assert(SawDepFS && "OverlayFS not based on DepFS");
830#endif
831
832 OriginalInvocation = createCompilerInvocation(
833 CommandLine, Diags&: *DiagEngineWithCmdAndOpts->DiagEngine);
834 if (!OriginalInvocation) {
835 DiagEngineWithCmdAndOpts->DiagEngine->Report(
836 DiagID: diag::err_fe_expected_compiler_job)
837 << llvm::join(R&: CommandLine, Separator: " ");
838 return false;
839 }
840
841 if (any(Val: Worker.Service.getOpts().OptimizeArgs &
842 ScanningOptimizations::Macros))
843 canonicalizeDefines(PPOpts&: OriginalInvocation->getPreprocessorOpts());
844
845 // Create the CompilerInstance.
846 std::shared_ptr<ModuleCache> ModCache =
847 makeInProcessModuleCache(Entries&: Worker.Service.getModuleCacheEntries());
848 CIPtr = std::make_unique<CompilerInstance>(
849 args: createScanCompilerInvocation(Invocation: *OriginalInvocation, Service: Worker.Service),
850 args&: Worker.PCHContainerOps, args: std::move(ModCache));
851 auto &CI = *CIPtr;
852
853 initializeScanCompilerInstance(
854 ScanInstance&: CI, FS: OverlayFS, DiagConsumer: DiagEngineWithCmdAndOpts->DiagEngine->getClient(),
855 Service&: Worker.Service, DepFS: Worker.DepFS);
856
857 StableDirs = getInitialStableDirs(ScanInstance: CI);
858 auto MaybePrebuiltModulesASTMap =
859 computePrebuiltModulesASTMap(ScanInstance&: CI, StableDirs);
860 if (!MaybePrebuiltModulesASTMap)
861 return false;
862
863 PrebuiltModuleASTMap = std::move(*MaybePrebuiltModulesASTMap);
864 OutputOpts = createDependencyOutputOptions(Invocation: *OriginalInvocation);
865
866 // We do not create the target in initializeScanCompilerInstance because
867 // setting it here is unique for by-name lookups. We create the target only
868 // once here, and the information is reused for all computeDependencies calls.
869 // We do not need to call createTarget explicitly if we go through
870 // CompilerInstance::ExecuteAction to perform scanning.
871 CI.createTarget();
872
873 return true;
874}
875
876bool CompilerInstanceWithContext::computeDependencies(
877 StringRef ModuleName, DependencyConsumer &Consumer,
878 DependencyActionController &Controller) {
879 assert(CIPtr && "CIPtr must be initialized before calling this method");
880 auto &CI = *CIPtr;
881
882 // We need to reset the diagnostics, so that the diagnostics issued
883 // during a previous computeDependencies call do not affect the current call.
884 // If we do not reset, we may inherit fatal errors from a previous call.
885 CI.getDiagnostics().Reset();
886
887 // We create this cleanup object because computeDependencies may exit
888 // early with errors.
889 llvm::scope_exit CleanUp([&]() {
890 CI.clearDependencyCollectors();
891 // The preprocessor may not be created at the entry of this method,
892 // but it must have been created when this method returns, whether
893 // there are errors during scanning or not.
894 CI.getPreprocessor().removePPCallbacks();
895 });
896
897 auto MDC = initializeScanInstanceDependencyCollector(
898 ScanInstance&: CI, DepOutputOpts: std::make_unique<DependencyOutputOptions>(args&: *OutputOpts), WorkingDirectory: CWD, Consumer,
899 Service&: Worker.Service,
900 /* The MDC's constructor makes a copy of the OriginalInvocation, so
901 we can pass it in without worrying that it might be changed across
902 invocations of computeDependencies. */
903 Inv&: *OriginalInvocation, Controller, PrebuiltModulesASTMap: PrebuiltModuleASTMap, StableDirs);
904
905 if (!SrcLocOffset) {
906 // When SrcLocOffset is zero, we are at the beginning of the fake source
907 // file. In this case, we call BeginSourceFile to initialize.
908 std::unique_ptr<FrontendAction> Action =
909 std::make_unique<PreprocessOnlyAction>();
910 auto *InputFile = CI.getFrontendOpts().Inputs.begin();
911 bool ActionBeginSucceeded = Action->BeginSourceFile(CI, Input: *InputFile);
912 assert(ActionBeginSucceeded && "Action BeginSourceFile must succeed");
913 (void)ActionBeginSucceeded;
914 }
915
916 Preprocessor &PP = CI.getPreprocessor();
917 SourceManager &SM = PP.getSourceManager();
918 FileID MainFileID = SM.getMainFileID();
919 SourceLocation FileStart = SM.getLocForStartOfFile(FID: MainFileID);
920 SourceLocation IDLocation = FileStart.getLocWithOffset(Offset: SrcLocOffset);
921 PPCallbacks *CB = nullptr;
922 if (!SrcLocOffset) {
923 // We need to call EnterSourceFile when SrcLocOffset is zero to initialize
924 // the preprocessor.
925 bool PPFailed = PP.EnterSourceFile(FID: MainFileID, Dir: nullptr, Loc: SourceLocation());
926 assert(!PPFailed && "Preprocess must be able to enter the main file.");
927 (void)PPFailed;
928 CB = MDC->getPPCallbacks();
929 } else {
930 // When SrcLocOffset is non-zero, the preprocessor has already been
931 // initialized through a previous call of computeDependencies. We want to
932 // preserve the PP's state, hence we do not call EnterSourceFile again.
933 MDC->attachToPreprocessor(PP);
934 CB = MDC->getPPCallbacks();
935
936 FileID PrevFID;
937 SrcMgr::CharacteristicKind FileType = SM.getFileCharacteristic(Loc: IDLocation);
938 CB->LexedFileChanged(FID: MainFileID,
939 Reason: PPChainedCallbacks::LexedFileChangeReason::EnterFile,
940 FileType, PrevFID, Loc: IDLocation);
941 }
942
943 // FIXME: Scan modules asynchronously here as well.
944
945 SrcLocOffset++;
946 SmallVector<IdentifierLoc, 2> Path;
947 IdentifierInfo *ModuleID = PP.getIdentifierInfo(Name: ModuleName);
948 Path.emplace_back(Args&: IDLocation, Args&: ModuleID);
949 auto ModResult = CI.loadModule(ImportLoc: IDLocation, Path, Visibility: Module::Hidden, IsInclusionDirective: false);
950
951 assert(CB && "Must have PPCallbacks after module loading");
952 CB->moduleImport(ImportLoc: SourceLocation(), Path, Imported: ModResult);
953 // Note that we are calling the CB's EndOfMainFile function, which
954 // forwards the results to the dependency consumer.
955 // It does not indicate the end of processing the fake file.
956 CB->EndOfMainFile();
957
958 if (!ModResult)
959 return false;
960
961 CompilerInvocation ModuleInvocation(*OriginalInvocation);
962 MDC->applyDiscoveredDependencies(CI&: ModuleInvocation);
963 Consumer.handleBuildCommand(
964 Cmd: {.Executable: CommandLine[0], .Arguments: ModuleInvocation.getCC1CommandLine()});
965
966 return true;
967}
968
969bool CompilerInstanceWithContext::finalize() {
970 DiagConsumer->finish();
971 return true;
972}
973