1//===- DependencyScannerImpl.cpp - Implements module dependency scanning --===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "clang/DependencyScanning/DependencyScannerImpl.h"
10#include "clang/Basic/DiagnosticFrontend.h"
11#include "clang/Basic/DiagnosticSerialization.h"
12#include "clang/DependencyScanning/DependencyScanningWorker.h"
13#include "clang/Driver/Driver.h"
14#include "clang/Frontend/FrontendActions.h"
15#include "llvm/ADT/ScopeExit.h"
16#include "llvm/TargetParser/Host.h"
17
18using namespace clang;
19using namespace dependencies;
20
21namespace {
22/// Forwards the gatherered dependencies to the consumer.
23class DependencyConsumerForwarder : public DependencyFileGenerator {
24public:
25 DependencyConsumerForwarder(std::unique_ptr<DependencyOutputOptions> Opts,
26 StringRef WorkingDirectory, DependencyConsumer &C)
27 : DependencyFileGenerator(*Opts), WorkingDirectory(WorkingDirectory),
28 Opts(std::move(Opts)), C(C) {}
29
30 void finishedMainFile(DiagnosticsEngine &Diags) override {
31 C.handleDependencyOutputOpts(Opts: *Opts);
32 llvm::SmallString<256> CanonPath;
33 for (const auto &File : getDependencies()) {
34 CanonPath = File;
35 llvm::sys::path::remove_dots(path&: CanonPath, /*remove_dot_dot=*/true);
36 llvm::sys::path::make_absolute(current_directory: WorkingDirectory, path&: CanonPath);
37 C.handleFileDependency(Filename: CanonPath);
38 }
39 }
40
41private:
42 StringRef WorkingDirectory;
43 std::unique_ptr<DependencyOutputOptions> Opts;
44 DependencyConsumer &C;
45};
46
47static bool checkHeaderSearchPaths(const HeaderSearchOptions &HSOpts,
48 const HeaderSearchOptions &ExistingHSOpts,
49 DiagnosticsEngine *Diags,
50 const LangOptions &LangOpts) {
51 if (LangOpts.Modules) {
52 if (HSOpts.VFSOverlayFiles != ExistingHSOpts.VFSOverlayFiles) {
53 if (Diags) {
54 Diags->Report(DiagID: diag::warn_pch_vfsoverlay_mismatch);
55 auto VFSNote = [&](int Type, ArrayRef<std::string> VFSOverlays) {
56 if (VFSOverlays.empty()) {
57 Diags->Report(DiagID: diag::note_pch_vfsoverlay_empty) << Type;
58 } else {
59 std::string Files = llvm::join(R&: VFSOverlays, Separator: "\n");
60 Diags->Report(DiagID: diag::note_pch_vfsoverlay_files) << Type << Files;
61 }
62 };
63 VFSNote(0, HSOpts.VFSOverlayFiles);
64 VFSNote(1, ExistingHSOpts.VFSOverlayFiles);
65 }
66 }
67 }
68 return false;
69}
70
71using PrebuiltModuleFilesT = decltype(HeaderSearchOptions::PrebuiltModuleFiles);
72
73/// A listener that collects the imported modules and the input
74/// files. While visiting, collect vfsoverlays and file inputs that determine
75/// whether prebuilt modules fully resolve in stable directories.
76class PrebuiltModuleListener : public ASTReaderListener {
77public:
78 PrebuiltModuleListener(PrebuiltModuleFilesT &PrebuiltModuleFiles,
79 llvm::SmallVector<std::string> &NewModuleFiles,
80 PrebuiltModulesAttrsMap &PrebuiltModulesASTMap,
81 const HeaderSearchOptions &HSOpts,
82 const LangOptions &LangOpts, DiagnosticsEngine &Diags,
83 const ArrayRef<StringRef> StableDirs)
84 : PrebuiltModuleFiles(PrebuiltModuleFiles),
85 NewModuleFiles(NewModuleFiles),
86 PrebuiltModulesASTMap(PrebuiltModulesASTMap), ExistingHSOpts(HSOpts),
87 ExistingLangOpts(LangOpts), Diags(Diags), StableDirs(StableDirs) {}
88
89 bool needsImportVisitation() const override { return true; }
90 bool needsInputFileVisitation() override { return true; }
91 bool needsSystemInputFileVisitation() override { return true; }
92
93 /// Accumulate the modules are transitively depended on by the initial
94 /// prebuilt module.
95 void visitImport(StringRef ModuleName, StringRef Filename) override {
96 if (PrebuiltModuleFiles.insert(x: {ModuleName.str(), Filename.str()}).second)
97 NewModuleFiles.push_back(Elt: Filename.str());
98
99 auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(Key: Filename);
100 PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second;
101 if (PrebuiltMapEntry.second)
102 PrebuiltModule.setInStableDir(!StableDirs.empty());
103
104 if (auto It = PrebuiltModulesASTMap.find(Key: CurrentFile);
105 It != PrebuiltModulesASTMap.end() && CurrentFile != Filename)
106 PrebuiltModule.addDependent(ModuleFile: It->getKey());
107 }
108
109 /// For each input file discovered, check whether it's external path is in a
110 /// stable directory. Traversal is stopped if the current module is not
111 /// considered stable.
112 bool visitInputFileAsRequested(StringRef FilenameAsRequested,
113 StringRef Filename, bool isSystem,
114 bool isOverridden, time_t StoredTime,
115 bool isExplicitModule) override {
116 if (StableDirs.empty())
117 return false;
118 auto PrebuiltEntryIt = PrebuiltModulesASTMap.find(Key: CurrentFile);
119 if ((PrebuiltEntryIt == PrebuiltModulesASTMap.end()) ||
120 (!PrebuiltEntryIt->second.isInStableDir()))
121 return false;
122
123 PrebuiltEntryIt->second.setInStableDir(
124 isPathInStableDir(Directories: StableDirs, Input: Filename));
125 return PrebuiltEntryIt->second.isInStableDir();
126 }
127
128 /// Update which module that is being actively traversed.
129 void visitModuleFile(StringRef Filename,
130 serialization::ModuleKind Kind) override {
131 // If the CurrentFile is not
132 // considered stable, update any of it's transitive dependents.
133 auto PrebuiltEntryIt = PrebuiltModulesASTMap.find(Key: CurrentFile);
134 if ((PrebuiltEntryIt != PrebuiltModulesASTMap.end()) &&
135 !PrebuiltEntryIt->second.isInStableDir())
136 PrebuiltEntryIt->second.updateDependentsNotInStableDirs(
137 PrebuiltModulesMap&: PrebuiltModulesASTMap);
138 CurrentFile = Filename;
139 }
140
141 /// Check the header search options for a given module when considering
142 /// if the module comes from stable directories.
143 bool ReadHeaderSearchOptions(const HeaderSearchOptions &HSOpts,
144 StringRef ModuleFilename,
145 StringRef SpecificModuleCachePath,
146 bool Complain) override {
147
148 auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(Key: CurrentFile);
149 PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second;
150 if (PrebuiltMapEntry.second)
151 PrebuiltModule.setInStableDir(!StableDirs.empty());
152
153 if (PrebuiltModule.isInStableDir())
154 PrebuiltModule.setInStableDir(areOptionsInStableDir(Directories: StableDirs, HSOpts));
155
156 return false;
157 }
158
159 /// Accumulate vfsoverlays used to build these prebuilt modules.
160 bool ReadHeaderSearchPaths(const HeaderSearchOptions &HSOpts,
161 bool Complain) override {
162
163 auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(Key: CurrentFile);
164 PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second;
165 if (PrebuiltMapEntry.second)
166 PrebuiltModule.setInStableDir(!StableDirs.empty());
167
168 PrebuiltModule.setVFS(
169 llvm::StringSet<>(llvm::from_range, HSOpts.VFSOverlayFiles));
170
171 return checkHeaderSearchPaths(
172 HSOpts, ExistingHSOpts, Diags: Complain ? &Diags : nullptr, LangOpts: ExistingLangOpts);
173 }
174
175private:
176 PrebuiltModuleFilesT &PrebuiltModuleFiles;
177 llvm::SmallVector<std::string> &NewModuleFiles;
178 PrebuiltModulesAttrsMap &PrebuiltModulesASTMap;
179 const HeaderSearchOptions &ExistingHSOpts;
180 const LangOptions &ExistingLangOpts;
181 DiagnosticsEngine &Diags;
182 std::string CurrentFile;
183 const ArrayRef<StringRef> StableDirs;
184};
185
186/// Visit the given prebuilt module and collect all of the modules it
187/// transitively imports and contributing input files.
188static bool visitPrebuiltModule(StringRef PrebuiltModuleFilename,
189 CompilerInstance &CI,
190 PrebuiltModuleFilesT &ModuleFiles,
191 PrebuiltModulesAttrsMap &PrebuiltModulesASTMap,
192 DiagnosticsEngine &Diags,
193 const ArrayRef<StringRef> StableDirs) {
194 // List of module files to be processed.
195 llvm::SmallVector<std::string> Worklist;
196
197 PrebuiltModuleListener Listener(ModuleFiles, Worklist, PrebuiltModulesASTMap,
198 CI.getHeaderSearchOpts(), CI.getLangOpts(),
199 Diags, StableDirs);
200
201 Listener.visitModuleFile(Filename: PrebuiltModuleFilename,
202 Kind: serialization::MK_ExplicitModule);
203 if (ASTReader::readASTFileControlBlock(
204 Filename: PrebuiltModuleFilename, FileMgr&: CI.getFileManager(), ModCache: CI.getModuleCache(),
205 PCHContainerRdr: CI.getPCHContainerReader(),
206 /*FindModuleFileExtensions=*/false, Listener,
207 /*ValidateDiagnosticOptions=*/false, ClientLoadCapabilities: ASTReader::ARR_OutOfDate))
208 return true;
209
210 while (!Worklist.empty()) {
211 Listener.visitModuleFile(Filename: Worklist.back(), Kind: serialization::MK_ExplicitModule);
212 if (ASTReader::readASTFileControlBlock(
213 Filename: Worklist.pop_back_val(), FileMgr&: CI.getFileManager(), ModCache: CI.getModuleCache(),
214 PCHContainerRdr: CI.getPCHContainerReader(),
215 /*FindModuleFileExtensions=*/false, Listener,
216 /*ValidateDiagnosticOptions=*/false))
217 return true;
218 }
219 return false;
220}
221
222/// Transform arbitrary file name into an object-like file name.
223static std::string makeObjFileName(StringRef FileName) {
224 SmallString<128> ObjFileName(FileName);
225 llvm::sys::path::replace_extension(path&: ObjFileName, extension: "o");
226 return std::string(ObjFileName);
227}
228
229/// Deduce the dependency target based on the output file and input files.
230static std::string
231deduceDepTarget(const std::string &OutputFile,
232 const SmallVectorImpl<FrontendInputFile> &InputFiles) {
233 if (OutputFile != "-")
234 return OutputFile;
235
236 if (InputFiles.empty() || !InputFiles.front().isFile())
237 return "clang-scan-deps\\ dependency";
238
239 return makeObjFileName(FileName: InputFiles.front().getFile());
240}
241
242// Clang implements -D and -U by splatting text into a predefines buffer. This
243// allows constructs such as `-DFඞ=3 "-D F\u{0D9E} 4 3 2”` to be accepted and
244// define the same macro, or adding C++ style comments before the macro name.
245//
246// This function checks that the first non-space characters in the macro
247// obviously form an identifier that can be uniqued on without lexing. Failing
248// to do this could lead to changing the final definition of a macro.
249//
250// We could set up a preprocessor and actually lex the name, but that's very
251// heavyweight for a situation that will almost never happen in practice.
252static std::optional<StringRef> getSimpleMacroName(StringRef Macro) {
253 StringRef Name = Macro.split(Separator: "=").first.ltrim(Chars: " \t");
254 std::size_t I = 0;
255
256 auto FinishName = [&]() -> std::optional<StringRef> {
257 StringRef SimpleName = Name.slice(Start: 0, End: I);
258 if (SimpleName.empty())
259 return std::nullopt;
260 return SimpleName;
261 };
262
263 for (; I != Name.size(); ++I) {
264 switch (Name[I]) {
265 case '(': // Start of macro parameter list
266 case ' ': // End of macro name
267 case '\t':
268 return FinishName();
269 case '_':
270 continue;
271 default:
272 if (llvm::isAlnum(C: Name[I]))
273 continue;
274 return std::nullopt;
275 }
276 }
277 return FinishName();
278}
279
280static void canonicalizeDefines(PreprocessorOptions &PPOpts) {
281 using MacroOpt = std::pair<StringRef, std::size_t>;
282 std::vector<MacroOpt> SimpleNames;
283 SimpleNames.reserve(n: PPOpts.Macros.size());
284 std::size_t Index = 0;
285 for (const auto &M : PPOpts.Macros) {
286 auto SName = getSimpleMacroName(Macro: M.first);
287 // Skip optimizing if we can't guarantee we can preserve relative order.
288 if (!SName)
289 return;
290 SimpleNames.emplace_back(args&: *SName, args&: Index);
291 ++Index;
292 }
293
294 llvm::stable_sort(Range&: SimpleNames, C: llvm::less_first());
295 // Keep the last instance of each macro name by going in reverse
296 auto NewEnd = std::unique(
297 first: SimpleNames.rbegin(), last: SimpleNames.rend(),
298 binary_pred: [](const MacroOpt &A, const MacroOpt &B) { return A.first == B.first; });
299 SimpleNames.erase(first: SimpleNames.begin(), last: NewEnd.base());
300
301 // Apply permutation.
302 decltype(PPOpts.Macros) NewMacros;
303 NewMacros.reserve(n: SimpleNames.size());
304 for (std::size_t I = 0, E = SimpleNames.size(); I != E; ++I) {
305 std::size_t OriginalIndex = SimpleNames[I].second;
306 // We still emit undefines here as they may be undefining a predefined macro
307 NewMacros.push_back(x: std::move(PPOpts.Macros[OriginalIndex]));
308 }
309 std::swap(x&: PPOpts.Macros, y&: NewMacros);
310}
311
312class ScanningDependencyDirectivesGetter : public DependencyDirectivesGetter {
313 DependencyScanningWorkerFilesystem *DepFS;
314
315public:
316 ScanningDependencyDirectivesGetter(FileManager &FileMgr) : DepFS(nullptr) {
317 FileMgr.getVirtualFileSystem().visit(Callback: [&](llvm::vfs::FileSystem &FS) {
318 auto *DFS = llvm::dyn_cast<DependencyScanningWorkerFilesystem>(Val: &FS);
319 if (DFS) {
320 assert(!DepFS && "Found multiple scanning VFSs");
321 DepFS = DFS;
322 }
323 });
324 assert(DepFS && "Did not find scanning VFS");
325 }
326
327 std::unique_ptr<DependencyDirectivesGetter>
328 cloneFor(FileManager &FileMgr) override {
329 return std::make_unique<ScanningDependencyDirectivesGetter>(args&: FileMgr);
330 }
331
332 std::optional<ArrayRef<dependency_directives_scan::Directive>>
333 operator()(FileEntryRef File) override {
334 return DepFS->getDirectiveTokens(Path: File.getName());
335 }
336};
337
338/// Sanitize diagnostic options for dependency scan.
339void sanitizeDiagOpts(DiagnosticOptions &DiagOpts) {
340 // Don't print 'X warnings and Y errors generated'.
341 DiagOpts.ShowCarets = false;
342 // Don't write out diagnostic file.
343 DiagOpts.DiagnosticSerializationFile.clear();
344 // Don't emit warnings except for scanning specific warnings.
345 // TODO: It would be useful to add a more principled way to ignore all
346 // warnings that come from source code. The issue is that we need to
347 // ignore warnings that could be surpressed by
348 // `#pragma clang diagnostic`, while still allowing some scanning
349 // warnings for things we're not ready to turn into errors yet.
350 // See `test/ClangScanDeps/diagnostic-pragmas.c` for an example.
351 llvm::erase_if(C&: DiagOpts.Warnings, P: [](StringRef Warning) {
352 return llvm::StringSwitch<bool>(Warning)
353 .Cases(CaseStrings: {"pch-vfs-diff", "error=pch-vfs-diff"}, Value: false)
354 .StartsWith(S: "no-error=", Value: false)
355 .Default(Value: true);
356 });
357}
358} // namespace
359
360std::unique_ptr<DiagnosticOptions>
361dependencies::createDiagOptions(ArrayRef<std::string> CommandLine) {
362 std::vector<const char *> CLI;
363 for (const std::string &Arg : CommandLine)
364 CLI.push_back(x: Arg.c_str());
365 auto DiagOpts = CreateAndPopulateDiagOpts(Argv: CLI);
366 sanitizeDiagOpts(DiagOpts&: *DiagOpts);
367 return DiagOpts;
368}
369
370DiagnosticsEngineWithDiagOpts::DiagnosticsEngineWithDiagOpts(
371 ArrayRef<std::string> CommandLine,
372 IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS, DiagnosticConsumer &DC) {
373 std::vector<const char *> CCommandLine(CommandLine.size(), nullptr);
374 llvm::transform(Range&: CommandLine, d_first: CCommandLine.begin(),
375 F: [](const std::string &Str) { return Str.c_str(); });
376 DiagOpts = CreateAndPopulateDiagOpts(Argv: CCommandLine);
377 sanitizeDiagOpts(DiagOpts&: *DiagOpts);
378 DiagEngine = CompilerInstance::createDiagnostics(VFS&: *FS, Opts&: *DiagOpts, Client: &DC,
379 /*ShouldOwnClient=*/false);
380}
381
382std::unique_ptr<CompilerInvocation>
383dependencies::createCompilerInvocation(ArrayRef<std::string> CommandLine,
384 DiagnosticsEngine &Diags) {
385 llvm::opt::ArgStringList Argv;
386 for (const std::string &Str : ArrayRef(CommandLine).drop_front())
387 Argv.push_back(Elt: Str.c_str());
388
389 auto Invocation = std::make_unique<CompilerInvocation>();
390 if (!CompilerInvocation::CreateFromArgs(Res&: *Invocation, CommandLineArgs: Argv, Diags)) {
391 // FIXME: Should we just go on like cc1_main does?
392 return nullptr;
393 }
394 return Invocation;
395}
396
397void dependencies::initializeScanCompilerInstance(
398 CompilerInstance &ScanInstance,
399 IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
400 DiagnosticConsumer *DiagConsumer, DependencyScanningService &Service,
401 IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS) {
402 ScanInstance.setBuildingModule(false);
403 ScanInstance.createVirtualFileSystem(BaseFS: FS, DC: DiagConsumer);
404 ScanInstance.createDiagnostics(Client: DiagConsumer, /*ShouldOwnClient=*/false);
405 ScanInstance.createFileManager();
406 ScanInstance.createSourceManager();
407
408 // Use DepFS for getting the dependency directives if requested to do so.
409 if (Service.getMode() == ScanningMode::DependencyDirectivesScan) {
410 DepFS->resetBypassedPathPrefix();
411 SmallString<256> ModulesCachePath;
412 normalizeModuleCachePath(FileMgr&: ScanInstance.getFileManager(),
413 Path: ScanInstance.getHeaderSearchOpts().ModuleCachePath,
414 NormalizedPath&: ModulesCachePath);
415 if (!ModulesCachePath.empty())
416 DepFS->setBypassedPathPrefix(ModulesCachePath);
417
418 ScanInstance.setDependencyDirectivesGetter(
419 std::make_unique<ScanningDependencyDirectivesGetter>(
420 args&: ScanInstance.getFileManager()));
421 }
422}
423
424/// Creates a CompilerInvocation suitable for the dependency scanner.
425static std::shared_ptr<CompilerInvocation>
426createScanCompilerInvocation(const CompilerInvocation &Invocation,
427 const DependencyScanningService &Service) {
428 auto ScanInvocation = std::make_shared<CompilerInvocation>(args: Invocation);
429
430 sanitizeDiagOpts(DiagOpts&: ScanInvocation->getDiagnosticOpts());
431
432 ScanInvocation->getPreprocessorOpts().AllowPCHWithDifferentModulesCachePath =
433 true;
434
435 if (ScanInvocation->getHeaderSearchOpts().ModulesValidateOncePerBuildSession)
436 ScanInvocation->getHeaderSearchOpts().BuildSessionTimestamp =
437 Service.getBuildSessionTimestamp();
438
439 ScanInvocation->getFrontendOpts().DisableFree = false;
440 ScanInvocation->getFrontendOpts().GenerateGlobalModuleIndex = false;
441 ScanInvocation->getFrontendOpts().UseGlobalModuleIndex = false;
442 ScanInvocation->getFrontendOpts().GenReducedBMI = false;
443 ScanInvocation->getFrontendOpts().ModuleOutputPath.clear();
444 // This will prevent us compiling individual modules asynchronously since
445 // FileManager is not thread-safe, but it does improve performance for now.
446 ScanInvocation->getFrontendOpts().ModulesShareFileManager = true;
447 ScanInvocation->getHeaderSearchOpts().ModuleFormat = "raw";
448 ScanInvocation->getHeaderSearchOpts().ModulesIncludeVFSUsage =
449 any(Val: Service.getOptimizeArgs() & ScanningOptimizations::VFS);
450
451 // Consider different header search and diagnostic options to create
452 // different modules. This avoids the unsound aliasing of module PCMs.
453 //
454 // TODO: Implement diagnostic bucketing to reduce the impact of strict
455 // context hashing.
456 ScanInvocation->getHeaderSearchOpts().ModulesStrictContextHash = true;
457 ScanInvocation->getHeaderSearchOpts().ModulesSerializeOnlyPreprocessor = true;
458 ScanInvocation->getHeaderSearchOpts().ModulesSkipDiagnosticOptions = true;
459 ScanInvocation->getHeaderSearchOpts().ModulesSkipHeaderSearchPaths = true;
460 ScanInvocation->getHeaderSearchOpts().ModulesSkipPragmaDiagnosticMappings =
461 true;
462 ScanInvocation->getHeaderSearchOpts().ModulesForceValidateUserHeaders = false;
463
464 // Avoid some checks and module map parsing when loading PCM files.
465 ScanInvocation->getPreprocessorOpts().ModulesCheckRelocated = false;
466
467 // Ensure that the scanner does not create new dependency collectors,
468 // and thus won't write out the extra '.d' files to disk.
469 ScanInvocation->getDependencyOutputOpts() = {};
470
471 return ScanInvocation;
472}
473
474llvm::SmallVector<StringRef>
475dependencies::getInitialStableDirs(const CompilerInstance &ScanInstance) {
476 // Create a collection of stable directories derived from the ScanInstance
477 // for determining whether module dependencies would fully resolve from
478 // those directories.
479 llvm::SmallVector<StringRef> StableDirs;
480 const StringRef Sysroot = ScanInstance.getHeaderSearchOpts().Sysroot;
481 if (!Sysroot.empty() && (llvm::sys::path::root_directory(path: Sysroot) != Sysroot))
482 StableDirs = {Sysroot, ScanInstance.getHeaderSearchOpts().ResourceDir};
483 return StableDirs;
484}
485
486std::optional<PrebuiltModulesAttrsMap>
487dependencies::computePrebuiltModulesASTMap(
488 CompilerInstance &ScanInstance, llvm::SmallVector<StringRef> &StableDirs) {
489 // Store a mapping of prebuilt module files and their properties like header
490 // search options. This will prevent the implicit build to create duplicate
491 // modules and will force reuse of the existing prebuilt module files
492 // instead.
493 PrebuiltModulesAttrsMap PrebuiltModulesASTMap;
494
495 if (!ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty())
496 if (visitPrebuiltModule(
497 PrebuiltModuleFilename: ScanInstance.getPreprocessorOpts().ImplicitPCHInclude, CI&: ScanInstance,
498 ModuleFiles&: ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles,
499 PrebuiltModulesASTMap, Diags&: ScanInstance.getDiagnostics(), StableDirs))
500 return {};
501
502 return PrebuiltModulesASTMap;
503}
504
505/// Creates dependency output options to be reported to the dependency consumer,
506/// deducing missing information if necessary.
507static std::unique_ptr<DependencyOutputOptions>
508createDependencyOutputOptions(const CompilerInvocation &Invocation) {
509 auto Opts = std::make_unique<DependencyOutputOptions>(
510 args: Invocation.getDependencyOutputOpts());
511 // We need at least one -MT equivalent for the generator of make dependency
512 // files to work.
513 if (Opts->Targets.empty())
514 Opts->Targets = {deduceDepTarget(OutputFile: Invocation.getFrontendOpts().OutputFile,
515 InputFiles: Invocation.getFrontendOpts().Inputs)};
516 Opts->IncludeSystemHeaders = true;
517
518 return Opts;
519}
520
521std::shared_ptr<ModuleDepCollector>
522dependencies::initializeScanInstanceDependencyCollector(
523 CompilerInstance &ScanInstance,
524 std::unique_ptr<DependencyOutputOptions> DepOutputOpts,
525 StringRef WorkingDirectory, DependencyConsumer &Consumer,
526 DependencyScanningService &Service, CompilerInvocation &Inv,
527 DependencyActionController &Controller,
528 PrebuiltModulesAttrsMap PrebuiltModulesASTMap,
529 llvm::SmallVector<StringRef> &StableDirs) {
530 std::shared_ptr<ModuleDepCollector> MDC;
531 switch (Service.getFormat()) {
532 case ScanningOutputFormat::Make:
533 ScanInstance.addDependencyCollector(
534 Listener: std::make_shared<DependencyConsumerForwarder>(
535 args: std::move(DepOutputOpts), args&: WorkingDirectory, args&: Consumer));
536 break;
537 case ScanningOutputFormat::P1689:
538 case ScanningOutputFormat::Full:
539 MDC = std::make_shared<ModuleDepCollector>(
540 args&: Service, args: std::move(DepOutputOpts), args&: ScanInstance, args&: Consumer, args&: Controller,
541 args&: Inv, args: std::move(PrebuiltModulesASTMap), args&: StableDirs);
542 ScanInstance.addDependencyCollector(Listener: MDC);
543 break;
544 }
545
546 return MDC;
547}
548
549bool DependencyScanningAction::runInvocation(
550 std::string Executable,
551 std::unique_ptr<CompilerInvocation> OriginalInvocation,
552 IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
553 std::shared_ptr<PCHContainerOperations> PCHContainerOps,
554 DiagnosticConsumer *DiagConsumer) {
555 // Making sure that we canonicalize the defines early to avoid unnecessary
556 // variants in both the scanner and in the resulting explicit command lines.
557 if (any(Val: Service.getOptimizeArgs() & ScanningOptimizations::Macros))
558 canonicalizeDefines(PPOpts&: OriginalInvocation->getPreprocessorOpts());
559
560 if (Scanned) {
561 // Scanning runs once for the first -cc1 invocation in a chain of driver
562 // jobs. For any dependent jobs, reuse the scanning result and just
563 // update the new invocation.
564 // FIXME: to support multi-arch builds, each arch requires a separate scan
565 if (MDC)
566 MDC->applyDiscoveredDependencies(CI&: *OriginalInvocation);
567 Consumer.handleBuildCommand(
568 Cmd: {.Executable: Executable, .Arguments: OriginalInvocation->getCC1CommandLine()});
569 return true;
570 }
571
572 Scanned = true;
573
574 // Create a compiler instance to handle the actual work.
575 auto ScanInvocation =
576 createScanCompilerInvocation(Invocation: *OriginalInvocation, Service);
577 auto ModCache = makeInProcessModuleCache(Entries&: Service.getModuleCacheEntries());
578 ScanInstanceStorage.emplace(args: std::move(ScanInvocation),
579 args: std::move(PCHContainerOps), args: std::move(ModCache));
580 CompilerInstance &ScanInstance = *ScanInstanceStorage;
581
582 assert(!DiagConsumerFinished && "attempt to reuse finished consumer");
583 initializeScanCompilerInstance(ScanInstance, FS, DiagConsumer, Service,
584 DepFS);
585
586 llvm::SmallVector<StringRef> StableDirs = getInitialStableDirs(ScanInstance);
587 auto MaybePrebuiltModulesASTMap =
588 computePrebuiltModulesASTMap(ScanInstance, StableDirs);
589 if (!MaybePrebuiltModulesASTMap)
590 return false;
591
592 auto DepOutputOpts = createDependencyOutputOptions(Invocation: *OriginalInvocation);
593
594 MDC = initializeScanInstanceDependencyCollector(
595 ScanInstance, DepOutputOpts: std::move(DepOutputOpts), WorkingDirectory, Consumer,
596 Service, Inv&: *OriginalInvocation, Controller, PrebuiltModulesASTMap: *MaybePrebuiltModulesASTMap,
597 StableDirs);
598
599 std::unique_ptr<FrontendAction> Action;
600
601 if (Service.getFormat() == ScanningOutputFormat::P1689)
602 Action = std::make_unique<PreprocessOnlyAction>();
603 else
604 Action = std::make_unique<ReadPCHAndPreprocessAction>();
605
606 if (ScanInstance.getDiagnostics().hasErrorOccurred())
607 return false;
608
609 const bool Result = ScanInstance.ExecuteAction(Act&: *Action);
610
611 // ExecuteAction is responsible for calling finish.
612 DiagConsumerFinished = true;
613
614 if (Result) {
615 if (MDC)
616 MDC->applyDiscoveredDependencies(CI&: *OriginalInvocation);
617 Consumer.handleBuildCommand(
618 Cmd: {.Executable: Executable, .Arguments: OriginalInvocation->getCC1CommandLine()});
619 }
620
621 return Result;
622}
623
624bool CompilerInstanceWithContext::initialize(
625 std::unique_ptr<DiagnosticsEngineWithDiagOpts> DiagEngineWithDiagOpts,
626 IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> OverlayFS) {
627 assert(DiagEngineWithDiagOpts && "Valid diagnostics engine required!");
628 DiagEngineWithCmdAndOpts = std::move(DiagEngineWithDiagOpts);
629 DiagConsumer = DiagEngineWithCmdAndOpts->DiagEngine->getClient();
630
631#ifndef NDEBUG
632 assert(OverlayFS && "OverlayFS required!");
633 bool SawDepFS = false;
634 OverlayFS->visit([&](llvm::vfs::FileSystem &VFS) {
635 SawDepFS |= &VFS == Worker.DepFS.get();
636 });
637 assert(SawDepFS && "OverlayFS not based on DepFS");
638#endif
639
640 OriginalInvocation = createCompilerInvocation(
641 CommandLine, Diags&: *DiagEngineWithCmdAndOpts->DiagEngine);
642 if (!OriginalInvocation) {
643 DiagEngineWithCmdAndOpts->DiagEngine->Report(
644 DiagID: diag::err_fe_expected_compiler_job)
645 << llvm::join(R&: CommandLine, Separator: " ");
646 return false;
647 }
648
649 if (any(Val: Worker.Service.getOptimizeArgs() & ScanningOptimizations::Macros))
650 canonicalizeDefines(PPOpts&: OriginalInvocation->getPreprocessorOpts());
651
652 // Create the CompilerInstance.
653 std::shared_ptr<ModuleCache> ModCache =
654 makeInProcessModuleCache(Entries&: Worker.Service.getModuleCacheEntries());
655 CIPtr = std::make_unique<CompilerInstance>(
656 args: createScanCompilerInvocation(Invocation: *OriginalInvocation, Service: Worker.Service),
657 args&: Worker.PCHContainerOps, args: std::move(ModCache));
658 auto &CI = *CIPtr;
659
660 initializeScanCompilerInstance(
661 ScanInstance&: CI, FS: OverlayFS, DiagConsumer: DiagEngineWithCmdAndOpts->DiagEngine->getClient(),
662 Service&: Worker.Service, DepFS: Worker.DepFS);
663
664 StableDirs = getInitialStableDirs(ScanInstance: CI);
665 auto MaybePrebuiltModulesASTMap =
666 computePrebuiltModulesASTMap(ScanInstance&: CI, StableDirs);
667 if (!MaybePrebuiltModulesASTMap)
668 return false;
669
670 PrebuiltModuleASTMap = std::move(*MaybePrebuiltModulesASTMap);
671 OutputOpts = createDependencyOutputOptions(Invocation: *OriginalInvocation);
672
673 // We do not create the target in initializeScanCompilerInstance because
674 // setting it here is unique for by-name lookups. We create the target only
675 // once here, and the information is reused for all computeDependencies calls.
676 // We do not need to call createTarget explicitly if we go through
677 // CompilerInstance::ExecuteAction to perform scanning.
678 CI.createTarget();
679
680 return true;
681}
682
683bool CompilerInstanceWithContext::computeDependencies(
684 StringRef ModuleName, DependencyConsumer &Consumer,
685 DependencyActionController &Controller) {
686 assert(CIPtr && "CIPtr must be initialized before calling this method");
687 auto &CI = *CIPtr;
688
689 // We need to reset the diagnostics, so that the diagnostics issued
690 // during a previous computeDependencies call do not affect the current call.
691 // If we do not reset, we may inherit fatal errors from a previous call.
692 CI.getDiagnostics().Reset();
693
694 // We create this cleanup object because computeDependencies may exit
695 // early with errors.
696 llvm::scope_exit CleanUp([&]() {
697 CI.clearDependencyCollectors();
698 // The preprocessor may not be created at the entry of this method,
699 // but it must have been created when this method returns, whether
700 // there are errors during scanning or not.
701 CI.getPreprocessor().removePPCallbacks();
702 });
703
704 auto MDC = initializeScanInstanceDependencyCollector(
705 ScanInstance&: CI, DepOutputOpts: std::make_unique<DependencyOutputOptions>(args&: *OutputOpts), WorkingDirectory: CWD, Consumer,
706 Service&: Worker.Service,
707 /* The MDC's constructor makes a copy of the OriginalInvocation, so
708 we can pass it in without worrying that it might be changed across
709 invocations of computeDependencies. */
710 Inv&: *OriginalInvocation, Controller, PrebuiltModulesASTMap: PrebuiltModuleASTMap, StableDirs);
711
712 if (!SrcLocOffset) {
713 // When SrcLocOffset is zero, we are at the beginning of the fake source
714 // file. In this case, we call BeginSourceFile to initialize.
715 std::unique_ptr<FrontendAction> Action =
716 std::make_unique<PreprocessOnlyAction>();
717 auto *InputFile = CI.getFrontendOpts().Inputs.begin();
718 bool ActionBeginSucceeded = Action->BeginSourceFile(CI, Input: *InputFile);
719 assert(ActionBeginSucceeded && "Action BeginSourceFile must succeed");
720 (void)ActionBeginSucceeded;
721 }
722
723 Preprocessor &PP = CI.getPreprocessor();
724 SourceManager &SM = PP.getSourceManager();
725 FileID MainFileID = SM.getMainFileID();
726 SourceLocation FileStart = SM.getLocForStartOfFile(FID: MainFileID);
727 SourceLocation IDLocation = FileStart.getLocWithOffset(Offset: SrcLocOffset);
728 PPCallbacks *CB = nullptr;
729 if (!SrcLocOffset) {
730 // We need to call EnterSourceFile when SrcLocOffset is zero to initialize
731 // the preprocessor.
732 bool PPFailed = PP.EnterSourceFile(FID: MainFileID, Dir: nullptr, Loc: SourceLocation());
733 assert(!PPFailed && "Preprocess must be able to enter the main file.");
734 (void)PPFailed;
735 CB = MDC->getPPCallbacks();
736 } else {
737 // When SrcLocOffset is non-zero, the preprocessor has already been
738 // initialized through a previous call of computeDependencies. We want to
739 // preserve the PP's state, hence we do not call EnterSourceFile again.
740 MDC->attachToPreprocessor(PP);
741 CB = MDC->getPPCallbacks();
742
743 FileID PrevFID;
744 SrcMgr::CharacteristicKind FileType = SM.getFileCharacteristic(Loc: IDLocation);
745 CB->LexedFileChanged(FID: MainFileID,
746 Reason: PPChainedCallbacks::LexedFileChangeReason::EnterFile,
747 FileType, PrevFID, Loc: IDLocation);
748 }
749
750 SrcLocOffset++;
751 SmallVector<IdentifierLoc, 2> Path;
752 IdentifierInfo *ModuleID = PP.getIdentifierInfo(Name: ModuleName);
753 Path.emplace_back(Args&: IDLocation, Args&: ModuleID);
754 auto ModResult = CI.loadModule(ImportLoc: IDLocation, Path, Visibility: Module::Hidden, IsInclusionDirective: false);
755
756 assert(CB && "Must have PPCallbacks after module loading");
757 CB->moduleImport(ImportLoc: SourceLocation(), Path, Imported: ModResult);
758 // Note that we are calling the CB's EndOfMainFile function, which
759 // forwards the results to the dependency consumer.
760 // It does not indicate the end of processing the fake file.
761 CB->EndOfMainFile();
762
763 if (!ModResult)
764 return false;
765
766 CompilerInvocation ModuleInvocation(*OriginalInvocation);
767 MDC->applyDiscoveredDependencies(CI&: ModuleInvocation);
768 Consumer.handleBuildCommand(
769 Cmd: {.Executable: CommandLine[0], .Arguments: ModuleInvocation.getCC1CommandLine()});
770
771 return true;
772}
773
774bool CompilerInstanceWithContext::finalize() {
775 DiagConsumer->finish();
776 return true;
777}
778