1//===- ModuleDepCollector.cpp - Callbacks to collect deps -------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "clang/DependencyScanning/ModuleDepCollector.h"
10
11#include "clang/Basic/MakeSupport.h"
12#include "clang/DependencyScanning/DependencyActionController.h"
13#include "clang/DependencyScanning/DependencyConsumer.h"
14#include "clang/DependencyScanning/DependencyScanningWorker.h"
15#include "clang/Frontend/CompilerInstance.h"
16#include "clang/Lex/Preprocessor.h"
17#include "llvm/ADT/STLExtras.h"
18#include "llvm/Support/BLAKE3.h"
19#include <optional>
20
21using namespace clang;
22using namespace dependencies;
23
24static PrebuiltModuleDep
25createPrebuiltModuleDep(const serialization::ModuleFile *MF) {
26 PrebuiltModuleDep Dep;
27 Dep.ModuleName = MF->ModuleName;
28 Dep.PCMFile = MF->FileName.str();
29 Dep.ModuleMapFile = MF->ModuleMapPath;
30 return Dep;
31}
32
33void PrebuiltModuleASTAttrs::updateDependentsNotInStableDirs(
34 PrebuiltModulesAttrsMap &PrebuiltModulesMap) {
35 setInStableDir();
36 for (const auto Dep : ModuleFileDependents) {
37 if (!PrebuiltModulesMap[Dep].isInStableDir())
38 return;
39 PrebuiltModulesMap[Dep].updateDependentsNotInStableDirs(PrebuiltModulesMap);
40 }
41}
42
43static void
44optimizeHeaderSearchOpts(HeaderSearchOptions &Opts, ASTReader &Reader,
45 const serialization::ModuleFile &MF,
46 const PrebuiltModulesAttrsMap &PrebuiltModulesASTMap,
47 ScanningOptimizations OptimizeArgs) {
48 if (any(Val: OptimizeArgs & ScanningOptimizations::HeaderSearch)) {
49 // Only preserve search paths that were used during the dependency scan.
50 std::vector<HeaderSearchOptions::Entry> Entries;
51 std::swap(x&: Opts.UserEntries, y&: Entries);
52
53 llvm::BitVector SearchPathUsage(Entries.size());
54 llvm::DenseSet<const serialization::ModuleFile *> Visited;
55 std::function<void(const serialization::ModuleFile *)> VisitMF =
56 [&](const serialization::ModuleFile *MF) {
57 SearchPathUsage |= MF->SearchPathUsage;
58 Visited.insert(V: MF);
59 for (const serialization::ModuleFile *Import : MF->Imports)
60 if (!Visited.contains(V: Import))
61 VisitMF(Import);
62 };
63 VisitMF(&MF);
64
65 if (SearchPathUsage.size() != Entries.size())
66 llvm::report_fatal_error(
67 reason: "Inconsistent search path options between modules detected");
68
69 for (auto Idx : SearchPathUsage.set_bits())
70 Opts.UserEntries.push_back(x: std::move(Entries[Idx]));
71 }
72 if (any(Val: OptimizeArgs & ScanningOptimizations::VFS)) {
73 std::vector<std::string> VFSOverlayFiles;
74 std::swap(x&: Opts.VFSOverlayFiles, y&: VFSOverlayFiles);
75
76 llvm::BitVector VFSUsage(VFSOverlayFiles.size());
77 llvm::DenseSet<const serialization::ModuleFile *> Visited;
78 std::function<void(const serialization::ModuleFile *)> VisitMF =
79 [&](const serialization::ModuleFile *MF) {
80 Visited.insert(V: MF);
81 if (MF->Kind == serialization::MK_ImplicitModule) {
82 VFSUsage |= MF->VFSUsage;
83 // We only need to recurse into implicit modules. Other module types
84 // will have the correct set of VFSs for anything they depend on.
85 for (const serialization::ModuleFile *Import : MF->Imports)
86 if (!Visited.contains(V: Import))
87 VisitMF(Import);
88 } else {
89 // This is not an implicitly built module, so it may have different
90 // VFS options. Fall back to a string comparison instead.
91 auto PrebuiltModulePropIt =
92 PrebuiltModulesASTMap.find(Key: MF->FileName);
93 if (PrebuiltModulePropIt == PrebuiltModulesASTMap.end())
94 return;
95 for (std::size_t I = 0, E = VFSOverlayFiles.size(); I != E; ++I) {
96 if (PrebuiltModulePropIt->second.getVFS().contains(
97 key: VFSOverlayFiles[I]))
98 VFSUsage[I] = true;
99 }
100 }
101 };
102 VisitMF(&MF);
103
104 if (VFSUsage.size() != VFSOverlayFiles.size())
105 llvm::report_fatal_error(
106 reason: "Inconsistent -ivfsoverlay options between modules detected");
107
108 for (auto Idx : VFSUsage.set_bits())
109 Opts.VFSOverlayFiles.push_back(x: std::move(VFSOverlayFiles[Idx]));
110 }
111}
112
113static void optimizeDiagnosticOpts(DiagnosticOptions &Opts,
114 bool IsSystemModule) {
115 // If this is not a system module or -Wsystem-headers was passed, don't
116 // optimize.
117 if (!IsSystemModule)
118 return;
119 bool Wsystem_headers = false;
120 for (StringRef Opt : Opts.Warnings) {
121 bool isPositive = !Opt.consume_front(Prefix: "no-");
122 if (Opt == "system-headers")
123 Wsystem_headers = isPositive;
124 }
125 if (Wsystem_headers)
126 return;
127
128 // Remove all warning flags. System modules suppress most, but not all,
129 // warnings.
130 Opts.Warnings.clear();
131 Opts.UndefPrefixes.clear();
132 Opts.Remarks.clear();
133}
134
135static void optimizeCWD(CowCompilerInvocation &BuildInvocation, StringRef CWD) {
136 BuildInvocation.getMutFileSystemOpts().WorkingDir.clear();
137 BuildInvocation.getMutCodeGenOpts().DebugCompilationDir.clear();
138 BuildInvocation.getMutCodeGenOpts().CoverageCompilationDir.clear();
139}
140
141static std::vector<std::string> splitString(std::string S, char Separator) {
142 SmallVector<StringRef> Segments;
143 StringRef(S).split(A&: Segments, Separator, /*MaxSplit=*/-1, /*KeepEmpty=*/false);
144 std::vector<std::string> Result;
145 Result.reserve(n: Segments.size());
146 for (StringRef Segment : Segments)
147 Result.push_back(x: Segment.str());
148 return Result;
149}
150
151void ModuleDepCollector::addOutputPaths(CowCompilerInvocation &CI,
152 ModuleDeps &Deps) {
153 CI.getMutFrontendOpts().OutputFile =
154 Controller.lookupModuleOutput(MD: Deps, Kind: ModuleOutputKind::ModuleFile);
155 if (!CI.getDiagnosticOpts().DiagnosticSerializationFile.empty())
156 CI.getMutDiagnosticOpts().DiagnosticSerializationFile =
157 Controller.lookupModuleOutput(
158 MD: Deps, Kind: ModuleOutputKind::DiagnosticSerializationFile);
159 if (!CI.getDependencyOutputOpts().OutputFile.empty()) {
160 CI.getMutDependencyOutputOpts().OutputFile =
161 Controller.lookupModuleOutput(MD: Deps, Kind: ModuleOutputKind::DependencyFile);
162 CI.getMutDependencyOutputOpts().Targets =
163 splitString(S: Controller.lookupModuleOutput(
164 MD: Deps, Kind: ModuleOutputKind::DependencyTargets),
165 Separator: '\0');
166 if (!CI.getDependencyOutputOpts().OutputFile.empty() &&
167 CI.getDependencyOutputOpts().Targets.empty()) {
168 // Fallback to -o as dependency target, as in the driver.
169 SmallString<128> Target;
170 quoteMakeTarget(Target: CI.getFrontendOpts().OutputFile, Res&: Target);
171 CI.getMutDependencyOutputOpts().Targets.push_back(x: std::string(Target));
172 }
173 }
174}
175
176void dependencies::resetBenignCodeGenOptions(frontend::ActionKind ProgramAction,
177 const LangOptions &LangOpts,
178 CodeGenOptions &CGOpts) {
179 // TODO: Figure out better way to set options to their default value.
180 if (ProgramAction == frontend::GenerateModule) {
181 CGOpts.MainFileName.clear();
182 CGOpts.DwarfDebugFlags.clear();
183 }
184 if (ProgramAction == frontend::GeneratePCH ||
185 (ProgramAction == frontend::GenerateModule && !LangOpts.ModulesCodegen)) {
186 CGOpts.DebugCompilationDir.clear();
187 CGOpts.CoverageCompilationDir.clear();
188 CGOpts.CoverageDataFile.clear();
189 CGOpts.CoverageNotesFile.clear();
190 CGOpts.ProfileInstrumentUsePath.clear();
191 CGOpts.SampleProfileFile.clear();
192 CGOpts.ProfileRemappingFile.clear();
193 }
194}
195
196bool dependencies::isPathInStableDir(const ArrayRef<StringRef> Directories,
197 const StringRef Input) {
198 using namespace llvm::sys;
199
200 if (!path::is_absolute(path: Input))
201 return false;
202
203 auto PathStartsWith = [](StringRef Prefix, StringRef Path) {
204 auto PrefixIt = path::begin(path: Prefix), PrefixEnd = path::end(path: Prefix);
205 for (auto PathIt = path::begin(path: Path), PathEnd = path::end(path: Path);
206 PrefixIt != PrefixEnd && PathIt != PathEnd; ++PrefixIt, ++PathIt) {
207 if (*PrefixIt != *PathIt)
208 return false;
209 }
210 return PrefixIt == PrefixEnd;
211 };
212
213 return any_of(Range: Directories, P: [&](StringRef Dir) {
214 return !Dir.empty() && PathStartsWith(Dir, Input);
215 });
216}
217
218bool dependencies::areOptionsInStableDir(const ArrayRef<StringRef> Directories,
219 const HeaderSearchOptions &HSOpts) {
220 assert(isPathInStableDir(Directories, HSOpts.Sysroot) &&
221 "Sysroots differ between module dependencies and current TU");
222
223 assert(isPathInStableDir(Directories, HSOpts.ResourceDir) &&
224 "ResourceDirs differ between module dependencies and current TU");
225
226 for (const auto &Entry : HSOpts.UserEntries) {
227 if (!Entry.IgnoreSysRoot)
228 continue;
229 if (!isPathInStableDir(Directories, Input: Entry.Path))
230 return false;
231 }
232
233 for (const auto &SysPrefix : HSOpts.SystemHeaderPrefixes) {
234 if (!isPathInStableDir(Directories, Input: SysPrefix.Prefix))
235 return false;
236 }
237
238 return true;
239}
240
241static CowCompilerInvocation
242makeCommonInvocationForModuleBuild(CompilerInvocation CI) {
243 CI.resetNonModularOptions();
244 CI.clearImplicitModuleBuildOptions();
245
246 // The scanner takes care to avoid passing non-affecting module maps to the
247 // explicit compiles. No need to do extra work just to find out there are no
248 // module map files to prune.
249 CI.getHeaderSearchOpts().ModulesPruneNonAffectingModuleMaps = false;
250
251 // Remove options incompatible with explicit module build or are likely to
252 // differ between identical modules discovered from different translation
253 // units.
254 CI.getFrontendOpts().Inputs.clear();
255 CI.getFrontendOpts().OutputFile.clear();
256 CI.getFrontendOpts().GenReducedBMI = false;
257 CI.getFrontendOpts().ModuleOutputPath.clear();
258 CI.getHeaderSearchOpts().ModulesSkipHeaderSearchPaths = false;
259 CI.getHeaderSearchOpts().ModulesSkipDiagnosticOptions = false;
260 // LLVM options are not going to affect the AST
261 CI.getFrontendOpts().LLVMArgs.clear();
262
263 resetBenignCodeGenOptions(ProgramAction: frontend::GenerateModule, LangOpts: CI.getLangOpts(),
264 CGOpts&: CI.getCodeGenOpts());
265
266 // Erase the command-line arguments. These don't make it into the final set of
267 // compilation arguments and will be re-populated during compilation itself.
268 // Keeping them around would make copies of the invocation expensive.
269 CI.getCodeGenOpts().Argv0 = nullptr;
270 CI.getCodeGenOpts().CommandLineArgs.clear();
271
272 // Map output paths that affect behaviour to "-" so their existence is in the
273 // context hash. The final path will be computed in addOutputPaths.
274 if (!CI.getDiagnosticOpts().DiagnosticSerializationFile.empty())
275 CI.getDiagnosticOpts().DiagnosticSerializationFile = "-";
276 if (!CI.getDependencyOutputOpts().OutputFile.empty())
277 CI.getDependencyOutputOpts().OutputFile = "-";
278 CI.getDependencyOutputOpts().Targets.clear();
279 CI.getDependencyOutputOpts().IncludeModuleFiles = MFDK_Direct;
280
281 CI.getFrontendOpts().ProgramAction = frontend::GenerateModule;
282 CI.getLangOpts().ModuleName.clear();
283
284 // Remove any macro definitions that are explicitly ignored.
285 if (!CI.getHeaderSearchOpts().ModulesIgnoreMacros.empty()) {
286 llvm::erase_if(
287 C&: CI.getPreprocessorOpts().Macros,
288 P: [&CI](const std::pair<std::string, bool> &Def) {
289 StringRef MacroDef = Def.first;
290 return CI.getHeaderSearchOpts().ModulesIgnoreMacros.contains(
291 key: llvm::CachedHashString(MacroDef.split(Separator: '=').first));
292 });
293 // Remove the now unused option.
294 CI.getHeaderSearchOpts().ModulesIgnoreMacros.clear();
295 }
296
297 // Remove any header search paths that are explicitly ignored.
298 if (!CI.getHeaderSearchOpts().ModulesIgnoreSearchPaths.empty()) {
299 llvm::erase_if(
300 C&: CI.getHeaderSearchOpts().UserEntries,
301 P: [&CI](const HeaderSearchOptions::Entry &E) {
302 return CI.getHeaderSearchOpts().ModulesIgnoreSearchPaths.contains(
303 key: llvm::CachedHashString(E.Path));
304 });
305 // Remove the now unused option.
306 CI.getHeaderSearchOpts().ModulesIgnoreSearchPaths.clear();
307 }
308
309 return CI;
310}
311
312CowCompilerInvocation
313ModuleDepCollector::getInvocationAdjustedForModuleBuildWithoutOutputs(
314 const ModuleDeps &Deps,
315 llvm::function_ref<void(CowCompilerInvocation &)> Optimize) const {
316 CowCompilerInvocation CI = CommonInvocation;
317
318 CI.getMutLangOpts().ModuleName = Deps.ID.ModuleName;
319 CI.getMutFrontendOpts().IsSystemModule = Deps.IsSystem;
320
321 // Inputs
322 InputKind ModuleMapInputKind(CI.getFrontendOpts().DashX.getLanguage(),
323 InputKind::Format::ModuleMap);
324 CI.getMutFrontendOpts().Inputs.emplace_back(Args: Deps.ClangModuleMapFile,
325 Args&: ModuleMapInputKind);
326
327 auto CurrentModuleMapEntry =
328 ScanInstance.getFileManager().getOptionalFileRef(Filename: Deps.ClangModuleMapFile);
329 assert(CurrentModuleMapEntry && "module map file entry not found");
330
331 // Remove directly passed modulemap files. They will get added back if they
332 // were actually used.
333 CI.getMutFrontendOpts().ModuleMapFiles.clear();
334
335 auto DepModuleMapFiles = collectModuleMapFiles(ClangModuleDeps: Deps.ClangModuleDeps);
336 for (StringRef ModuleMapFile : Deps.ModuleMapFileDeps) {
337 // TODO: Track these as `FileEntryRef` to simplify the equality check below.
338 auto ModuleMapEntry =
339 ScanInstance.getFileManager().getOptionalFileRef(Filename: ModuleMapFile);
340 assert(ModuleMapEntry && "module map file entry not found");
341
342 // Don't report module maps describing eagerly-loaded dependency. This
343 // information will be deserialized from the PCM.
344 // TODO: Verify this works fine when modulemap for module A is eagerly
345 // loaded from A.pcm, and module map passed on the command line contains
346 // definition of a submodule: "explicit module A.Private { ... }".
347 if (Service.getOpts().EagerLoadModules &&
348 DepModuleMapFiles.contains(V: *ModuleMapEntry))
349 continue;
350
351 // Don't report module map file of the current module unless it also
352 // describes a dependency (for symmetry).
353 if (*ModuleMapEntry == *CurrentModuleMapEntry &&
354 !DepModuleMapFiles.contains(V: *ModuleMapEntry))
355 continue;
356
357 CI.getMutFrontendOpts().ModuleMapFiles.emplace_back(args&: ModuleMapFile);
358 }
359
360 // Report the prebuilt modules this module uses.
361 for (const auto &PrebuiltModule : Deps.PrebuiltModuleDeps)
362 CI.getMutFrontendOpts().ModuleFiles.push_back(x: PrebuiltModule.PCMFile);
363
364 // Add module file inputs from dependencies.
365 addModuleFiles(CI, ClangModuleDeps: Deps.ClangModuleDeps);
366
367 if (!CI.getDiagnosticOpts().SystemHeaderWarningsModules.empty()) {
368 // Apply -Wsystem-headers-in-module for the current module.
369 if (llvm::is_contained(Range: CI.getDiagnosticOpts().SystemHeaderWarningsModules,
370 Element: Deps.ID.ModuleName))
371 CI.getMutDiagnosticOpts().Warnings.push_back(x: "system-headers");
372 // Remove the now unused option(s).
373 CI.getMutDiagnosticOpts().SystemHeaderWarningsModules.clear();
374 }
375
376 Optimize(CI);
377
378 return CI;
379}
380
381llvm::DenseSet<const FileEntry *> ModuleDepCollector::collectModuleMapFiles(
382 ArrayRef<ModuleID> ClangModuleDeps) const {
383 llvm::DenseSet<const FileEntry *> ModuleMapFiles;
384 for (const ModuleID &MID : ClangModuleDeps) {
385 ModuleDeps *MD = ModuleDepsByID.lookup(Val: MID);
386 assert(MD && "Inconsistent dependency info");
387 // TODO: Track ClangModuleMapFile as `FileEntryRef`.
388 auto FE = ScanInstance.getFileManager().getOptionalFileRef(
389 Filename: MD->ClangModuleMapFile);
390 assert(FE && "Missing module map file that was previously found");
391 ModuleMapFiles.insert(V: *FE);
392 }
393 return ModuleMapFiles;
394}
395
396void ModuleDepCollector::addModuleMapFiles(
397 CompilerInvocation &CI, ArrayRef<ModuleID> ClangModuleDeps) const {
398 if (Service.getOpts().EagerLoadModules)
399 return; // Only pcm is needed for eager load.
400
401 for (const ModuleID &MID : ClangModuleDeps) {
402 ModuleDeps *MD = ModuleDepsByID.lookup(Val: MID);
403 assert(MD && "Inconsistent dependency info");
404 CI.getFrontendOpts().ModuleMapFiles.push_back(x: MD->ClangModuleMapFile);
405 }
406}
407
408void ModuleDepCollector::addModuleFiles(
409 CompilerInvocation &CI, ArrayRef<ModuleID> ClangModuleDeps) const {
410 for (const ModuleID &MID : ClangModuleDeps) {
411 ModuleDeps *MD = ModuleDepsByID.lookup(Val: MID);
412 std::string PCMPath =
413 Controller.lookupModuleOutput(MD: *MD, Kind: ModuleOutputKind::ModuleFile);
414
415 if (Service.getOpts().EagerLoadModules)
416 CI.getFrontendOpts().ModuleFiles.push_back(x: std::move(PCMPath));
417 else
418 CI.getHeaderSearchOpts().PrebuiltModuleFiles.insert(
419 x: {MID.ModuleName, std::move(PCMPath)});
420 }
421}
422
423void ModuleDepCollector::addModuleFiles(
424 CowCompilerInvocation &CI, ArrayRef<ModuleID> ClangModuleDeps) const {
425 for (const ModuleID &MID : ClangModuleDeps) {
426 ModuleDeps *MD = ModuleDepsByID.lookup(Val: MID);
427 std::string PCMPath =
428 Controller.lookupModuleOutput(MD: *MD, Kind: ModuleOutputKind::ModuleFile);
429
430 if (Service.getOpts().EagerLoadModules)
431 CI.getMutFrontendOpts().ModuleFiles.push_back(x: std::move(PCMPath));
432 else
433 CI.getMutHeaderSearchOpts().PrebuiltModuleFiles.insert(
434 x: {MID.ModuleName, std::move(PCMPath)});
435 }
436}
437
438static bool needsModules(FrontendInputFile FIF) {
439 switch (FIF.getKind().getLanguage()) {
440 case Language::Unknown:
441 case Language::Asm:
442 case Language::LLVM_IR:
443 return false;
444 default:
445 return true;
446 }
447}
448
449void ModuleDepCollector::applyDiscoveredDependencies(CompilerInvocation &CI) {
450 CI.clearImplicitModuleBuildOptions();
451 resetBenignCodeGenOptions(ProgramAction: CI.getFrontendOpts().ProgramAction,
452 LangOpts: CI.getLangOpts(), CGOpts&: CI.getCodeGenOpts());
453 CI.getDependencyOutputOpts().IncludeModuleFiles = MFDK_Direct;
454
455 if (llvm::any_of(Range&: CI.getFrontendOpts().Inputs, P: needsModules)) {
456 Preprocessor &PP = ScanInstance.getPreprocessor();
457 if (Module *CurrentModule = PP.getCurrentModuleImplementation())
458 if (OptionalFileEntryRef CurrentModuleMap =
459 PP.getHeaderSearchInfo()
460 .getModuleMap()
461 .getModuleMapFileForUniquing(M: CurrentModule))
462 CI.getFrontendOpts().ModuleMapFiles.emplace_back(
463 args: CurrentModuleMap->getNameAsRequested());
464
465 SmallVector<ModuleID> DirectDeps;
466 for (const auto &KV : ModularDeps)
467 if (DirectModularDeps.contains(key: KV.first))
468 DirectDeps.push_back(Elt: KV.second->ID);
469
470 // TODO: Report module maps the same way it's done for modular dependencies.
471 addModuleMapFiles(CI, ClangModuleDeps: DirectDeps);
472
473 addModuleFiles(CI, ClangModuleDeps: DirectDeps);
474
475 for (const auto &KV : DirectPrebuiltModularDeps)
476 CI.getFrontendOpts().ModuleFiles.push_back(x: KV.second.PCMFile);
477 }
478}
479
480static bool isSafeToIgnoreCWD(const CowCompilerInvocation &CI) {
481 // Check if the command line input uses relative paths.
482 // It is not safe to ignore the current working directory if any of the
483 // command line inputs use relative paths.
484 bool AnyRelative = false;
485 CI.visitPaths(Cb: [&](StringRef Path) {
486 assert(!AnyRelative && "Continuing path visitation despite relative path");
487 AnyRelative |= !Path.empty() && !llvm::sys::path::is_absolute(path: Path);
488 return CowCompilerInvocation::VisitConstResult{/*Terminate=*/AnyRelative};
489 });
490 return !AnyRelative;
491}
492
493static std::string getModuleContextHash(const ModuleDeps &MD,
494 const CowCompilerInvocation &CI,
495 bool EagerLoadModules,
496 llvm::vfs::FileSystem &VFS) {
497 llvm::HashBuilder<llvm::TruncatedBLAKE3<16>, llvm::endianness::native>
498 HashBuilder;
499
500 // Hash the compiler version and serialization version to ensure the module
501 // will be readable.
502 HashBuilder.add(Value: getClangFullRepositoryVersion());
503 HashBuilder.add(Args: serialization::VERSION_MAJOR, Args: serialization::VERSION_MINOR);
504 llvm::ErrorOr<std::string> CWD = VFS.getCurrentWorkingDirectory();
505 if (CWD && !MD.IgnoreCWD)
506 HashBuilder.add(Value: *CWD);
507
508 // Hash the BuildInvocation without any input files.
509 SmallString<0> ArgVec;
510 ArgVec.reserve(N: 4096);
511 CI.generateCC1CommandLine(Consumer: [&](const Twine &Arg) {
512 Arg.toVector(Out&: ArgVec);
513 ArgVec.push_back(Elt: '\0');
514 });
515 HashBuilder.add(Value: ArgVec);
516
517 // Hash the module dependencies. These paths may differ even if the invocation
518 // is identical if they depend on the contents of the files in the TU -- for
519 // example, case-insensitive paths to modulemap files. Usually such a case
520 // would indicate a missed optimization to canonicalize, but it may be
521 // difficult to canonicalize all cases when there is a VFS.
522 for (const auto &ID : MD.ClangModuleDeps) {
523 HashBuilder.add(Value: ID.ModuleName);
524 HashBuilder.add(Value: ID.ContextHash);
525 }
526
527 HashBuilder.add(Value: EagerLoadModules);
528
529 llvm::BLAKE3Result<16> Hash = HashBuilder.final();
530 std::array<uint64_t, 2> Words;
531 static_assert(sizeof(Hash) == sizeof(Words), "Hash must match Words");
532 std::memcpy(dest: Words.data(), src: Hash.data(), n: sizeof(Hash));
533 return toString(I: llvm::APInt(sizeof(Words) * 8, Words), Radix: 36, /*Signed=*/false);
534}
535
536void ModuleDepCollector::associateWithContextHash(
537 const CowCompilerInvocation &CI, ModuleDeps &Deps) {
538 Deps.ID.ContextHash =
539 getModuleContextHash(MD: Deps, CI, EagerLoadModules: Service.getOpts().EagerLoadModules,
540 VFS&: ScanInstance.getVirtualFileSystem());
541 bool Inserted = ModuleDepsByID.insert(KV: {Deps.ID, &Deps}).second;
542 (void)Inserted;
543 assert(Inserted && "duplicate module mapping");
544}
545
546/// Callback that records textual includes and direct modular includes/imports
547/// during preprocessing.
548class ModuleDepCollector::ModuleDepCollectorPP final : public PPCallbacks {
549 /// The parent dependency collector.
550 ModuleDepCollector &MDC;
551
552public:
553 ModuleDepCollectorPP(ModuleDepCollector &MDC) : MDC(MDC) {}
554
555 void LexedFileChanged(FileID FID, LexedFileChangeReason Reason,
556 SrcMgr::CharacteristicKind FileType, FileID PrevFID,
557 SourceLocation Loc) override {
558 if (Reason != LexedFileChangeReason::EnterFile)
559 return;
560
561 SourceManager &SM = MDC.ScanInstance.getSourceManager();
562
563 // Dependency generation really does want to go all the way to the
564 // file entry for a source location to find out what is depended on.
565 // We do not want #line markers to affect dependency generation!
566 if (std::optional<StringRef> Filename = SM.getNonBuiltinFilenameForID(FID))
567 MDC.addFileDep(Path: llvm::sys::path::remove_leading_dotslash(path: *Filename));
568 }
569
570 void HasInclude(SourceLocation Loc, StringRef FileName, bool IsAngled,
571 OptionalFileEntryRef File,
572 SrcMgr::CharacteristicKind FileType) override {
573 if (File)
574 MDC.addFileDep(Path: File->getName());
575 }
576
577 void InclusionDirective(SourceLocation HashLoc, const Token &IncludeTok,
578 StringRef FileName, bool IsAngled,
579 CharSourceRange FilenameRange,
580 OptionalFileEntryRef File, StringRef SearchPath,
581 StringRef RelativePath, const Module *SuggestedModule,
582 bool ModuleImported,
583 SrcMgr::CharacteristicKind FileType) override {
584 if (!File && !ModuleImported) {
585 // This is a non-modular include that HeaderSearch failed to find. Add it
586 // here as `FileChanged` will never see it.
587 MDC.addFileDep(Path: FileName);
588 }
589 if (ModuleImported && SuggestedModule &&
590 MDC.ScanInstance.getPreprocessorOpts()
591 .DependencyScanningModuleMapImports)
592 MDC.addRequiredStdCXXModule(ModuleName: SuggestedModule->getFullModuleName());
593 MDC.handleImport(Imported: SuggestedModule);
594 }
595
596 void moduleImport(SourceLocation ImportLoc, ModuleIdPath Path,
597 const Module *Imported) override {
598 auto &PP = MDC.ScanInstance.getPreprocessor();
599 if (PP.getLangOpts().CPlusPlusModules && PP.isImportingCXXNamedModules()) {
600 MDC.addRequiredStdCXXModule(ModuleName: Path[0].getIdentifierInfo()->getName());
601 return;
602 }
603
604 MDC.handleImport(Imported);
605 }
606};
607
608void ModuleDepCollector::addRequiredStdCXXModule(StringRef ModuleName) {
609 if (llvm::any_of(Range&: RequiredStdCXXModules,
610 P: [ModuleName](const P1689ModuleInfo &RequiredModule) {
611 return RequiredModule.ModuleName == ModuleName;
612 }))
613 return;
614
615 P1689ModuleInfo RequiredModule;
616 RequiredModule.ModuleName = ModuleName.str();
617 RequiredModule.Type = P1689ModuleInfo::ModuleType::NamedCXXModule;
618 RequiredStdCXXModules.push_back(x: std::move(RequiredModule));
619}
620
621void ModuleDepCollector::handleImport(const Module *Imported) {
622 auto &MDC = *this;
623
624 if (!Imported)
625 return;
626
627 const Module *TopLevelModule = Imported->getTopLevelModule();
628 const ModuleFileKey *MFKey = TopLevelModule->getASTFileKey();
629 if (!MFKey)
630 return;
631 serialization::ModuleFile *MF =
632 MDC.ScanInstance.getASTReader()->getModuleManager().lookup(Key: *MFKey);
633
634 if (MDC.isPrebuiltModule(MF))
635 MDC.DirectPrebuiltModularDeps.insert(KV: {MF, createPrebuiltModuleDep(MF)});
636 else {
637 MDC.DirectModularDeps.insert(X: MF);
638 MDC.DirectImports.insert(Ptr: Imported);
639 }
640}
641
642void ModuleDepCollector::run(DependencyConsumer &Consumer) {
643 auto &MDC = *this;
644
645 FileID MainFileID = MDC.ScanInstance.getSourceManager().getMainFileID();
646 MDC.MainFile = std::string(MDC.ScanInstance.getSourceManager()
647 .getFileEntryRefForID(FID: MainFileID)
648 ->getName());
649
650 auto &PP = MDC.ScanInstance.getPreprocessor();
651 if (PP.isInNamedModule()) {
652 P1689ModuleInfo ProvidedModule;
653 ProvidedModule.ModuleName = PP.getNamedModuleName();
654 ProvidedModule.Type = P1689ModuleInfo::ModuleType::NamedCXXModule;
655 ProvidedModule.IsStdCXXModuleInterface = PP.isInNamedInterfaceUnit();
656 // Don't put implementation (non partition) unit as Provide.
657 // Put the module as required instead. Since the implementation
658 // unit will import the primary module implicitly.
659 if (PP.isInImplementationUnit())
660 MDC.addRequiredStdCXXModule(ModuleName: ProvidedModule.ModuleName);
661 else
662 MDC.ProvidedStdCXXModule = ProvidedModule;
663 }
664
665 if (!MDC.ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty())
666 MDC.addFileDep(Path: MDC.ScanInstance.getPreprocessorOpts().ImplicitPCHInclude);
667
668 for (StringRef VFS : MDC.ScanInstance.getHeaderSearchOpts().VFSOverlayFiles)
669 MDC.addFileDep(Path: VFS);
670
671 if (Module *CurrentModule = PP.getCurrentModuleImplementation()) {
672 if (OptionalFileEntryRef CurrentModuleMap =
673 PP.getHeaderSearchInfo().getModuleMap().getModuleMapFileForUniquing(
674 M: CurrentModule))
675 MDC.addFileDep(Path: CurrentModuleMap->getName());
676 }
677
678 for (const Module *M :
679 MDC.ScanInstance.getPreprocessor().getAffectingClangModules()) {
680 serialization::ModuleFile *MF =
681 MDC.ScanInstance.getASTReader()->getModuleManager().lookup(
682 Key: *M->getASTFileKey());
683 if (!MDC.isPrebuiltModule(MF))
684 MDC.DirectModularDeps.insert(X: MF);
685 }
686
687 if (MDC.Service.getOpts().ReportVisibleModules)
688 MDC.addVisibleModules();
689
690 for (serialization::ModuleFile *MF : MDC.DirectModularDeps)
691 handleTopLevelModule(MF);
692
693 Consumer.handleContextHash(
694 Hash: MDC.ScanInstance.getInvocation().computeContextHash());
695
696 Consumer.handleDependencyOutputOpts(Opts: *MDC.Opts);
697
698 Consumer.handleProvidedAndRequiredStdCXXModules(Provided: MDC.ProvidedStdCXXModule,
699 Requires: MDC.RequiredStdCXXModules);
700
701 for (auto &&I : MDC.ModularDeps)
702 Consumer.handleModuleDependency(MD: *I.second);
703
704 for (serialization::ModuleFile *MF : MDC.DirectModularDeps) {
705 auto It = MDC.ModularDeps.find(Key: MF);
706 // Only report direct dependencies that were successfully handled.
707 if (It != MDC.ModularDeps.end())
708 Consumer.handleDirectModuleDependency(MD: It->second->ID);
709 }
710
711 for (auto &&I : MDC.VisibleModules)
712 Consumer.handleVisibleModule(ModuleName: std::string(I.getKey()));
713
714 for (auto &&I : MDC.FileDeps)
715 Consumer.handleFileDependency(Filename: I);
716
717 for (auto &&I : MDC.DirectPrebuiltModularDeps)
718 Consumer.handlePrebuiltModuleDependency(PMD: I.second);
719}
720
721static StringRef makeAbsoluteAndCanonicalize(CompilerInstance &CI,
722 StringRef Path,
723 SmallVectorImpl<char> &Storage) {
724 // FIXME: Consider skipping if path is already absolute & canonicalized.
725
726 Storage.assign(in_start: Path.begin(), in_end: Path.end());
727 CI.getFileManager().makeAbsolutePath(Path&: Storage, /*Canonicalize=*/true);
728 return StringRef(Storage.data(), Storage.size());
729}
730
731std::optional<ModuleID>
732ModuleDepCollector::handleTopLevelModule(serialization::ModuleFile *MF) {
733 auto &MDC = *this;
734
735 // If this module has been handled already, just return its ID.
736 if (auto ModI = MDC.ModularDeps.find(Key: MF); ModI != MDC.ModularDeps.end())
737 return ModI->second->ID;
738
739 Module *M = MDC.ScanInstance.getPreprocessor()
740 .getHeaderSearchInfo()
741 .getModuleMap()
742 .findModule(Name: MF->ModuleName);
743 assert(M && M == M->getTopLevelModule() &&
744 "ModuleFile without top-level Module");
745
746 auto OwnedMD = std::make_unique<ModuleDeps>();
747 ModuleDeps &MD = *OwnedMD;
748
749 MD.ID.ModuleName = MF->ModuleName;
750 MD.IsSystem = M->IsSystem;
751
752 // Start off with the assumption that this module is shareable when there
753 // are stable directories. As more dependencies are discovered, check if those
754 // come from the provided directories.
755 MD.IsInStableDirectories = !MDC.StableDirs.empty();
756
757 // For modules which use export_as link name, the linked product that of the
758 // corresponding export_as-named module.
759 if (!M->UseExportAsModuleLinkName)
760 MD.LinkLibraries = M->LinkLibraries;
761
762 ModuleMap &ModMapInfo =
763 MDC.ScanInstance.getPreprocessor().getHeaderSearchInfo().getModuleMap();
764
765 if (auto ModuleMap = ModMapInfo.getModuleMapFileForUniquing(M)) {
766 SmallString<128> Path = ModuleMap->getNameAsRequested();
767 ModMapInfo.canonicalizeModuleMapPath(Path);
768 MD.ClangModuleMapFile = std::string(Path);
769 }
770
771 llvm::SmallString<256> Storage;
772 MD.FileDepsBaseDir =
773 makeAbsoluteAndCanonicalize(CI&: MDC.ScanInstance, Path: MF->BaseDirectory, Storage);
774 MDC.ScanInstance.getASTReader()->visitInputFileInfos(
775 MF&: *MF, /*IncludeSystem=*/true,
776 Visitor: [&](const serialization::InputFileInfo &IFI, bool IsSystem) {
777 // The __inferred_module.map file is an insignificant implementation
778 // detail of implicitly-built modules. The PCM will also report the
779 // actual on-disk module map file that allowed inferring the module,
780 // which is what we need for building the module explicitly
781 // Let's ignore this file.
782 if (IFI.UnresolvedImportedFilename.ends_with(Suffix: "__inferred_module.map"))
783 return;
784 MDC.addFileDep(MD, Path: IFI.UnresolvedImportedFilename);
785 });
786
787 addAllModuleDeps(MF&: *MF, MD);
788
789 SmallString<0> PathBuf;
790 PathBuf.reserve(N: 256);
791 MDC.ScanInstance.getASTReader()->visitInputFileInfos(
792 MF&: *MF, /*IncludeSystem=*/true,
793 Visitor: [&](const serialization::InputFileInfo &IFI, bool IsSystem) {
794 if (MD.IsInStableDirectories) {
795 auto FullFilePath = ASTReader::ResolveImportedPath(
796 Buf&: PathBuf, Path: IFI.UnresolvedImportedFilename, Prefix: MF->BaseDirectory);
797 MD.IsInStableDirectories =
798 isPathInStableDir(Directories: MDC.StableDirs, Input: *FullFilePath);
799 }
800 if (!(IFI.TopLevel && IFI.ModuleMap))
801 return;
802 if (IFI.UnresolvedImportedFilenameAsRequested.ends_with(
803 Suffix: "__inferred_module.map"))
804 return;
805 auto ResolvedFilenameAsRequested = ASTReader::ResolveImportedPath(
806 Buf&: PathBuf, Path: IFI.UnresolvedImportedFilenameAsRequested,
807 Prefix: MF->BaseDirectory);
808 MD.ModuleMapFileDeps.emplace_back(args: *ResolvedFilenameAsRequested);
809 });
810
811 bool IgnoreCWD = false;
812 CowCompilerInvocation CI =
813 MDC.getInvocationAdjustedForModuleBuildWithoutOutputs(
814 Deps: MD, Optimize: [&](CowCompilerInvocation &BuildInvocation) {
815 if (any(Val: MDC.Service.getOpts().OptimizeArgs &
816 (ScanningOptimizations::HeaderSearch |
817 ScanningOptimizations::VFS)))
818 optimizeHeaderSearchOpts(Opts&: BuildInvocation.getMutHeaderSearchOpts(),
819 Reader&: *MDC.ScanInstance.getASTReader(), MF: *MF,
820 PrebuiltModulesASTMap: MDC.PrebuiltModulesASTMap,
821 OptimizeArgs: MDC.Service.getOpts().OptimizeArgs);
822
823 if (any(Val: MDC.Service.getOpts().OptimizeArgs &
824 ScanningOptimizations::SystemWarnings))
825 optimizeDiagnosticOpts(
826 Opts&: BuildInvocation.getMutDiagnosticOpts(),
827 IsSystemModule: BuildInvocation.getFrontendOpts().IsSystemModule);
828
829 IgnoreCWD = any(Val: MDC.Service.getOpts().OptimizeArgs &
830 ScanningOptimizations::IgnoreCWD) &&
831 isSafeToIgnoreCWD(CI: BuildInvocation);
832 if (IgnoreCWD) {
833 llvm::ErrorOr<std::string> CWD =
834 MDC.ScanInstance.getVirtualFileSystem()
835 .getCurrentWorkingDirectory();
836 if (CWD)
837 optimizeCWD(BuildInvocation, CWD: *CWD);
838 }
839 });
840
841 // FIXME: Propagate errors up.
842 (void)MDC.Controller.finalizeModuleInvocation(ScanInstance&: MDC.ScanInstance, CI, MD);
843
844 // Check provided input paths from the invocation for determining
845 // IsInStableDirectories.
846 if (MD.IsInStableDirectories)
847 MD.IsInStableDirectories =
848 areOptionsInStableDir(Directories: MDC.StableDirs, HSOpts: CI.getHeaderSearchOpts());
849
850 MD.IgnoreCWD = IgnoreCWD;
851 MDC.associateWithContextHash(CI, Deps&: MD);
852
853 // Finish the compiler invocation. Requires dependencies and the context hash.
854 MDC.addOutputPaths(CI, Deps&: MD);
855
856 MD.BuildInfo = std::move(CI);
857
858 MDC.ModularDeps.insert(KV: {MF, std::move(OwnedMD)});
859
860 return MD.ID;
861}
862
863void ModuleDepCollector::addAllModuleDeps(serialization::ModuleFile &MF,
864 ModuleDeps &MD) {
865 auto &MDC = *this;
866
867 llvm::DenseSet<const Module *> Seen;
868 for (serialization::ModuleFile *Import : MF.Imports) {
869 if (MDC.isPrebuiltModule(MF: Import)) {
870 MD.PrebuiltModuleDeps.push_back(x: createPrebuiltModuleDep(MF: Import));
871 if (MD.IsInStableDirectories) {
872 auto It = MDC.PrebuiltModulesASTMap.find(
873 Key: MD.PrebuiltModuleDeps.back().PCMFile);
874 MD.IsInStableDirectories =
875 It != MDC.PrebuiltModulesASTMap.end() && It->second.isInStableDir();
876 }
877 } else {
878 if (auto ID = handleTopLevelModule(MF: Import)) {
879 MD.ClangModuleDeps.push_back(x: std::move(*ID));
880 if (MD.IsInStableDirectories)
881 MD.IsInStableDirectories =
882 MDC.ModularDeps[Import]->IsInStableDirectories;
883 }
884 }
885 }
886}
887
888ModuleDepCollector::ModuleDepCollector(
889 DependencyScanningService &Service,
890 std::unique_ptr<DependencyOutputOptions> Opts,
891 CompilerInstance &ScanInstance, DependencyActionController &Controller,
892 CompilerInvocation OriginalCI,
893 const PrebuiltModulesAttrsMap PrebuiltModulesASTMap,
894 const ArrayRef<StringRef> StableDirs)
895 : Service(Service), ScanInstance(ScanInstance), Controller(Controller),
896 PrebuiltModulesASTMap(std::move(PrebuiltModulesASTMap)),
897 StableDirs(StableDirs), Opts(std::move(Opts)),
898 CommonInvocation(
899 makeCommonInvocationForModuleBuild(CI: std::move(OriginalCI))) {}
900
901void ModuleDepCollector::attachToPreprocessor(Preprocessor &PP) {
902 auto CollectorPP = std::make_unique<ModuleDepCollectorPP>(args&: *this);
903 CollectorPPPtr = CollectorPP.get();
904 PP.addPPCallbacks(C: std::move(CollectorPP));
905}
906
907void ModuleDepCollector::attachToASTReader(ASTReader &R) {}
908
909bool ModuleDepCollector::isPrebuiltModule(const serialization::ModuleFile *MF) {
910 const auto &PrebuiltModuleFiles =
911 ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles;
912 auto PrebuiltModuleFileIt = PrebuiltModuleFiles.find(x: MF->ModuleName);
913 if (PrebuiltModuleFileIt == PrebuiltModuleFiles.end())
914 return false;
915 assert("Prebuilt module came from the expected AST file" &&
916 PrebuiltModuleFileIt->second == MF->FileName.str());
917 return true;
918}
919
920void ModuleDepCollector::addVisibleModules() {
921 llvm::DenseSet<const Module *> ImportedModules;
922 auto InsertVisibleModules = [&](const Module *M) {
923 if (ImportedModules.contains(V: M))
924 return;
925
926 VisibleModules.insert(key: M->getTopLevelModuleName());
927 SmallVector<Module *> Stack;
928 M->getExportedModules(Exported&: Stack);
929 while (!Stack.empty()) {
930 const Module *CurrModule = Stack.pop_back_val();
931 if (ImportedModules.contains(V: CurrModule))
932 continue;
933 ImportedModules.insert(V: CurrModule);
934 VisibleModules.insert(key: CurrModule->getTopLevelModuleName());
935 CurrModule->getExportedModules(Exported&: Stack);
936 }
937 };
938
939 for (const Module *Import : DirectImports)
940 InsertVisibleModules(Import);
941}
942
943void ModuleDepCollector::addFileDep(StringRef Path) {
944 if (!Service.getOpts().ReportAbsolutePaths) {
945 FileDeps.emplace_back(args&: Path);
946 return;
947 }
948
949 llvm::SmallString<256> Storage;
950 Path = makeAbsoluteAndCanonicalize(CI&: ScanInstance, Path, Storage);
951 FileDeps.emplace_back(args&: Path);
952}
953
954void ModuleDepCollector::addFileDep(ModuleDeps &MD, StringRef Path) {
955 MD.FileDeps.emplace_back(args&: Path);
956}
957