1//===- ModuleDepCollector.cpp - Callbacks to collect deps -------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "clang/DependencyScanning/ModuleDepCollector.h"
10
11#include "clang/Basic/MakeSupport.h"
12#include "clang/DependencyScanning/DependencyScanningWorker.h"
13#include "clang/Frontend/CompilerInstance.h"
14#include "clang/Lex/Preprocessor.h"
15#include "llvm/ADT/STLExtras.h"
16#include "llvm/Support/BLAKE3.h"
17#include <optional>
18
19using namespace clang;
20using namespace dependencies;
21
22void ModuleDeps::forEachFileDep(llvm::function_ref<void(StringRef)> Cb) const {
23 SmallString<0> PathBuf;
24 PathBuf.reserve(N: 256);
25 for (StringRef FileDep : FileDeps) {
26 auto ResolvedFileDep =
27 ASTReader::ResolveImportedPath(Buf&: PathBuf, Path: FileDep, Prefix: FileDepsBaseDir);
28 Cb(*ResolvedFileDep);
29 }
30}
31
32const std::vector<std::string> &ModuleDeps::getBuildArguments() const {
33 // FIXME: this operation is not thread safe and is expected to be called
34 // on a single thread. Otherwise it should be protected with a lock.
35 assert(!std::holds_alternative<std::monostate>(BuildInfo) &&
36 "Using uninitialized ModuleDeps");
37 if (const auto *CI = std::get_if<CowCompilerInvocation>(ptr: &BuildInfo))
38 BuildInfo = CI->getCC1CommandLine();
39 return std::get<std::vector<std::string>>(v&: BuildInfo);
40}
41
42void PrebuiltModuleASTAttrs::updateDependentsNotInStableDirs(
43 PrebuiltModulesAttrsMap &PrebuiltModulesMap) {
44 setInStableDir();
45 for (const auto Dep : ModuleFileDependents) {
46 if (!PrebuiltModulesMap[Dep].isInStableDir())
47 return;
48 PrebuiltModulesMap[Dep].updateDependentsNotInStableDirs(PrebuiltModulesMap);
49 }
50}
51
52static void
53optimizeHeaderSearchOpts(HeaderSearchOptions &Opts, ASTReader &Reader,
54 const serialization::ModuleFile &MF,
55 const PrebuiltModulesAttrsMap &PrebuiltModulesASTMap,
56 ScanningOptimizations OptimizeArgs) {
57 if (any(Val: OptimizeArgs & ScanningOptimizations::HeaderSearch)) {
58 // Only preserve search paths that were used during the dependency scan.
59 std::vector<HeaderSearchOptions::Entry> Entries;
60 std::swap(x&: Opts.UserEntries, y&: Entries);
61
62 llvm::BitVector SearchPathUsage(Entries.size());
63 llvm::DenseSet<const serialization::ModuleFile *> Visited;
64 std::function<void(const serialization::ModuleFile *)> VisitMF =
65 [&](const serialization::ModuleFile *MF) {
66 SearchPathUsage |= MF->SearchPathUsage;
67 Visited.insert(V: MF);
68 for (const serialization::ModuleFile *Import : MF->Imports)
69 if (!Visited.contains(V: Import))
70 VisitMF(Import);
71 };
72 VisitMF(&MF);
73
74 if (SearchPathUsage.size() != Entries.size())
75 llvm::report_fatal_error(
76 reason: "Inconsistent search path options between modules detected");
77
78 for (auto Idx : SearchPathUsage.set_bits())
79 Opts.UserEntries.push_back(x: std::move(Entries[Idx]));
80 }
81 if (any(Val: OptimizeArgs & ScanningOptimizations::VFS)) {
82 std::vector<std::string> VFSOverlayFiles;
83 std::swap(x&: Opts.VFSOverlayFiles, y&: VFSOverlayFiles);
84
85 llvm::BitVector VFSUsage(VFSOverlayFiles.size());
86 llvm::DenseSet<const serialization::ModuleFile *> Visited;
87 std::function<void(const serialization::ModuleFile *)> VisitMF =
88 [&](const serialization::ModuleFile *MF) {
89 Visited.insert(V: MF);
90 if (MF->Kind == serialization::MK_ImplicitModule) {
91 VFSUsage |= MF->VFSUsage;
92 // We only need to recurse into implicit modules. Other module types
93 // will have the correct set of VFSs for anything they depend on.
94 for (const serialization::ModuleFile *Import : MF->Imports)
95 if (!Visited.contains(V: Import))
96 VisitMF(Import);
97 } else {
98 // This is not an implicitly built module, so it may have different
99 // VFS options. Fall back to a string comparison instead.
100 auto PrebuiltModulePropIt =
101 PrebuiltModulesASTMap.find(Key: MF->FileName);
102 if (PrebuiltModulePropIt == PrebuiltModulesASTMap.end())
103 return;
104 for (std::size_t I = 0, E = VFSOverlayFiles.size(); I != E; ++I) {
105 if (PrebuiltModulePropIt->second.getVFS().contains(
106 key: VFSOverlayFiles[I]))
107 VFSUsage[I] = true;
108 }
109 }
110 };
111 VisitMF(&MF);
112
113 if (VFSUsage.size() != VFSOverlayFiles.size())
114 llvm::report_fatal_error(
115 reason: "Inconsistent -ivfsoverlay options between modules detected");
116
117 for (auto Idx : VFSUsage.set_bits())
118 Opts.VFSOverlayFiles.push_back(x: std::move(VFSOverlayFiles[Idx]));
119 }
120}
121
122static void optimizeDiagnosticOpts(DiagnosticOptions &Opts,
123 bool IsSystemModule) {
124 // If this is not a system module or -Wsystem-headers was passed, don't
125 // optimize.
126 if (!IsSystemModule)
127 return;
128 bool Wsystem_headers = false;
129 for (StringRef Opt : Opts.Warnings) {
130 bool isPositive = !Opt.consume_front(Prefix: "no-");
131 if (Opt == "system-headers")
132 Wsystem_headers = isPositive;
133 }
134 if (Wsystem_headers)
135 return;
136
137 // Remove all warning flags. System modules suppress most, but not all,
138 // warnings.
139 Opts.Warnings.clear();
140 Opts.UndefPrefixes.clear();
141 Opts.Remarks.clear();
142}
143
144static void optimizeCWD(CowCompilerInvocation &BuildInvocation, StringRef CWD) {
145 BuildInvocation.getMutFileSystemOpts().WorkingDir.clear();
146 BuildInvocation.getMutCodeGenOpts().DebugCompilationDir.clear();
147 BuildInvocation.getMutCodeGenOpts().CoverageCompilationDir.clear();
148}
149
150static std::vector<std::string> splitString(std::string S, char Separator) {
151 SmallVector<StringRef> Segments;
152 StringRef(S).split(A&: Segments, Separator, /*MaxSplit=*/-1, /*KeepEmpty=*/false);
153 std::vector<std::string> Result;
154 Result.reserve(n: Segments.size());
155 for (StringRef Segment : Segments)
156 Result.push_back(x: Segment.str());
157 return Result;
158}
159
160void ModuleDepCollector::addOutputPaths(CowCompilerInvocation &CI,
161 ModuleDeps &Deps) {
162 CI.getMutFrontendOpts().OutputFile =
163 Controller.lookupModuleOutput(MD: Deps, Kind: ModuleOutputKind::ModuleFile);
164 if (!CI.getDiagnosticOpts().DiagnosticSerializationFile.empty())
165 CI.getMutDiagnosticOpts().DiagnosticSerializationFile =
166 Controller.lookupModuleOutput(
167 MD: Deps, Kind: ModuleOutputKind::DiagnosticSerializationFile);
168 if (!CI.getDependencyOutputOpts().OutputFile.empty()) {
169 CI.getMutDependencyOutputOpts().OutputFile =
170 Controller.lookupModuleOutput(MD: Deps, Kind: ModuleOutputKind::DependencyFile);
171 CI.getMutDependencyOutputOpts().Targets =
172 splitString(S: Controller.lookupModuleOutput(
173 MD: Deps, Kind: ModuleOutputKind::DependencyTargets),
174 Separator: '\0');
175 if (!CI.getDependencyOutputOpts().OutputFile.empty() &&
176 CI.getDependencyOutputOpts().Targets.empty()) {
177 // Fallback to -o as dependency target, as in the driver.
178 SmallString<128> Target;
179 quoteMakeTarget(Target: CI.getFrontendOpts().OutputFile, Res&: Target);
180 CI.getMutDependencyOutputOpts().Targets.push_back(x: std::string(Target));
181 }
182 }
183}
184
185void dependencies::resetBenignCodeGenOptions(frontend::ActionKind ProgramAction,
186 const LangOptions &LangOpts,
187 CodeGenOptions &CGOpts) {
188 // TODO: Figure out better way to set options to their default value.
189 if (ProgramAction == frontend::GenerateModule) {
190 CGOpts.MainFileName.clear();
191 CGOpts.DwarfDebugFlags.clear();
192 }
193 if (ProgramAction == frontend::GeneratePCH ||
194 (ProgramAction == frontend::GenerateModule && !LangOpts.ModulesCodegen)) {
195 CGOpts.DebugCompilationDir.clear();
196 CGOpts.CoverageCompilationDir.clear();
197 CGOpts.CoverageDataFile.clear();
198 CGOpts.CoverageNotesFile.clear();
199 CGOpts.ProfileInstrumentUsePath.clear();
200 CGOpts.SampleProfileFile.clear();
201 CGOpts.ProfileRemappingFile.clear();
202 }
203}
204
205bool dependencies::isPathInStableDir(const ArrayRef<StringRef> Directories,
206 const StringRef Input) {
207 using namespace llvm::sys;
208
209 if (!path::is_absolute(path: Input))
210 return false;
211
212 auto PathStartsWith = [](StringRef Prefix, StringRef Path) {
213 auto PrefixIt = path::begin(path: Prefix), PrefixEnd = path::end(path: Prefix);
214 for (auto PathIt = path::begin(path: Path), PathEnd = path::end(path: Path);
215 PrefixIt != PrefixEnd && PathIt != PathEnd; ++PrefixIt, ++PathIt) {
216 if (*PrefixIt != *PathIt)
217 return false;
218 }
219 return PrefixIt == PrefixEnd;
220 };
221
222 return any_of(Range: Directories, P: [&](StringRef Dir) {
223 return !Dir.empty() && PathStartsWith(Dir, Input);
224 });
225}
226
227bool dependencies::areOptionsInStableDir(const ArrayRef<StringRef> Directories,
228 const HeaderSearchOptions &HSOpts) {
229 assert(isPathInStableDir(Directories, HSOpts.Sysroot) &&
230 "Sysroots differ between module dependencies and current TU");
231
232 assert(isPathInStableDir(Directories, HSOpts.ResourceDir) &&
233 "ResourceDirs differ between module dependencies and current TU");
234
235 for (const auto &Entry : HSOpts.UserEntries) {
236 if (!Entry.IgnoreSysRoot)
237 continue;
238 if (!isPathInStableDir(Directories, Input: Entry.Path))
239 return false;
240 }
241
242 for (const auto &SysPrefix : HSOpts.SystemHeaderPrefixes) {
243 if (!isPathInStableDir(Directories, Input: SysPrefix.Prefix))
244 return false;
245 }
246
247 return true;
248}
249
250static CowCompilerInvocation
251makeCommonInvocationForModuleBuild(CompilerInvocation CI) {
252 CI.resetNonModularOptions();
253 CI.clearImplicitModuleBuildOptions();
254
255 // The scanner takes care to avoid passing non-affecting module maps to the
256 // explicit compiles. No need to do extra work just to find out there are no
257 // module map files to prune.
258 CI.getHeaderSearchOpts().ModulesPruneNonAffectingModuleMaps = false;
259
260 // Remove options incompatible with explicit module build or are likely to
261 // differ between identical modules discovered from different translation
262 // units.
263 CI.getFrontendOpts().Inputs.clear();
264 CI.getFrontendOpts().OutputFile.clear();
265 CI.getFrontendOpts().GenReducedBMI = false;
266 CI.getFrontendOpts().ModuleOutputPath.clear();
267 CI.getHeaderSearchOpts().ModulesSkipHeaderSearchPaths = false;
268 CI.getHeaderSearchOpts().ModulesSkipDiagnosticOptions = false;
269 // LLVM options are not going to affect the AST
270 CI.getFrontendOpts().LLVMArgs.clear();
271
272 resetBenignCodeGenOptions(ProgramAction: frontend::GenerateModule, LangOpts: CI.getLangOpts(),
273 CGOpts&: CI.getCodeGenOpts());
274
275 // Map output paths that affect behaviour to "-" so their existence is in the
276 // context hash. The final path will be computed in addOutputPaths.
277 if (!CI.getDiagnosticOpts().DiagnosticSerializationFile.empty())
278 CI.getDiagnosticOpts().DiagnosticSerializationFile = "-";
279 if (!CI.getDependencyOutputOpts().OutputFile.empty())
280 CI.getDependencyOutputOpts().OutputFile = "-";
281 CI.getDependencyOutputOpts().Targets.clear();
282
283 CI.getFrontendOpts().ProgramAction = frontend::GenerateModule;
284 CI.getLangOpts().ModuleName.clear();
285
286 // Remove any macro definitions that are explicitly ignored.
287 if (!CI.getHeaderSearchOpts().ModulesIgnoreMacros.empty()) {
288 llvm::erase_if(
289 C&: CI.getPreprocessorOpts().Macros,
290 P: [&CI](const std::pair<std::string, bool> &Def) {
291 StringRef MacroDef = Def.first;
292 return CI.getHeaderSearchOpts().ModulesIgnoreMacros.contains(
293 key: llvm::CachedHashString(MacroDef.split(Separator: '=').first));
294 });
295 // Remove the now unused option.
296 CI.getHeaderSearchOpts().ModulesIgnoreMacros.clear();
297 }
298
299 return CI;
300}
301
302CowCompilerInvocation
303ModuleDepCollector::getInvocationAdjustedForModuleBuildWithoutOutputs(
304 const ModuleDeps &Deps,
305 llvm::function_ref<void(CowCompilerInvocation &)> Optimize) const {
306 CowCompilerInvocation CI = CommonInvocation;
307
308 CI.getMutLangOpts().ModuleName = Deps.ID.ModuleName;
309 CI.getMutFrontendOpts().IsSystemModule = Deps.IsSystem;
310
311 // Inputs
312 InputKind ModuleMapInputKind(CI.getFrontendOpts().DashX.getLanguage(),
313 InputKind::Format::ModuleMap);
314 CI.getMutFrontendOpts().Inputs.emplace_back(Args: Deps.ClangModuleMapFile,
315 Args&: ModuleMapInputKind);
316
317 auto CurrentModuleMapEntry =
318 ScanInstance.getFileManager().getOptionalFileRef(Filename: Deps.ClangModuleMapFile);
319 assert(CurrentModuleMapEntry && "module map file entry not found");
320
321 // Remove directly passed modulemap files. They will get added back if they
322 // were actually used.
323 CI.getMutFrontendOpts().ModuleMapFiles.clear();
324
325 auto DepModuleMapFiles = collectModuleMapFiles(ClangModuleDeps: Deps.ClangModuleDeps);
326 for (StringRef ModuleMapFile : Deps.ModuleMapFileDeps) {
327 // TODO: Track these as `FileEntryRef` to simplify the equality check below.
328 auto ModuleMapEntry =
329 ScanInstance.getFileManager().getOptionalFileRef(Filename: ModuleMapFile);
330 assert(ModuleMapEntry && "module map file entry not found");
331
332 // Don't report module maps describing eagerly-loaded dependency. This
333 // information will be deserialized from the PCM.
334 // TODO: Verify this works fine when modulemap for module A is eagerly
335 // loaded from A.pcm, and module map passed on the command line contains
336 // definition of a submodule: "explicit module A.Private { ... }".
337 if (Service.getOpts().EagerLoadModules &&
338 DepModuleMapFiles.contains(V: *ModuleMapEntry))
339 continue;
340
341 // Don't report module map file of the current module unless it also
342 // describes a dependency (for symmetry).
343 if (*ModuleMapEntry == *CurrentModuleMapEntry &&
344 !DepModuleMapFiles.contains(V: *ModuleMapEntry))
345 continue;
346
347 CI.getMutFrontendOpts().ModuleMapFiles.emplace_back(args&: ModuleMapFile);
348 }
349
350 // Report the prebuilt modules this module uses.
351 for (const auto &PrebuiltModule : Deps.PrebuiltModuleDeps)
352 CI.getMutFrontendOpts().ModuleFiles.push_back(x: PrebuiltModule.PCMFile);
353
354 // Add module file inputs from dependencies.
355 addModuleFiles(CI, ClangModuleDeps: Deps.ClangModuleDeps);
356
357 if (!CI.getDiagnosticOpts().SystemHeaderWarningsModules.empty()) {
358 // Apply -Wsystem-headers-in-module for the current module.
359 if (llvm::is_contained(Range: CI.getDiagnosticOpts().SystemHeaderWarningsModules,
360 Element: Deps.ID.ModuleName))
361 CI.getMutDiagnosticOpts().Warnings.push_back(x: "system-headers");
362 // Remove the now unused option(s).
363 CI.getMutDiagnosticOpts().SystemHeaderWarningsModules.clear();
364 }
365
366 Optimize(CI);
367
368 return CI;
369}
370
371llvm::DenseSet<const FileEntry *> ModuleDepCollector::collectModuleMapFiles(
372 ArrayRef<ModuleID> ClangModuleDeps) const {
373 llvm::DenseSet<const FileEntry *> ModuleMapFiles;
374 for (const ModuleID &MID : ClangModuleDeps) {
375 ModuleDeps *MD = ModuleDepsByID.lookup(Val: MID);
376 assert(MD && "Inconsistent dependency info");
377 // TODO: Track ClangModuleMapFile as `FileEntryRef`.
378 auto FE = ScanInstance.getFileManager().getOptionalFileRef(
379 Filename: MD->ClangModuleMapFile);
380 assert(FE && "Missing module map file that was previously found");
381 ModuleMapFiles.insert(V: *FE);
382 }
383 return ModuleMapFiles;
384}
385
386void ModuleDepCollector::addModuleMapFiles(
387 CompilerInvocation &CI, ArrayRef<ModuleID> ClangModuleDeps) const {
388 if (Service.getOpts().EagerLoadModules)
389 return; // Only pcm is needed for eager load.
390
391 for (const ModuleID &MID : ClangModuleDeps) {
392 ModuleDeps *MD = ModuleDepsByID.lookup(Val: MID);
393 assert(MD && "Inconsistent dependency info");
394 CI.getFrontendOpts().ModuleMapFiles.push_back(x: MD->ClangModuleMapFile);
395 }
396}
397
398void ModuleDepCollector::addModuleFiles(
399 CompilerInvocation &CI, ArrayRef<ModuleID> ClangModuleDeps) const {
400 for (const ModuleID &MID : ClangModuleDeps) {
401 ModuleDeps *MD = ModuleDepsByID.lookup(Val: MID);
402 std::string PCMPath =
403 Controller.lookupModuleOutput(MD: *MD, Kind: ModuleOutputKind::ModuleFile);
404
405 if (Service.getOpts().EagerLoadModules)
406 CI.getFrontendOpts().ModuleFiles.push_back(x: std::move(PCMPath));
407 else
408 CI.getHeaderSearchOpts().PrebuiltModuleFiles.insert(
409 x: {MID.ModuleName, std::move(PCMPath)});
410 }
411}
412
413void ModuleDepCollector::addModuleFiles(
414 CowCompilerInvocation &CI, ArrayRef<ModuleID> ClangModuleDeps) const {
415 for (const ModuleID &MID : ClangModuleDeps) {
416 ModuleDeps *MD = ModuleDepsByID.lookup(Val: MID);
417 std::string PCMPath =
418 Controller.lookupModuleOutput(MD: *MD, Kind: ModuleOutputKind::ModuleFile);
419
420 if (Service.getOpts().EagerLoadModules)
421 CI.getMutFrontendOpts().ModuleFiles.push_back(x: std::move(PCMPath));
422 else
423 CI.getMutHeaderSearchOpts().PrebuiltModuleFiles.insert(
424 x: {MID.ModuleName, std::move(PCMPath)});
425 }
426}
427
428static bool needsModules(FrontendInputFile FIF) {
429 switch (FIF.getKind().getLanguage()) {
430 case Language::Unknown:
431 case Language::Asm:
432 case Language::LLVM_IR:
433 return false;
434 default:
435 return true;
436 }
437}
438
439void ModuleDepCollector::applyDiscoveredDependencies(CompilerInvocation &CI) {
440 CI.clearImplicitModuleBuildOptions();
441 resetBenignCodeGenOptions(ProgramAction: CI.getFrontendOpts().ProgramAction,
442 LangOpts: CI.getLangOpts(), CGOpts&: CI.getCodeGenOpts());
443
444 if (llvm::any_of(Range&: CI.getFrontendOpts().Inputs, P: needsModules)) {
445 Preprocessor &PP = ScanInstance.getPreprocessor();
446 if (Module *CurrentModule = PP.getCurrentModuleImplementation())
447 if (OptionalFileEntryRef CurrentModuleMap =
448 PP.getHeaderSearchInfo()
449 .getModuleMap()
450 .getModuleMapFileForUniquing(M: CurrentModule))
451 CI.getFrontendOpts().ModuleMapFiles.emplace_back(
452 args: CurrentModuleMap->getNameAsRequested());
453
454 SmallVector<ModuleID> DirectDeps;
455 for (const auto &KV : ModularDeps)
456 if (DirectModularDeps.contains(key: KV.first))
457 DirectDeps.push_back(Elt: KV.second->ID);
458
459 // TODO: Report module maps the same way it's done for modular dependencies.
460 addModuleMapFiles(CI, ClangModuleDeps: DirectDeps);
461
462 addModuleFiles(CI, ClangModuleDeps: DirectDeps);
463
464 for (const auto &KV : DirectPrebuiltModularDeps)
465 CI.getFrontendOpts().ModuleFiles.push_back(x: KV.second.PCMFile);
466 }
467}
468
469static bool isSafeToIgnoreCWD(const CowCompilerInvocation &CI) {
470 // Check if the command line input uses relative paths.
471 // It is not safe to ignore the current working directory if any of the
472 // command line inputs use relative paths.
473 bool AnyRelative = false;
474 CI.visitPaths(Callback: [&](StringRef Path) {
475 assert(!AnyRelative && "Continuing path visitation despite returning true");
476 AnyRelative |= !Path.empty() && !llvm::sys::path::is_absolute(path: Path);
477 return AnyRelative;
478 });
479 return !AnyRelative;
480}
481
482static std::string getModuleContextHash(const ModuleDeps &MD,
483 const CowCompilerInvocation &CI,
484 bool EagerLoadModules, bool IgnoreCWD,
485 llvm::vfs::FileSystem &VFS) {
486 llvm::HashBuilder<llvm::TruncatedBLAKE3<16>, llvm::endianness::native>
487 HashBuilder;
488
489 // Hash the compiler version and serialization version to ensure the module
490 // will be readable.
491 HashBuilder.add(Value: getClangFullRepositoryVersion());
492 HashBuilder.add(Args: serialization::VERSION_MAJOR, Args: serialization::VERSION_MINOR);
493 llvm::ErrorOr<std::string> CWD = VFS.getCurrentWorkingDirectory();
494 if (CWD && !IgnoreCWD)
495 HashBuilder.add(Value: *CWD);
496
497 // Hash the BuildInvocation without any input files.
498 SmallString<0> ArgVec;
499 ArgVec.reserve(N: 4096);
500 CI.generateCC1CommandLine(Consumer: [&](const Twine &Arg) {
501 Arg.toVector(Out&: ArgVec);
502 ArgVec.push_back(Elt: '\0');
503 });
504 HashBuilder.add(Value: ArgVec);
505
506 // Hash the module dependencies. These paths may differ even if the invocation
507 // is identical if they depend on the contents of the files in the TU -- for
508 // example, case-insensitive paths to modulemap files. Usually such a case
509 // would indicate a missed optimization to canonicalize, but it may be
510 // difficult to canonicalize all cases when there is a VFS.
511 for (const auto &ID : MD.ClangModuleDeps) {
512 HashBuilder.add(Value: ID.ModuleName);
513 HashBuilder.add(Value: ID.ContextHash);
514 }
515
516 HashBuilder.add(Value: EagerLoadModules);
517
518 llvm::BLAKE3Result<16> Hash = HashBuilder.final();
519 std::array<uint64_t, 2> Words;
520 static_assert(sizeof(Hash) == sizeof(Words), "Hash must match Words");
521 std::memcpy(dest: Words.data(), src: Hash.data(), n: sizeof(Hash));
522 return toString(I: llvm::APInt(sizeof(Words) * 8, Words), Radix: 36, /*Signed=*/false);
523}
524
525void ModuleDepCollector::associateWithContextHash(
526 const CowCompilerInvocation &CI, bool IgnoreCWD, ModuleDeps &Deps) {
527 Deps.ID.ContextHash =
528 getModuleContextHash(MD: Deps, CI, EagerLoadModules: Service.getOpts().EagerLoadModules,
529 IgnoreCWD, VFS&: ScanInstance.getVirtualFileSystem());
530 bool Inserted = ModuleDepsByID.insert(KV: {Deps.ID, &Deps}).second;
531 (void)Inserted;
532 assert(Inserted && "duplicate module mapping");
533}
534
535void ModuleDepCollectorPP::LexedFileChanged(FileID FID,
536 LexedFileChangeReason Reason,
537 SrcMgr::CharacteristicKind FileType,
538 FileID PrevFID,
539 SourceLocation Loc) {
540 if (Reason != LexedFileChangeReason::EnterFile)
541 return;
542
543 SourceManager &SM = MDC.ScanInstance.getSourceManager();
544
545 // Dependency generation really does want to go all the way to the
546 // file entry for a source location to find out what is depended on.
547 // We do not want #line markers to affect dependency generation!
548 if (std::optional<StringRef> Filename = SM.getNonBuiltinFilenameForID(FID))
549 MDC.addFileDep(Path: llvm::sys::path::remove_leading_dotslash(path: *Filename));
550}
551
552void ModuleDepCollectorPP::InclusionDirective(
553 SourceLocation HashLoc, const Token &IncludeTok, StringRef FileName,
554 bool IsAngled, CharSourceRange FilenameRange, OptionalFileEntryRef File,
555 StringRef SearchPath, StringRef RelativePath, const Module *SuggestedModule,
556 bool ModuleImported, SrcMgr::CharacteristicKind FileType) {
557 if (!File && !ModuleImported) {
558 // This is a non-modular include that HeaderSearch failed to find. Add it
559 // here as `FileChanged` will never see it.
560 MDC.addFileDep(Path: FileName);
561 }
562 handleImport(Imported: SuggestedModule);
563}
564
565void ModuleDepCollectorPP::moduleImport(SourceLocation ImportLoc,
566 ModuleIdPath Path,
567 const Module *Imported) {
568 auto &PP = MDC.ScanInstance.getPreprocessor();
569 if (PP.getLangOpts().CPlusPlusModules && PP.isImportingCXXNamedModules()) {
570 P1689ModuleInfo RequiredModule;
571 RequiredModule.ModuleName = Path[0].getIdentifierInfo()->getName().str();
572 RequiredModule.Type = P1689ModuleInfo::ModuleType::NamedCXXModule;
573 MDC.RequiredStdCXXModules.push_back(x: std::move(RequiredModule));
574 return;
575 }
576
577 handleImport(Imported);
578}
579
580void ModuleDepCollectorPP::handleImport(const Module *Imported) {
581 if (!Imported)
582 return;
583
584 const Module *TopLevelModule = Imported->getTopLevelModule();
585
586 if (MDC.isPrebuiltModule(M: TopLevelModule))
587 MDC.DirectPrebuiltModularDeps.insert(
588 KV: {TopLevelModule, PrebuiltModuleDep{TopLevelModule}});
589 else {
590 MDC.DirectModularDeps.insert(X: TopLevelModule);
591 MDC.DirectImports.insert(Ptr: Imported);
592 }
593}
594
595void ModuleDepCollectorPP::EndOfMainFile() {
596 FileID MainFileID = MDC.ScanInstance.getSourceManager().getMainFileID();
597 MDC.MainFile = std::string(MDC.ScanInstance.getSourceManager()
598 .getFileEntryRefForID(FID: MainFileID)
599 ->getName());
600
601 auto &PP = MDC.ScanInstance.getPreprocessor();
602 if (PP.isInNamedModule()) {
603 P1689ModuleInfo ProvidedModule;
604 ProvidedModule.ModuleName = PP.getNamedModuleName();
605 ProvidedModule.Type = P1689ModuleInfo::ModuleType::NamedCXXModule;
606 ProvidedModule.IsStdCXXModuleInterface = PP.isInNamedInterfaceUnit();
607 // Don't put implementation (non partition) unit as Provide.
608 // Put the module as required instead. Since the implementation
609 // unit will import the primary module implicitly.
610 if (PP.isInImplementationUnit())
611 MDC.RequiredStdCXXModules.push_back(x: ProvidedModule);
612 else
613 MDC.ProvidedStdCXXModule = ProvidedModule;
614 }
615
616 if (!MDC.ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty())
617 MDC.addFileDep(Path: MDC.ScanInstance.getPreprocessorOpts().ImplicitPCHInclude);
618
619 for (StringRef VFS : MDC.ScanInstance.getHeaderSearchOpts().VFSOverlayFiles)
620 MDC.addFileDep(Path: VFS);
621
622 if (Module *CurrentModule = PP.getCurrentModuleImplementation()) {
623 if (OptionalFileEntryRef CurrentModuleMap =
624 PP.getHeaderSearchInfo().getModuleMap().getModuleMapFileForUniquing(
625 M: CurrentModule))
626 MDC.addFileDep(Path: CurrentModuleMap->getName());
627 }
628
629 for (const Module *M :
630 MDC.ScanInstance.getPreprocessor().getAffectingClangModules())
631 if (!MDC.isPrebuiltModule(M))
632 MDC.DirectModularDeps.insert(X: M);
633
634 MDC.addVisibleModules();
635
636 for (const Module *M : MDC.DirectModularDeps)
637 handleTopLevelModule(M);
638
639 MDC.Consumer.handleContextHash(
640 Hash: MDC.ScanInstance.getInvocation().computeContextHash());
641
642 MDC.Consumer.handleDependencyOutputOpts(Opts: *MDC.Opts);
643
644 MDC.Consumer.handleProvidedAndRequiredStdCXXModules(
645 Provided: MDC.ProvidedStdCXXModule, Requires: MDC.RequiredStdCXXModules);
646
647 for (auto &&I : MDC.ModularDeps)
648 MDC.Consumer.handleModuleDependency(MD: *I.second);
649
650 for (const Module *M : MDC.DirectModularDeps) {
651 auto It = MDC.ModularDeps.find(Key: M);
652 // Only report direct dependencies that were successfully handled.
653 if (It != MDC.ModularDeps.end())
654 MDC.Consumer.handleDirectModuleDependency(MD: It->second->ID);
655 }
656
657 for (auto &&I : MDC.VisibleModules)
658 MDC.Consumer.handleVisibleModule(ModuleName: std::string(I.getKey()));
659
660 for (auto &&I : MDC.FileDeps)
661 MDC.Consumer.handleFileDependency(Filename: I);
662
663 for (auto &&I : MDC.DirectPrebuiltModularDeps)
664 MDC.Consumer.handlePrebuiltModuleDependency(PMD: I.second);
665}
666
667static StringRef makeAbsoluteAndCanonicalize(CompilerInstance &CI,
668 StringRef Path,
669 SmallVectorImpl<char> &Storage) {
670 // FIXME: Consider skipping if path is already absolute & canonicalized.
671
672 Storage.assign(in_start: Path.begin(), in_end: Path.end());
673 CI.getFileManager().makeAbsolutePath(Path&: Storage, /*Canonicalize=*/true);
674 return StringRef(Storage.data(), Storage.size());
675}
676
677std::optional<ModuleID>
678ModuleDepCollectorPP::handleTopLevelModule(const Module *M) {
679 assert(M == M->getTopLevelModule() && "Expected top level module!");
680
681 // A top-level module might not be actually imported as a module when
682 // -fmodule-name is used to compile a translation unit that imports this
683 // module. In that case it can be skipped. The appropriate header
684 // dependencies will still be reported as expected.
685 if (!M->getASTFile())
686 return {};
687
688 // If this module has been handled already, just return its ID.
689 if (auto ModI = MDC.ModularDeps.find(Key: M); ModI != MDC.ModularDeps.end())
690 return ModI->second->ID;
691
692 auto OwnedMD = std::make_unique<ModuleDeps>();
693 ModuleDeps &MD = *OwnedMD;
694
695 MD.ID.ModuleName = M->getFullModuleName();
696 MD.IsSystem = M->IsSystem;
697
698 // Start off with the assumption that this module is shareable when there
699 // are stable directories. As more dependencies are discovered, check if those
700 // come from the provided directories.
701 MD.IsInStableDirectories = !MDC.StableDirs.empty();
702
703 // For modules which use export_as link name, the linked product that of the
704 // corresponding export_as-named module.
705 if (!M->UseExportAsModuleLinkName)
706 MD.LinkLibraries = M->LinkLibraries;
707
708 ModuleMap &ModMapInfo =
709 MDC.ScanInstance.getPreprocessor().getHeaderSearchInfo().getModuleMap();
710
711 if (auto ModuleMap = ModMapInfo.getModuleMapFileForUniquing(M)) {
712 SmallString<128> Path = ModuleMap->getNameAsRequested();
713 ModMapInfo.canonicalizeModuleMapPath(Path);
714 MD.ClangModuleMapFile = std::string(Path);
715 }
716
717 serialization::ModuleFile *MF =
718 MDC.ScanInstance.getASTReader()->getModuleManager().lookup(
719 File: *M->getASTFile());
720
721 llvm::SmallString<256> Storage;
722 MD.FileDepsBaseDir =
723 makeAbsoluteAndCanonicalize(CI&: MDC.ScanInstance, Path: MF->BaseDirectory, Storage);
724 MDC.ScanInstance.getASTReader()->visitInputFileInfos(
725 MF&: *MF, /*IncludeSystem=*/true,
726 Visitor: [&](const serialization::InputFileInfo &IFI, bool IsSystem) {
727 // The __inferred_module.map file is an insignificant implementation
728 // detail of implicitly-built modules. The PCM will also report the
729 // actual on-disk module map file that allowed inferring the module,
730 // which is what we need for building the module explicitly
731 // Let's ignore this file.
732 if (IFI.UnresolvedImportedFilename.ends_with(Suffix: "__inferred_module.map"))
733 return;
734 MDC.addFileDep(MD, Path: IFI.UnresolvedImportedFilename);
735 });
736
737 llvm::DenseSet<const Module *> SeenDeps;
738 addAllSubmodulePrebuiltDeps(M, MD, SeenSubmodules&: SeenDeps);
739 addAllSubmoduleDeps(M, MD, AddedModules&: SeenDeps);
740 addAllAffectingClangModules(M, MD, AddedModules&: SeenDeps);
741
742 SmallString<0> PathBuf;
743 PathBuf.reserve(N: 256);
744 MDC.ScanInstance.getASTReader()->visitInputFileInfos(
745 MF&: *MF, /*IncludeSystem=*/true,
746 Visitor: [&](const serialization::InputFileInfo &IFI, bool IsSystem) {
747 if (MD.IsInStableDirectories) {
748 auto FullFilePath = ASTReader::ResolveImportedPath(
749 Buf&: PathBuf, Path: IFI.UnresolvedImportedFilename, Prefix: MF->BaseDirectory);
750 MD.IsInStableDirectories =
751 isPathInStableDir(Directories: MDC.StableDirs, Input: *FullFilePath);
752 }
753 if (!(IFI.TopLevel && IFI.ModuleMap))
754 return;
755 if (IFI.UnresolvedImportedFilenameAsRequested.ends_with(
756 Suffix: "__inferred_module.map"))
757 return;
758 auto ResolvedFilenameAsRequested = ASTReader::ResolveImportedPath(
759 Buf&: PathBuf, Path: IFI.UnresolvedImportedFilenameAsRequested,
760 Prefix: MF->BaseDirectory);
761 MD.ModuleMapFileDeps.emplace_back(args: *ResolvedFilenameAsRequested);
762 });
763
764 bool IgnoreCWD = false;
765 CowCompilerInvocation CI =
766 MDC.getInvocationAdjustedForModuleBuildWithoutOutputs(
767 Deps: MD, Optimize: [&](CowCompilerInvocation &BuildInvocation) {
768 if (any(Val: MDC.Service.getOpts().OptimizeArgs &
769 (ScanningOptimizations::HeaderSearch |
770 ScanningOptimizations::VFS)))
771 optimizeHeaderSearchOpts(Opts&: BuildInvocation.getMutHeaderSearchOpts(),
772 Reader&: *MDC.ScanInstance.getASTReader(), MF: *MF,
773 PrebuiltModulesASTMap: MDC.PrebuiltModulesASTMap,
774 OptimizeArgs: MDC.Service.getOpts().OptimizeArgs);
775
776 if (any(Val: MDC.Service.getOpts().OptimizeArgs &
777 ScanningOptimizations::SystemWarnings))
778 optimizeDiagnosticOpts(
779 Opts&: BuildInvocation.getMutDiagnosticOpts(),
780 IsSystemModule: BuildInvocation.getFrontendOpts().IsSystemModule);
781
782 IgnoreCWD = any(Val: MDC.Service.getOpts().OptimizeArgs &
783 ScanningOptimizations::IgnoreCWD) &&
784 isSafeToIgnoreCWD(CI: BuildInvocation);
785 if (IgnoreCWD) {
786 llvm::ErrorOr<std::string> CWD =
787 MDC.ScanInstance.getVirtualFileSystem()
788 .getCurrentWorkingDirectory();
789 if (CWD)
790 optimizeCWD(BuildInvocation, CWD: *CWD);
791 }
792 });
793
794 // Check provided input paths from the invocation for determining
795 // IsInStableDirectories.
796 if (MD.IsInStableDirectories)
797 MD.IsInStableDirectories =
798 areOptionsInStableDir(Directories: MDC.StableDirs, HSOpts: CI.getHeaderSearchOpts());
799
800 MDC.associateWithContextHash(CI, IgnoreCWD, Deps&: MD);
801
802 // Finish the compiler invocation. Requires dependencies and the context hash.
803 MDC.addOutputPaths(CI, Deps&: MD);
804
805 MD.BuildInfo = std::move(CI);
806
807 MDC.ModularDeps.insert(KV: {M, std::move(OwnedMD)});
808
809 return MD.ID;
810}
811
812static void forEachSubmoduleSorted(const Module *M,
813 llvm::function_ref<void(const Module *)> F) {
814 // Submodule order depends on order of header includes for inferred submodules
815 // we don't care about the exact order, so sort so that it's consistent across
816 // TUs to improve sharing.
817 SmallVector<const Module *> Submodules(M->submodules());
818 llvm::stable_sort(Range&: Submodules, C: [](const Module *A, const Module *B) {
819 return A->Name < B->Name;
820 });
821 for (const Module *SubM : Submodules)
822 F(SubM);
823}
824
825void ModuleDepCollectorPP::addAllSubmodulePrebuiltDeps(
826 const Module *M, ModuleDeps &MD,
827 llvm::DenseSet<const Module *> &SeenSubmodules) {
828 addModulePrebuiltDeps(M, MD, SeenSubmodules);
829
830 forEachSubmoduleSorted(M, F: [&](const Module *SubM) {
831 addAllSubmodulePrebuiltDeps(M: SubM, MD, SeenSubmodules);
832 });
833}
834
835void ModuleDepCollectorPP::addModulePrebuiltDeps(
836 const Module *M, ModuleDeps &MD,
837 llvm::DenseSet<const Module *> &SeenSubmodules) {
838 for (const Module *Import : M->Imports)
839 if (Import->getTopLevelModule() != M->getTopLevelModule())
840 if (MDC.isPrebuiltModule(M: Import->getTopLevelModule()))
841 if (SeenSubmodules.insert(V: Import->getTopLevelModule()).second) {
842 MD.PrebuiltModuleDeps.emplace_back(args: Import->getTopLevelModule());
843 if (MD.IsInStableDirectories) {
844 auto PrebuiltModulePropIt = MDC.PrebuiltModulesASTMap.find(
845 Key: MD.PrebuiltModuleDeps.back().PCMFile);
846 MD.IsInStableDirectories =
847 (PrebuiltModulePropIt != MDC.PrebuiltModulesASTMap.end()) &&
848 PrebuiltModulePropIt->second.isInStableDir();
849 }
850 }
851}
852
853void ModuleDepCollectorPP::addAllSubmoduleDeps(
854 const Module *M, ModuleDeps &MD,
855 llvm::DenseSet<const Module *> &AddedModules) {
856 addModuleDep(M, MD, AddedModules);
857
858 forEachSubmoduleSorted(M, F: [&](const Module *SubM) {
859 addAllSubmoduleDeps(M: SubM, MD, AddedModules);
860 });
861}
862
863void ModuleDepCollectorPP::addOneModuleDep(const Module *M, const ModuleID ID,
864 ModuleDeps &MD) {
865 MD.ClangModuleDeps.push_back(x: std::move(ID));
866 if (MD.IsInStableDirectories)
867 MD.IsInStableDirectories = MDC.ModularDeps[M]->IsInStableDirectories;
868}
869
870void ModuleDepCollectorPP::addModuleDep(
871 const Module *M, ModuleDeps &MD,
872 llvm::DenseSet<const Module *> &AddedModules) {
873 for (const Module *Import : M->Imports) {
874 if (Import->getTopLevelModule() != M->getTopLevelModule() &&
875 !MDC.isPrebuiltModule(M: Import)) {
876 if (auto ImportID = handleTopLevelModule(M: Import->getTopLevelModule()))
877 if (AddedModules.insert(V: Import->getTopLevelModule()).second)
878 addOneModuleDep(M: Import->getTopLevelModule(), ID: *ImportID, MD);
879 }
880 }
881}
882
883void ModuleDepCollectorPP::addAllAffectingClangModules(
884 const Module *M, ModuleDeps &MD,
885 llvm::DenseSet<const Module *> &AddedModules) {
886 addAffectingClangModule(M, MD, AddedModules);
887
888 for (const Module *SubM : M->submodules())
889 addAllAffectingClangModules(M: SubM, MD, AddedModules);
890}
891
892void ModuleDepCollectorPP::addAffectingClangModule(
893 const Module *M, ModuleDeps &MD,
894 llvm::DenseSet<const Module *> &AddedModules) {
895 for (const Module *Affecting : M->AffectingClangModules) {
896 assert(Affecting == Affecting->getTopLevelModule() &&
897 "Not quite import not top-level module");
898 if (Affecting != M->getTopLevelModule() &&
899 !MDC.isPrebuiltModule(M: Affecting)) {
900 if (auto ImportID = handleTopLevelModule(M: Affecting))
901 if (AddedModules.insert(V: Affecting).second)
902 addOneModuleDep(M: Affecting, ID: *ImportID, MD);
903 }
904 }
905}
906
907ModuleDepCollector::ModuleDepCollector(
908 DependencyScanningService &Service,
909 std::unique_ptr<DependencyOutputOptions> Opts,
910 CompilerInstance &ScanInstance, DependencyConsumer &C,
911 DependencyActionController &Controller, CompilerInvocation OriginalCI,
912 const PrebuiltModulesAttrsMap PrebuiltModulesASTMap,
913 const ArrayRef<StringRef> StableDirs)
914 : Service(Service), ScanInstance(ScanInstance), Consumer(C),
915 Controller(Controller),
916 PrebuiltModulesASTMap(std::move(PrebuiltModulesASTMap)),
917 StableDirs(StableDirs), Opts(std::move(Opts)),
918 CommonInvocation(
919 makeCommonInvocationForModuleBuild(CI: std::move(OriginalCI))) {}
920
921void ModuleDepCollector::attachToPreprocessor(Preprocessor &PP) {
922 auto CollectorPP = std::make_unique<ModuleDepCollectorPP>(args&: *this);
923 CollectorPPPtr = CollectorPP.get();
924 PP.addPPCallbacks(C: std::move(CollectorPP));
925}
926
927void ModuleDepCollector::attachToASTReader(ASTReader &R) {}
928
929bool ModuleDepCollector::isPrebuiltModule(const Module *M) {
930 std::string Name(M->getTopLevelModuleName());
931 const auto &PrebuiltModuleFiles =
932 ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles;
933 auto PrebuiltModuleFileIt = PrebuiltModuleFiles.find(x: Name);
934 if (PrebuiltModuleFileIt == PrebuiltModuleFiles.end())
935 return false;
936 assert("Prebuilt module came from the expected AST file" &&
937 PrebuiltModuleFileIt->second == M->getASTFile()->getName());
938 return true;
939}
940
941void ModuleDepCollector::addVisibleModules() {
942 llvm::DenseSet<const Module *> ImportedModules;
943 auto InsertVisibleModules = [&](const Module *M) {
944 if (ImportedModules.contains(V: M))
945 return;
946
947 VisibleModules.insert(key: M->getTopLevelModuleName());
948 SmallVector<Module *> Stack;
949 M->getExportedModules(Exported&: Stack);
950 while (!Stack.empty()) {
951 const Module *CurrModule = Stack.pop_back_val();
952 if (ImportedModules.contains(V: CurrModule))
953 continue;
954 ImportedModules.insert(V: CurrModule);
955 VisibleModules.insert(key: CurrModule->getTopLevelModuleName());
956 CurrModule->getExportedModules(Exported&: Stack);
957 }
958 };
959
960 for (const Module *Import : DirectImports)
961 InsertVisibleModules(Import);
962}
963
964void ModuleDepCollector::addFileDep(StringRef Path) {
965 if (!Service.getOpts().ReportAbsolutePaths) {
966 FileDeps.emplace_back(args&: Path);
967 return;
968 }
969
970 llvm::SmallString<256> Storage;
971 Path = makeAbsoluteAndCanonicalize(CI&: ScanInstance, Path, Storage);
972 FileDeps.emplace_back(args&: Path);
973}
974
975void ModuleDepCollector::addFileDep(ModuleDeps &MD, StringRef Path) {
976 MD.FileDeps.emplace_back(args&: Path);
977}
978